diff --git a/RELEASES.md b/RELEASES.md index eb661556..2ba500ee 100644 --- a/RELEASES.md +++ b/RELEASES.md @@ -94,6 +94,7 @@ historical records) are documented in: | Cloud + browser storage adapters (`s3` / `gcs` / `r2` / `opfs` storage types, Azure Blob, plus the `storage.branch` option) | `filesystem` and `memory` only. Cloud backup is now an operator concern: `db.persist()` produces a plain directory — sync it with `gsutil` / `aws s3 cp` / `rclone` / `azcopy`. Passing a removed storage type throws at construction with this exact guidance. | | Browser support | Node.js 22 LTS or Bun ≥ 1.0 (`engines` enforces `node: 22.x`); Deno works through its Node compatibility layer. | | `brain.migrateToDiskAnn()` / `brain.migrateToHnsw()` | Gone — there is one canonical `'vector'` provider slot. A registered native vector provider selects its own operating mode; there is nothing to call. | +| Distributed-clustering subsystem — `config.distributed`, the `DistributedRole` enum, the 13 `BRAINY_*` cluster env vars (`BRAINY_DISTRIBUTED`, `BRAINY_ROLE`, `BRAINY_HTTP_PORT`, `BRAINY_WS_PORT`, `BRAINY_DNS`, `BRAINY_SERVICE`, `BRAINY_NAMESPACE`, `BRAINY_CONSENSUS`, `BRAINY_COORDINATOR`, `BRAINY_NODES`, `BRAINY_REPLICAS`, `BRAINY_SHARDS`, `BRAINY_TRANSPORT`), the storage `setDistributedComponents` hook, and the `@soulcraft/brainy/config` preset/augmentation registry | Removed. Brainy 8.0 is a single-process library — there is no coordinator, peer discovery, or consensus to operate. Scale via: the optional native provider (`@soulcraft/cortex` — on-disk DiskANN to 10B+ vectors on one machine); per-tenant pools (one instance + storage dir per tenant); and horizontal read scaling (many reader processes against one shared store, single writer). The `mode: 'reader' \| 'writer'` multi-process roles are unchanged. | | CLI `fork` / `branch` / `checkout` / `migrate` | CLI `snapshot ` / `restore ` / `history` / `generation`. | If you used 7.x COW branches, **materialize every branch you care about while diff --git a/docs/EXTENDING_STORAGE.md b/docs/EXTENDING_STORAGE.md deleted file mode 100644 index 8ecc700a..00000000 --- a/docs/EXTENDING_STORAGE.md +++ /dev/null @@ -1,396 +0,0 @@ -# 🔌 Extending Brainy Storage with Augmentations - -## Overview - -Brainy's zero-config system is **fully extensible**. Augmentations can register new storage providers, presets, and auto-detection logic that integrates seamlessly with the existing system. - -## How Storage Extensions Work - -### 1. Storage Provider Registration - -When an augmentation is installed, it can register a new storage provider: - -```typescript -import { StorageProvider, registerStorageAugmentation } from '@soulcraft/brainy/config' - -const redisProvider: StorageProvider = { - type: 'redis', - name: 'Redis Storage', - description: 'High-performance in-memory data store', - priority: 10, // Higher priority = checked first in auto-detection - - // Auto-detection logic - async detect(): Promise { - // Check if Redis is available - if (process.env.REDIS_URL) { - try { - const redis = await import('ioredis') - const client = new redis.default(process.env.REDIS_URL) - await client.ping() - await client.quit() - return true - } catch { - return false - } - } - return false - }, - - // Configuration builder - async getConfig(): Promise { - return { - type: 'redis', - redisStorage: { - url: process.env.REDIS_URL, - prefix: 'brainy:', - ttl: 3600 - } - } - } -} - -// Register the provider -registerStorageAugmentation(redisProvider) -``` - -### 2. Using Extended Storage - -Once registered, the new storage type works with zero-config: - -```typescript -// Auto-detection will now check Redis -const brain = new Brainy() // Will use Redis if available! - -// Or explicitly specify -const brain = new Brainy({ storage: 'redis' }) - -// Or with custom config -const brain = new Brainy({ - storage: { - type: 'redis', - redisStorage: { - url: 'redis://localhost:6379', - prefix: 'myapp:' - } - } -}) -``` - -## Real-World Examples - -### Redis Augmentation - -```typescript -// @soulcraft/brainy-redis package -export class RedisStorageAugmentation { - async init() { - // Register the storage provider - registerStorageAugmentation({ - type: 'redis', - name: 'Redis Storage', - priority: 10, - - async detect() { - return !!(process.env.REDIS_URL || process.env.REDIS_HOST) - }, - - async getConfig() { - return { - type: 'redis', - redisStorage: { - url: process.env.REDIS_URL || - `redis://${process.env.REDIS_HOST}:${process.env.REDIS_PORT || 6379}` - } - } - } - }) - - // Register Redis-specific presets - registerPresetAugmentation('redis-cache', { - storage: 'redis', - model: ModelPrecision.Q8, - features: ['core', 'cache'], - distributed: true, - description: 'Redis-backed cache layer', - category: PresetCategory.SERVICE - }) - } -} -``` - -### MongoDB Augmentation - -```typescript -// @soulcraft/brainy-mongodb package -export class MongoStorageAugmentation { - async init() { - registerStorageAugmentation({ - type: 'mongodb', - name: 'MongoDB Storage', - priority: 8, - - async detect() { - return !!(process.env.MONGODB_URI || process.env.MONGO_URL) - }, - - async getConfig() { - return { - type: 'mongodb', - mongoStorage: { - uri: process.env.MONGODB_URI, - database: 'brainy', - collection: 'vectors' - } - } - } - }) - } -} -``` - -### PostgreSQL + pgvector Augmentation - -```typescript -// @soulcraft/brainy-postgres package -export class PostgresStorageAugmentation { - async init() { - registerStorageAugmentation({ - type: 'postgres', - name: 'PostgreSQL + pgvector', - priority: 9, - - async detect() { - const url = process.env.DATABASE_URL - if (url?.includes('postgres')) { - // Check for pgvector extension - const client = new Client({ connectionString: url }) - await client.connect() - const result = await client.query( - "SELECT * FROM pg_extension WHERE extname = 'vector'" - ) - await client.end() - return result.rows.length > 0 - } - return false - }, - - async getConfig() { - return { - type: 'postgres', - postgresStorage: { - connectionString: process.env.DATABASE_URL, - table: 'brainy_vectors' - } - } - } - }) - } -} -``` - -## Auto-Detection Priority - -Storage providers are checked in priority order: - -1. **Custom providers** (highest priority first) -2. **Cloud storage** (S3, GCS, R2) -3. **Database storage** (Redis, MongoDB, PostgreSQL) -4. **Local storage** (filesystem, OPFS) -5. **Memory** (fallback) - -```typescript -// Example priority chain -Redis (priority: 10) → PostgreSQL (9) → MongoDB (8) → S3 (5) → Filesystem (1) → Memory (0) -``` - -## Creating Custom Presets - -Augmentations can also register new presets: - -```typescript -registerPresetAugmentation('redis-cluster', { - storage: 'redis', - model: ModelPrecision.Q8, - features: ['core', 'cache', 'cluster'], - distributed: true, - role: DistributedRole.HYBRID, - cache: { - hotCacheMaxSize: 100000, // Large distributed cache - autoTune: true - }, - description: 'Redis Cluster configuration', - category: PresetCategory.SERVICE -}) - -// Users can then use: -const brain = new Brainy('redis-cluster') -``` - -## Type Safety with Extensions - -To maintain type safety with dynamic storage types: - -```typescript -// Augmentation declares its types -declare module '@soulcraft/brainy' { - interface StorageTypes { - redis: { - url: string - prefix?: string - ttl?: number - } - } - - interface PresetNames { - 'redis-cache': 'redis-cache' - 'redis-cluster': 'redis-cluster' - } -} -``` - -## Best Practices for Storage Augmentations - -1. **Always provide auto-detection** - Check environment variables and connectivity -2. **Set appropriate priority** - Higher for specialized storage, lower for general -3. **Handle failures gracefully** - Return false from detect() if not available -4. **Document requirements** - List required packages and environment variables -5. **Provide presets** - Include common configuration patterns -6. **Maintain compatibility** - Ensure model precision matches across instances - -## Example: Complete Redis Augmentation - -```typescript -import { - StorageProvider, - registerStorageAugmentation, - registerPresetAugmentation, - PresetCategory, - ModelPrecision, - DistributedRole -} from '@soulcraft/brainy/config' -import Redis from 'ioredis' - -export class BrainyRedisAugmentation { - private client: Redis - - async init() { - // Register storage provider - registerStorageAugmentation({ - type: 'redis', - name: 'Redis Vector Storage', - description: 'Redis with RediSearch for vector similarity', - priority: 10, - - requirements: { - env: ['REDIS_URL'], - packages: ['ioredis', 'redis'] - }, - - async detect() { - if (!process.env.REDIS_URL) return false - - try { - const client = new Redis(process.env.REDIS_URL) - - // Check for RediSearch module - const modules = await client.call('MODULE', 'LIST') - const hasRediSearch = modules.some(m => m[1] === 'search') - - await client.quit() - return hasRediSearch - } catch { - return false - } - }, - - async getConfig() { - return { - type: 'redis', - redisStorage: { - url: process.env.REDIS_URL, - prefix: process.env.REDIS_PREFIX || 'brainy:', - index: process.env.REDIS_INDEX || 'brainy-vectors', - ttl: process.env.REDIS_TTL ? parseInt(process.env.REDIS_TTL) : undefined - } - } - } - }) - - // Register presets - this.registerPresets() - } - - private registerPresets() { - // Fast cache preset - registerPresetAugmentation('redis-fast-cache', { - storage: 'redis' as any, - model: ModelPrecision.Q8, - features: ['core', 'cache', 'search'], - distributed: false, - cache: { - hotCacheMaxSize: 10000, - autoTune: true - }, - description: 'Redis-backed fast cache', - category: PresetCategory.SERVICE - }) - - // Distributed cache preset - registerPresetAugmentation('redis-distributed', { - storage: 'redis' as any, - model: ModelPrecision.AUTO, - features: ['core', 'cache', 'search', 'cluster'], - distributed: true, - role: DistributedRole.HYBRID, - cache: { - hotCacheMaxSize: 50000, - autoTune: true - }, - description: 'Redis distributed cache cluster', - category: PresetCategory.SERVICE - }) - - // Session store preset - registerPresetAugmentation('redis-sessions', { - storage: 'redis' as any, - model: ModelPrecision.Q8, - features: ['core', 'cache'], - distributed: false, - cache: { - hotCacheMaxSize: 5000, - autoTune: false - }, - description: 'Redis session storage', - category: PresetCategory.SERVICE - }) - } -} - -// Usage after installing the augmentation: -import { Brainy } from '@soulcraft/brainy' -import '@soulcraft/brainy-redis' // Registers the augmentation - -// Now Redis is automatically detected! -const brain = new Brainy() // Uses Redis if REDIS_URL is set - -// Or use a Redis preset -const brain = new Brainy('redis-fast-cache') - -// Or explicitly configure -const brain = new Brainy({ - storage: 'redis', - model: ModelPrecision.FP32 -}) -``` - -## Summary - -The extensible configuration system allows: - -1. **New storage types** via `registerStorageAugmentation()` -2. **Custom presets** via `registerPresetAugmentation()` -3. **Auto-detection logic** that integrates with zero-config -4. **Type-safe extensions** with TypeScript declarations -5. **Priority-based selection** for intelligent defaults - -This ensures Brainy can grow with new storage technologies while maintaining its zero-configuration philosophy! \ No newline at end of file diff --git a/docs/MIGRATION-V3-TO-V4.md b/docs/MIGRATION-V3-TO-V4.md index 069bda73..29c409ac 100644 --- a/docs/MIGRATION-V3-TO-V4.md +++ b/docs/MIGRATION-V3-TO-V4.md @@ -426,16 +426,6 @@ npm install @soulcraft/brainy@^3.50.0 7. Verify data integrity thoroughly 8. Enable lifecycle policies gradually -### Scenario 4: Multi-Node Distributed System - -**Recommended approach:** -1. Perform blue-green deployment: - - Keep v3 nodes running (blue) - - Deploy v4 nodes (green) - - Migrate data once - - Switch traffic to v4 nodes - - Decommission v3 nodes - ## Cost Savings After Migration ### Enable All v4.0.0 Features diff --git a/docs/README.md b/docs/README.md index 3abab230..3290001f 100644 --- a/docs/README.md +++ b/docs/README.md @@ -84,7 +84,6 @@ See [vfs/](./vfs/) for the complete VFS documentation set. | Document | Description | |----------|-------------| | [Storage Architecture](./architecture/storage-architecture.md) | Filesystem and memory adapters, on-disk artifact layout, operator-layer backup | -| [Extending Storage](./EXTENDING_STORAGE.md) | Create custom storage adapters | | [Capacity Planning](./operations/capacity-planning.md) | Scale to millions of entities | --- diff --git a/docs/architecture/augmentations-actual.md b/docs/architecture/augmentations-actual.md index 808c37aa..81ace98d 100644 --- a/docs/architecture/augmentations-actual.md +++ b/docs/architecture/augmentations-actual.md @@ -73,10 +73,10 @@ import { MemoryStorageAugmentation } from 'brainy' ``` ### 11. Server Search Augmentation ✅ -Distributed search capabilities. +Server-side search delegation over a conduit. ```typescript import { ServerSearchConduitAugmentation } from 'brainy' -// Distributed query execution +// Forwards queries to a remote Brainy server ``` ### 12. Neural Import Augmentation ✅ @@ -100,7 +100,7 @@ await neuralImport.detectRelationships(entities) await neuralImport.generateInsights(data) ``` -### Distributed Operation Modes (Fully Implemented!) +### Operation Modes (Fully Implemented!) ```typescript // Read-only mode with optimized caching const readerMode = new ReaderMode() @@ -239,15 +239,10 @@ const cacheConfig = await getCacheAutoConfig() ## 🎨 How to Use Hidden Features -### Enable Distributed Modes +### Enable Reader / Writer Modes ```typescript const brain = new Brainy({ - mode: 'reader', // or 'writer' or 'hybrid' - distributed: { - role: 'reader', - cacheStrategy: 'aggressive', - prefetch: true - } + mode: 'reader' // or 'writer' or 'hybrid' }) ``` @@ -285,7 +280,7 @@ const freshStats = await brain.getStatistics({ ## 📝 What Needs Documentation These features EXIST but need better docs: -1. Distributed operation modes +1. Reader / writer operation modes 2. Neural import full API 3. 3-level cache configuration 4. Performance monitoring API @@ -297,7 +292,7 @@ These features EXIST but need better docs: ## 💡 The Truth Brainy is MORE powerful than its own documentation suggests! Most "missing" features are actually implemented but hidden or not properly exposed. The codebase contains sophisticated systems for: -- Distributed operations +- Reader / writer operation modes - AI-powered import - Advanced caching - Performance monitoring diff --git a/docs/architecture/storage-architecture.md b/docs/architecture/storage-architecture.md index 0c876638..616502c7 100644 --- a/docs/architecture/storage-architecture.md +++ b/docs/architecture/storage-architecture.md @@ -291,8 +291,8 @@ console.log(stats) ### Choose the Right Adapter 1. **Development & tests**: `memory` for speed, `filesystem` when you need persistence -2. **Single-node production**: `filesystem` with off-site backup via `gsutil` / `aws s3 sync` / `rclone` -3. **Multi-node production**: Run Brainy behind a service layer; replicate the on-disk artifact via your operator tooling +2. **Single-process production**: `filesystem` with off-site backup via `gsutil` / `aws s3 sync` / `rclone` +3. **Horizontal scaling**: Brainy runs in one process — there is no built-in cluster. Run independent instances behind a service layer and replicate the on-disk artifact with your operator tooling; or run many reader processes against one shared store with a single writer ### Optimize for Your Use Case 1. **Read-heavy**: Enable caching and let the OS page cache do its job diff --git a/docs/features/complete-feature-list.md b/docs/features/complete-feature-list.md index b936ad71..7b10b827 100644 --- a/docs/features/complete-feature-list.md +++ b/docs/features/complete-feature-list.md @@ -66,7 +66,7 @@ import { BatchProcessingAugmentation } from 'brainy' ```typescript import { ConnectionPoolAugmentation } from 'brainy' // Auto-scaling connection management -// Optimized for distributed operations +// Optimized for high-concurrency workloads ``` ### 7. Request Deduplicator ✅ @@ -100,8 +100,7 @@ import { MemoryStorageAugmentation } from 'brainy' ### 11. Server Search Conduit ✅ ```typescript import { ServerSearchConduitAugmentation } from 'brainy' -// Distributed query execution -// Load balancing across nodes +// Forwards queries to a remote Brainy server ``` ### 12. Neural Import ✅ @@ -155,7 +154,9 @@ await brain.init() - **No environment variables needed** - **Works in all environments** (Node, Browser, Workers) -## 🏢 Distributed Operation Modes +## 🏢 Operation Modes + +Run many reader processes against one shared on-disk store with a single writer. ### Reader Mode ✅ ```typescript @@ -374,7 +375,7 @@ await brain.init() - ✅ All engines (vector, graph, field, neural) - ✅ All augmentations (12+) - ✅ All storage adapters -- ✅ All distributed modes +- ✅ Reader / writer / hybrid operation modes - ✅ Complete statistics - ✅ GPU support - ✅ No feature limitations diff --git a/docs/features/v3-features.md b/docs/features/v3-features.md index 987ae7c4..51125855 100644 --- a/docs/features/v3-features.md +++ b/docs/features/v3-features.md @@ -34,58 +34,6 @@ brain.add({ name: "John", email: "john@example.com" }, 'entity') - Automatic query optimization - Pattern-based query rewriting -## 🏢 Enterprise Features - -### Distributed Coordination ✅ -Raft consensus for multi-node deployments: -```typescript -import { DistributedCoordinator } from '@soulcraft/brainy' - -const coordinator = createCoordinator({ - nodeId: 'node-1', - peers: ['node-2', 'node-3'], - electionTimeout: 500 -}) -// Automatic leader election and failover -``` - -### Horizontal Sharding ✅ -Consistent hashing for data distribution: -```typescript -import { ShardManager } from '@soulcraft/brainy' - -const shards = createShardManager({ - nodes: ['node-1', 'node-2', 'node-3'], - replicationFactor: 2, - virtualNodes: 150 -}) -// Automatic shard rebalancing on node changes -``` - -### Read/Write Separation ✅ -Primary-replica architecture for scale: -```typescript -import { ReadWriteSeparation } from '@soulcraft/brainy' - -const replication = createReadWriteSeparation({ - role: 'auto', // Automatic primary/replica detection - consistencyLevel: 'strong', // or 'eventual' - readPreference: 'nearest' -}) -``` - -### Cross-Instance Cache Sync ✅ -Version vector-based cache synchronization: -```typescript -import { CacheSync } from '@soulcraft/brainy' - -const cache = createCacheSync({ - nodeId: 'node-1', - syncInterval: 100, - conflictResolution: 'version-vector' -}) -``` - ## 🔐 Security & Compliance ### Rate Limiting ✅ @@ -281,14 +229,12 @@ await brain.restore('backup.bin') - **10,000+ items**: Sub-10ms search - **1M+ operations**: Stable memory usage - **100+ concurrent users**: No performance degradation -- **Multi-node clusters**: Automatic failover ## 🚫 NOT Implemented (Planned) These features are documented but NOT yet implemented: - GraphQL API (use REST API instead) - Kubernetes operators (use Docker) -- Some distributed features require manual configuration --- diff --git a/docs/guides/enterprise-for-everyone.md b/docs/guides/enterprise-for-everyone.md index c970d2f3..de099d04 100644 --- a/docs/guides/enterprise-for-everyone.md +++ b/docs/guides/enterprise-for-everyone.md @@ -165,32 +165,28 @@ await brain.syncWith({ - **Webhook support**: React to changes - **API generation**: Auto-generate REST/GraphQL APIs -### 🌍 Enterprise Scale 🚧 Coming Soon +### 🌍 Scale -**Everyone gets planetary scale:** +**Everyone gets the same scale model:** ```typescript -// Same architecture Netflix uses, free for you -const brain = new Brainy({ - clustering: { - enabled: true, // Distributed mode - sharding: 'automatic', // Auto-sharding - replication: 3, // Triple replication - consensus: 'raft', // Strong consistency - geoDistribution: true // Multi-region support - } -}) +// Pure JS by default; install the optional native provider for billions of vectors +const brain = new Brainy() -// Handles everything from 1 to 1 billion entities +// 1 → ~1M vectors: pure-JS HNSW, zero extra setup +// 1M → 10B+ vectors: install @soulcraft/cortex for the native DiskANN provider ``` -**Scaling features:** -- **Horizontal scaling**: Add nodes as needed -- **Auto-sharding**: Distributes data automatically -- **Multi-region**: Global distribution -- **Load balancing**: Automatic request distribution -- **Zero-downtime upgrades**: Rolling updates -- **Infinite scale**: No upper limits +**Scaling model:** +- **Single process, no cluster**: Brainy runs in one process — no coordinator, + no peer discovery, no consensus to operate +- **Optional native provider**: install `@soulcraft/cortex` to back the index with + on-disk DiskANN that scales to 10B+ vectors on one machine +- **Per-tenant pools**: isolate tenants by giving each its own Brainy instance and + storage directory +- **Horizontal read scaling**: run many reader processes against one shared on-disk + store (single writer, many readers); replicate the artifact with your operator + tooling ### 🛡️ Enterprise Compliance 🚧 Coming Soon diff --git a/docs/operations/capacity-planning.md b/docs/operations/capacity-planning.md index 263b37c0..11d061ff 100644 --- a/docs/operations/capacity-planning.md +++ b/docs/operations/capacity-planning.md @@ -219,7 +219,6 @@ Actual cache size: ~40GB (prevents waste on 128GB systems) **Recommendations:** - ✅ Use FP32 model for maximum accuracy -- ✅ Enable distributed storage (S3/GCS) - ✅ Monitor fairness violations (HNSW shouldn't dominate cache) - ✅ Consider sharding beyond 50M entities - ✅ Implement application-level caching for hot queries diff --git a/docs/transactions.md b/docs/transactions.md index 3ab1ff66..4bfde262 100644 --- a/docs/transactions.md +++ b/docs/transactions.md @@ -458,7 +458,6 @@ describe('Transaction Tests', () => { See `tests/transaction/integration/` for comprehensive integration tests covering: - Sharding integration (`sharding-transactions.test.ts`) - Type-aware integration (`typeaware-transactions.test.ts`) -- Distributed scenarios (`distributed-transactions.test.ts`) The atomicity guarantees of `brain.transact()` — including crash recovery through the real recovery path — are proven in diff --git a/docs/vfs/README.md b/docs/vfs/README.md index d0dac6ca..1e716665 100644 --- a/docs/vfs/README.md +++ b/docs/vfs/README.md @@ -332,7 +332,7 @@ const urgent = await vfs.search('', { ## Advanced Features -> **Note:** See [VFS ROADMAP](./ROADMAP.md) for planned advanced features like version history, distributed filesystem, and more. +> **Note:** See [VFS ROADMAP](./ROADMAP.md) for planned advanced features like version history and more. ## Integration Possibilities @@ -351,7 +351,6 @@ Brainy VFS is designed for speed and scale: **PROJECTED at larger scales (not yet tested):** - **Vector search** <100ms for millions of files (projected) - **Streaming support** for files of any size (architecture supports, see [limitations in ROADMAP](./ROADMAP.md)) -- **Distributed sharding** for billions of files (architecture supports, not tested at scale) See tests in `tests/vfs/` for actual measured performance. @@ -377,7 +376,7 @@ Brainy VFS fully leverages Brainy's revolutionary Triple Intelligence system: | Manual organization | Self-organizing with intelligent fusion | | No relationships | Rich knowledge graph with traversal | | Static metadata | Dynamic, queryable metadata with field intelligence | -| Single server | Distributed & federated | +| Manual scaling | Horizontal read scaling — many readers, one writer | ## Installation @@ -438,10 +437,10 @@ The VFS is built with production scalability in mind: - Parent-child relationship caching - Hot path detection and optimization -2. **Distributed Architecture** - - Sharding by path prefix - - Read replicas for hot directories - - CDN integration for static files +2. **Horizontal Read Scaling** + - On-disk store partitioned into 256 hex directory buckets + - Many reader processes against one shared store + - Single writer keeps the store consistent 3. **Intelligent Indexing** - Compound indexes on (parent, name) @@ -589,49 +588,6 @@ vfs.on('file:added', async (path) => { }) ``` -#### **5. Distributed Team Workspace** - -Collaborative file management: - -```javascript -// Track file ownership and access -await vfs.writeFile('/projects/alpha/spec.md', content, { - metadata: { - owner: userId, - team: 'engineering', - permissions: { - [userId]: 'rw', - 'team:engineering': 'r', - 'others': '-' - } - } -}) - -// Add collaborative features -await vfs.addTodo('/projects/alpha/spec.md', { - task: 'Review security section', - assignee: 'alice@company.com', - due: '2024-02-01', - priority: 'high' -}) - -// Track who's working on what -await vfs.setxattr('/projects/alpha/spec.md', 'locks', { - section3: { - user: 'bob@company.com', - since: Date.now() - } -}) - -// Find all files assigned to a user -const assigned = await vfs.search('', { - where: { - 'todos.assignee': 'alice@company.com', - 'todos.status': 'pending' - } -}) -``` - ### Monitoring & Operations (Planned) > **Note:** Production monitoring features are planned for v1.1. See [ROADMAP](./ROADMAP.md). @@ -665,27 +621,11 @@ await vfs.init({ }) ``` -#### **Distributed Cluster** -```javascript -const vfs = new VirtualFileSystem() -await vfs.init({ - distributed: true, - nodes: [ - 'vfs1.internal:8080', - 'vfs2.internal:8080', - 'vfs3.internal:8080' - ], - replication: 3, - consistency: 'eventual' -}) -``` - ## Roadmap & Future Features See [VFS ROADMAP](./ROADMAP.md) for planned features including: - Enhanced streaming support (v1.1) - Version history (v1.2) -- Distributed filesystem (v1.2) - AI-powered automation (v2.0) - FUSE driver (v2.0 research) - And more community-requested features diff --git a/docs/vfs/ROADMAP.md b/docs/vfs/ROADMAP.md index 14833def..93c5b901 100644 --- a/docs/vfs/ROADMAP.md +++ b/docs/vfs/ROADMAP.md @@ -65,27 +65,6 @@ const diff = await vfs.diffVersions('/important-doc.md', v1, v2) **Status:** Planned **Effort:** 4-5 weeks -### Distributed Filesystem -Mount remote Brainy instances for federated file access. - -```typescript -// Planned API (not yet implemented) -await vfs.mount('/remote-team', { - type: 'brainy-remote', - url: 'https://team-brainy.example.com', - credentials: {...} -}) - -// Federated search across mounted instances -const results = await vfs.search('project docs', { includeMounted: true }) - -// Sync directories -await vfs.sync('/local/docs', '/remote-team/docs') -``` - -**Status:** Planned -**Effort:** 6-8 weeks - ### Backup & Recovery API Built-in backup and recovery operations. @@ -215,12 +194,6 @@ Vite integration with VFS for semantic module resolution. ## Far Future (v5.0+) -### Automatic Node Discovery -Zero-config multi-node setup with automatic discovery via UDP broadcast, Kubernetes DNS, or cloud provider APIs. - -### Automatic Failover -Health monitoring and automatic failover in distributed deployments. - ### Cloud Provider Auto-Detection ```typescript // Far future concept diff --git a/docs/vfs/VFS_CORE.md b/docs/vfs/VFS_CORE.md index 892ee819..1eeaf9f8 100644 --- a/docs/vfs/VFS_CORE.md +++ b/docs/vfs/VFS_CORE.md @@ -358,7 +358,6 @@ VFS scales to millions of files: - O(1) path lookup with caching - Efficient graph traversal for directories - Chunked storage for large files -- Distributed storage backend support - Vector search scales with HNSW index ## Method Availability diff --git a/src/brainy.ts b/src/brainy.ts index 15e4fc19..42dea76f 100644 --- a/src/brainy.ts +++ b/src/brainy.ts @@ -72,14 +72,10 @@ import { DeleteVerbMetadataOperation } from './transaction/operations/index.js' import { - DistributedCoordinator, - ShardManager, - CacheSync, - ReadWriteSeparation, BaseOperationalMode, ReaderMode, HybridMode -} from './distributed/index.js' +} from './storage/operationalModes.js' import { Entity, Relation, @@ -199,7 +195,6 @@ interface StorageBackgroundInitHooks { type ResolvedBrainyConfig = Required< Omit< BrainyConfig, - | 'distributed' | 'maxQueryLimit' | 'reservedQueryMemory' | 'vector' @@ -211,7 +206,6 @@ type ResolvedBrainyConfig = Required< > & Pick< BrainyConfig, - | 'distributed' | 'maxQueryLimit' | 'reservedQueryMemory' | 'vector' @@ -329,12 +323,6 @@ export class Brainy implements BrainyInterface { /** Cap for {@link Brainy.verbIntWarmCache} (~100k entries ≈ a few MB). */ private static readonly VERB_INT_WARM_CACHE_MAX = 100_000 - // Distributed components (optional) - private coordinator?: DistributedCoordinator - private shardManager?: ShardManager - private cacheSync?: CacheSync - private readWriteSeparation?: ReadWriteSeparation - // Silent mode state private originalConsole?: { log: typeof console.log @@ -414,9 +402,9 @@ export class Brainy implements BrainyInterface { this.config = this.normalizeConfig(config) // Multi-process mode — default 'writer' uses HybridMode (read + write), - // 'reader' uses ReaderMode (read-only, all mutations throw). - // `WriterMode` from operationalModes.ts blocks reads, which is too strict - // for a Brainy instance — a writer needs to read its own data. + // 'reader' uses ReaderMode (read-only, all mutations throw). A writer needs + // to read its own data, so the writer role is read-write rather than + // write-only. this.operationalMode = this.config.mode === 'reader' ? new ReaderMode() : new HybridMode() @@ -435,11 +423,6 @@ export class Brainy implements BrainyInterface { this.embedder = this.setupEmbedder() this.transactionManager = new TransactionManager() - // Setup distributed components if enabled - if (this.config.distributed?.enabled) { - this.setupDistributedComponents() - } - // Initialize ready Promise // This allows consumers to await brain.ready before using the database this._readyPromise = new Promise((resolve, reject) => { @@ -846,9 +829,6 @@ export class Brainy implements BrainyInterface { // Check for pending data migrations await this.checkMigrations() - // Connect distributed components to storage - await this.connectDistributedStorage() - // Register shutdown hooks for graceful count flushing (once globally) if (!Brainy.shutdownHooksRegisteredGlobally) { this.registerShutdownHooks() @@ -10511,13 +10491,8 @@ export class Brainy implements BrainyInterface { ) } - // Distributed mode is explicit opt-in only (config or BRAINY_DISTRIBUTED - // env var) — never inferred from deployment heuristics. - const distributedConfig = this.resolveDistributedConfig(config?.distributed) - return { storage: config?.storage || { type: 'auto' }, - distributed: distributedConfig, verbose: config?.verbose ?? false, silent: config?.silent ?? false, // false = auto-decide based on dataset size (inline vs lazy rebuild) @@ -11144,140 +11119,6 @@ export class Brainy implements BrainyInterface { // operation (ensureInitialized() throws once this is set). this.closed = true } - - /** - * Intelligently auto-detect distributed configuration - * Zero-config: Automatically determines best distributed settings - */ - /** - * Resolve distributed-cluster configuration. **Explicit opt-in only**: - * either `config.distributed.enabled === true` or the - * `BRAINY_DISTRIBUTED=true` environment variable. Once opted in, the - * remaining fields default sensibly (64 shards, 3 replicas, raft, http, - * hostname-derived nodeId). - * - * Earlier versions auto-enabled cluster mode from deployment heuristics - * (`NODE_ENV=production`, a Kubernetes service host, `CLUSTER_SIZE`). - * That violated zero-config safety: a single-process production - * deployment — the overwhelmingly common case — would silently start a - * coordinator, bind a port, and run raft against itself. Deployment - * environment does not imply a multi-node Brainy cluster, so the - * heuristics are gone. - */ - private resolveDistributedConfig(config?: BrainyConfig['distributed']): BrainyConfig['distributed'] { - // Env-var opt-in for deployments that can't touch code (the ONLY - // environment signal honoured — it names brainy explicitly). - if (!config && process.env.BRAINY_DISTRIBUTED === 'true') { - return { - enabled: true, - nodeId: process.env.HOSTNAME || process.env.NODE_ID || `node-${Date.now()}`, - nodes: process.env.BRAINY_NODES?.split(',') || [], - coordinatorUrl: process.env.BRAINY_COORDINATOR || undefined, - shardCount: parseInt(process.env.BRAINY_SHARDS || '64'), - replicationFactor: parseInt(process.env.BRAINY_REPLICAS || '3'), - consensus: (process.env.BRAINY_CONSENSUS as 'raft' | 'none' | undefined) || 'raft', - transport: (process.env.BRAINY_TRANSPORT as 'tcp' | 'http' | 'udp' | undefined) || 'http' - } - } - - // Explicit config: fill the per-field defaults. - return config ? { - ...config, - nodeId: config.nodeId || process.env.HOSTNAME || `node-${Date.now()}`, - shardCount: config.shardCount || 64, - replicationFactor: config.replicationFactor || 3, - consensus: config.consensus || 'raft', - transport: config.transport || 'http' - } : undefined - } - - /** - * Setup distributed components with zero-config intelligence - */ - private setupDistributedComponents(): void { - const distConfig = this.config.distributed - if (!distConfig?.enabled) return - - console.log('🌍 Initializing distributed mode:', { - nodeId: distConfig.nodeId, - shards: distConfig.shardCount, - replicas: distConfig.replicationFactor - }) - - // Initialize coordinator for consensus - this.coordinator = new DistributedCoordinator({ - nodeId: distConfig.nodeId, - address: distConfig.coordinatorUrl?.split(':')[0] || 'localhost', - port: parseInt(distConfig.coordinatorUrl?.split(':')[1] || '8080'), - nodes: distConfig.nodes - }) - - // Start the coordinator to establish leadership - this.coordinator.start().catch(err => { - console.warn('Coordinator start failed (will retry on init):', err.message) - }) - - // Initialize shard manager for data distribution - this.shardManager = new ShardManager({ - shardCount: distConfig.shardCount, - replicationFactor: distConfig.replicationFactor, - virtualNodes: 150, // Optimal for consistent distribution - autoRebalance: true - }) - - // Initialize cache synchronization - this.cacheSync = new CacheSync({ - nodeId: distConfig.nodeId!, - syncInterval: 1000 - }) - - // Initialize read/write separation if we have replicas - // Note: Will be properly initialized after coordinator starts - if (distConfig.replicationFactor && distConfig.replicationFactor > 1) { - // Defer creation until coordinator is ready - setTimeout(() => { - this.readWriteSeparation = new ReadWriteSeparation( - { - nodeId: distConfig.nodeId!, - consistencyLevel: 'eventual', - role: 'replica', // Start as replica, will promote if leader - syncInterval: 5000 - }, - this.coordinator!, - this.shardManager!, - this.cacheSync! - ) - }, 100) - } - } - - /** - * Pass distributed components to storage adapter - */ - private async connectDistributedStorage(): Promise { - if (!this.config.distributed?.enabled) return - - // Check if storage supports distributed operations. No in-repo adapter - // implements this hook — it exists for external/plugin adapters, so the - // `in` guard above is the contract and this cast just names its shape. - if ('setDistributedComponents' in this.storage) { - (this.storage as BaseStorage & { - setDistributedComponents(components: { - coordinator?: DistributedCoordinator - shardManager?: ShardManager - cacheSync?: CacheSync - readWriteSeparation?: ReadWriteSeparation - }): void - }).setDistributedComponents({ - coordinator: this.coordinator, - shardManager: this.shardManager, - cacheSync: this.cacheSync, - readWriteSeparation: this.readWriteSeparation - }) - - console.log('✅ Distributed storage connected') - } - } } /** diff --git a/src/config/distributedPresets.ts b/src/config/distributedPresets.ts deleted file mode 100644 index 8a5ae25e..00000000 --- a/src/config/distributedPresets.ts +++ /dev/null @@ -1,357 +0,0 @@ -/** - * Extended Distributed Configuration Presets - * Common patterns for distributed and multi-service architectures - * All strongly typed with enums for compile-time safety - */ - -/** - * Strongly typed enum for preset names - */ -export enum PresetName { - // Basic presets - PRODUCTION = 'production', - DEVELOPMENT = 'development', - MINIMAL = 'minimal', - ZERO = 'zero', - - // Distributed presets - WRITER = 'writer', - READER = 'reader', - - // Service-specific presets - INGESTION_SERVICE = 'ingestion-service', - SEARCH_API = 'search-api', - ANALYTICS_SERVICE = 'analytics-service', - EDGE_CACHE = 'edge-cache', - BATCH_PROCESSOR = 'batch-processor', - STREAMING_SERVICE = 'streaming-service', - ML_TRAINING = 'ml-training', - SIDECAR = 'sidecar' -} - -/** - * Preset categories for organization - */ -export enum PresetCategory { - BASIC = 'basic', - DISTRIBUTED = 'distributed', - SERVICE = 'service' -} - -/** - * Model precision options - */ -export enum ModelPrecision { - FP32 = 'fp32', - Q8 = 'q8', - AUTO = 'auto', - FAST = 'fast', - SMALL = 'small' -} - -/** - * Storage options - */ -export enum StorageOption { - AUTO = 'auto', - MEMORY = 'memory', - DISK = 'disk', - CLOUD = 'cloud' -} - -/** - * Feature set options - */ -export enum FeatureSet { - MINIMAL = 'minimal', - DEFAULT = 'default', - FULL = 'full', - CUSTOM = 'custom' // For custom feature arrays -} - -/** - * Distributed role options - */ -export enum DistributedRole { - WRITER = 'writer', - READER = 'reader', - HYBRID = 'hybrid' -} - -/** - * Preset configuration interface - */ -export interface PresetConfig { - storage: StorageOption - model: ModelPrecision - features: FeatureSet | string[] - distributed: boolean - role?: DistributedRole - readOnly?: boolean - writeOnly?: boolean - allowDirectReads?: boolean - lazyLoadInReadOnlyMode?: boolean - cache?: { - hotCacheMaxSize?: number - autoTune?: boolean - batchSize?: number - } - verbose?: boolean - description: string - category: PresetCategory -} - -/** - * Strongly typed preset configurations - */ -export const PRESET_CONFIGS: Readonly> = { - // Basic presets - [PresetName.PRODUCTION]: { - storage: StorageOption.DISK, - model: ModelPrecision.AUTO, - features: FeatureSet.DEFAULT, - distributed: false, - verbose: false, - description: 'Optimized for production use', - category: PresetCategory.BASIC - }, - - [PresetName.DEVELOPMENT]: { - storage: StorageOption.MEMORY, - model: ModelPrecision.FP32, - features: FeatureSet.FULL, - distributed: false, - verbose: true, - description: 'Optimized for development with verbose logging', - category: PresetCategory.BASIC - }, - - [PresetName.MINIMAL]: { - storage: StorageOption.MEMORY, - model: ModelPrecision.Q8, - features: FeatureSet.MINIMAL, - distributed: false, - verbose: false, - description: 'Minimal footprint configuration', - category: PresetCategory.BASIC - }, - - [PresetName.ZERO]: { - storage: StorageOption.AUTO, - model: ModelPrecision.AUTO, - features: FeatureSet.DEFAULT, - distributed: false, - verbose: false, - description: 'True zero configuration with auto-detection', - category: PresetCategory.BASIC - }, - - // Distributed basic presets - [PresetName.WRITER]: { - storage: StorageOption.AUTO, - model: ModelPrecision.AUTO, - features: FeatureSet.MINIMAL, - distributed: true, - role: DistributedRole.WRITER, - writeOnly: true, - allowDirectReads: true, - verbose: false, - description: 'Write-only instance for distributed setups', - category: PresetCategory.DISTRIBUTED - }, - - [PresetName.READER]: { - storage: StorageOption.AUTO, - model: ModelPrecision.AUTO, - features: FeatureSet.DEFAULT, - distributed: true, - role: DistributedRole.READER, - readOnly: true, - lazyLoadInReadOnlyMode: true, - verbose: false, - description: 'Read-only instance for distributed setups', - category: PresetCategory.DISTRIBUTED - }, - - // Service-specific presets - [PresetName.INGESTION_SERVICE]: { - storage: StorageOption.CLOUD, - model: ModelPrecision.Q8, - features: ['core', 'batch-processing', 'entity-registry'], - distributed: true, - role: DistributedRole.WRITER, - writeOnly: true, - allowDirectReads: true, - verbose: false, - description: 'High-throughput data ingestion service', - category: PresetCategory.SERVICE - }, - - [PresetName.SEARCH_API]: { - storage: StorageOption.CLOUD, - model: ModelPrecision.FP32, - features: ['core', 'search', 'cache', 'triple-intelligence'], - distributed: true, - role: DistributedRole.READER, - readOnly: true, - lazyLoadInReadOnlyMode: true, - cache: { - hotCacheMaxSize: 10000, - autoTune: true - }, - verbose: false, - description: 'Low-latency search API service', - category: PresetCategory.SERVICE - }, - - [PresetName.ANALYTICS_SERVICE]: { - storage: StorageOption.CLOUD, - model: ModelPrecision.AUTO, - features: ['core', 'search', 'metrics', 'monitoring'], - distributed: true, - role: DistributedRole.HYBRID, - verbose: false, - description: 'Analytics and data processing service', - category: PresetCategory.SERVICE - }, - - [PresetName.EDGE_CACHE]: { - storage: StorageOption.AUTO, - model: ModelPrecision.Q8, - features: ['core', 'search', 'cache'], - distributed: true, - role: DistributedRole.READER, - readOnly: true, - lazyLoadInReadOnlyMode: true, - cache: { - hotCacheMaxSize: 1000, - autoTune: false - }, - verbose: false, - description: 'Edge location cache with minimal footprint', - category: PresetCategory.SERVICE - }, - - [PresetName.BATCH_PROCESSOR]: { - storage: StorageOption.CLOUD, - model: ModelPrecision.Q8, - features: ['core', 'batch-processing', 'neural-api'], - distributed: true, - role: DistributedRole.HYBRID, - cache: { - hotCacheMaxSize: 5000, - batchSize: 500 - }, - verbose: false, - description: 'Batch processing and bulk operations', - category: PresetCategory.SERVICE - }, - - [PresetName.STREAMING_SERVICE]: { - storage: StorageOption.CLOUD, - model: ModelPrecision.Q8, - features: ['core', 'batch-processing', 'wal'], - distributed: true, - role: DistributedRole.WRITER, - writeOnly: true, - allowDirectReads: false, - verbose: false, - description: 'Real-time data streaming service', - category: PresetCategory.SERVICE - }, - - [PresetName.ML_TRAINING]: { - storage: StorageOption.CLOUD, - model: ModelPrecision.FP32, - features: FeatureSet.FULL, - distributed: true, - role: DistributedRole.HYBRID, - cache: { - hotCacheMaxSize: 20000, - autoTune: true - }, - verbose: true, - description: 'Machine learning training service', - category: PresetCategory.SERVICE - }, - - [PresetName.SIDECAR]: { - storage: StorageOption.AUTO, - model: ModelPrecision.Q8, - features: FeatureSet.MINIMAL, - distributed: false, - verbose: false, - description: 'Lightweight sidecar for microservices', - category: PresetCategory.SERVICE - } -} as const - -/** - * Type-safe preset getter - */ -export function getPreset(name: PresetName): PresetConfig { - return PRESET_CONFIGS[name] -} - -/** - * Check if a string is a valid preset name - */ -export function isValidPreset(name: string): name is PresetName { - return Object.values(PresetName).includes(name as PresetName) -} - -/** - * Get presets by category - */ -export function getPresetsByCategory(category: PresetCategory): PresetName[] { - return Object.entries(PRESET_CONFIGS) - .filter(([_, config]) => config.category === category) - .map(([name]) => name as PresetName) -} - -/** - * Get all preset names - */ -export function getAllPresetNames(): PresetName[] { - return Object.values(PresetName) -} - -/** - * Get preset description - */ -export function getPresetDescription(name: PresetName): string { - return PRESET_CONFIGS[name].description -} - -/** - * Convert preset config to Brainy config - */ -export function presetToBrainyConfig(preset: PresetConfig): any { - const config: any = { - storage: preset.storage, - model: preset.model, - verbose: preset.verbose - } - - // Handle features - if (Array.isArray(preset.features)) { - config.features = preset.features - } else { - config.features = preset.features // Will be expanded by processZeroConfig - } - - // Handle distributed settings - if (preset.distributed) { - config.distributed = { - enabled: true, - role: preset.role - } - - if (preset.readOnly) config.readOnly = true - if (preset.writeOnly) config.writeOnly = true - if (preset.allowDirectReads) config.allowDirectReads = true - if (preset.lazyLoadInReadOnlyMode) config.lazyLoadInReadOnlyMode = true - } - - return config -} \ No newline at end of file diff --git a/src/config/extensibleConfig.ts b/src/config/extensibleConfig.ts deleted file mode 100644 index 4cc33122..00000000 --- a/src/config/extensibleConfig.ts +++ /dev/null @@ -1,342 +0,0 @@ -/** - * Extensible Configuration System - * Allows augmentations to register new storage types, presets, and configurations - */ - -import { StorageOption, PresetName, PresetConfig, ModelPrecision, DistributedRole, PresetCategory } from './distributedPresets.js' -import { StorageConfigResult } from './storageAutoConfig.js' - -/** - * Storage provider registration interface - */ -export interface StorageProvider { - type: string // e.g., 'redis', 'mongodb', 'postgres' - name: string - description: string - - // Detection function - returns true if this storage should be auto-selected - detect: () => Promise - - // Configuration builder - getConfig: () => Promise - - // Priority for auto-detection (higher = checked first) - priority?: number - - // Required environment variables or packages - requirements?: { - env?: string[] - packages?: string[] - } -} - -/** - * Preset extension interface - */ -export interface PresetExtension { - name: string - config: PresetConfig - override?: boolean // Override existing preset -} - -/** - * Global registry for extensions - */ -class ConfigurationRegistry { - private static instance: ConfigurationRegistry - - // Registered storage providers - private storageProviders: Map = new Map() - - // Registered preset extensions - private presetExtensions: Map = new Map() - - // Custom auto-detection hooks - private autoDetectHooks: Array<() => Promise> = [] - - private constructor() { - // Initialize with built-in providers - this.registerBuiltInProviders() - } - - static getInstance(): ConfigurationRegistry { - if (!ConfigurationRegistry.instance) { - ConfigurationRegistry.instance = new ConfigurationRegistry() - } - return ConfigurationRegistry.instance - } - - /** - * Register a new storage provider - * This is how augmentations add new storage types - */ - registerStorageProvider(provider: StorageProvider): void { - console.log(`📦 Registering storage provider: ${provider.type} (${provider.name})`) - this.storageProviders.set(provider.type, provider) - } - - /** - * Register a new preset - */ - registerPreset(name: string, extension: PresetExtension): void { - console.log(`🎨 Registering preset: ${name}`) - this.presetExtensions.set(name, extension) - } - - /** - * Register an auto-detection hook - */ - registerAutoDetectHook(hook: () => Promise): void { - this.autoDetectHooks.push(hook) - } - - /** - * Get all registered storage providers - */ - getStorageProviders(): StorageProvider[] { - return Array.from(this.storageProviders.values()) - .sort((a, b) => (b.priority || 0) - (a.priority || 0)) - } - - /** - * Get all registered presets (built-in + extensions) - */ - getAllPresets(): Map { - // Start with built-in presets - const allPresets = new Map() - - // Note: Would import from distributedPresets-new.ts - // Add extended presets - for (const [name, extension] of this.presetExtensions) { - if (extension.override || !allPresets.has(name)) { - allPresets.set(name, extension.config) - } - } - - return allPresets - } - - /** - * Auto-detect storage including extensions - */ - async autoDetectStorage(): Promise { - // Check registered providers first (in priority order) - for (const provider of this.getStorageProviders()) { - try { - if (await provider.detect()) { - const config = await provider.getConfig() - return { - // Registered providers deliberately extend the built-in storage - // vocabulary ('redis', 'mongodb', ...) — the registry contract is - // open, so narrow the provider's free-form type string here. - type: provider.type as StorageConfigResult['type'], - config, - reason: `Auto-detected ${provider.name}`, - autoSelected: true - } - } - } catch (error) { - console.warn(`Failed to detect ${provider.type}:`, error) - } - } - - // Fallback to built-in detection - const { autoDetectStorage } = await import('./storageAutoConfig.js') - return autoDetectStorage() - } - - /** - * Register built-in providers - */ - private registerBuiltInProviders(): void { - // These would be the built-in ones, but could be overridden - } -} - -/** - * Example: Redis storage provider registration - * This would be in the redis augmentation package - */ -export const redisStorageProvider: StorageProvider = { - type: 'redis', - name: 'Redis Storage', - description: 'High-performance in-memory data store', - priority: 10, // Check before filesystem - - requirements: { - env: ['REDIS_URL', 'REDIS_HOST'], - packages: ['redis', 'ioredis'] - }, - - async detect(): Promise { - // Check for Redis connection info - if (process.env.REDIS_URL || process.env.REDIS_HOST) { - try { - // Try to connect to Redis (dynamic import for optional dependency) - const redis: any = await new Function('return import("ioredis")')().catch(() => null) - if (!redis) return false - - const client = new redis.default(process.env.REDIS_URL) - await client.ping() - await client.quit() - return true - } catch { - // Redis not available - } - } - return false - }, - - async getConfig(): Promise { - return { - type: 'redis', - redisStorage: { - url: process.env.REDIS_URL || `redis://${process.env.REDIS_HOST}:${process.env.REDIS_PORT || 6379}`, - prefix: process.env.REDIS_PREFIX || 'brainy:', - ttl: process.env.REDIS_TTL ? parseInt(process.env.REDIS_TTL) : undefined - } - } - } -} - -/** - * Example: MongoDB storage provider - */ -export const mongoStorageProvider: StorageProvider = { - type: 'mongodb', - name: 'MongoDB Storage', - description: 'Document database for complex data', - priority: 8, - - requirements: { - env: ['MONGODB_URI', 'MONGO_URL'], - packages: ['mongodb'] - }, - - async detect(): Promise { - if (process.env.MONGODB_URI || process.env.MONGO_URL) { - try { - const mongodb: any = await new Function('return import("mongodb")')().catch(() => null) - if (!mongodb) return false - - const client = new mongodb.MongoClient(process.env.MONGODB_URI || process.env.MONGO_URL!) - await client.connect() - await client.close() - return true - } catch { - // MongoDB not available - } - } - return false - }, - - async getConfig(): Promise { - return { - type: 'mongodb', - mongoStorage: { - uri: process.env.MONGODB_URI || process.env.MONGO_URL, - database: process.env.MONGO_DATABASE || 'brainy', - collection: process.env.MONGO_COLLECTION || 'vectors' - } - } - } -} - -/** - * Example: PostgreSQL with pgvector extension - */ -export const postgresStorageProvider: StorageProvider = { - type: 'postgres', - name: 'PostgreSQL Storage', - description: 'PostgreSQL with pgvector for scalable vector search', - priority: 9, - - requirements: { - env: ['DATABASE_URL', 'POSTGRES_URL'], - packages: ['pg', 'pgvector'] - }, - - async detect(): Promise { - const url = process.env.DATABASE_URL || process.env.POSTGRES_URL - if (url && url.includes('postgres')) { - try { - const pg: any = await new Function('return import("pg")')().catch(() => null) - if (!pg) return false - - const client = new pg.Client({ connectionString: url }) - await client.connect() - - // Check for pgvector extension - const result = await client.query( - "SELECT * FROM pg_extension WHERE extname = 'vector'" - ) - await client.end() - - return result.rows.length > 0 - } catch { - // PostgreSQL not available or pgvector not installed - } - } - return false - }, - - async getConfig(): Promise { - return { - type: 'postgres', - postgresStorage: { - connectionString: process.env.DATABASE_URL || process.env.POSTGRES_URL, - table: process.env.POSTGRES_TABLE || 'brainy_vectors', - schema: process.env.POSTGRES_SCHEMA || 'public' - } - } - } -} - -/** - * How an augmentation would register its storage provider - */ -export function registerStorageAugmentation(provider: StorageProvider): void { - const registry = ConfigurationRegistry.getInstance() - registry.registerStorageProvider(provider) -} - -/** - * How to register a new preset - */ -export function registerPresetAugmentation(name: string, config: PresetConfig): void { - const registry = ConfigurationRegistry.getInstance() - registry.registerPreset(name, { - name, - config, - override: false - }) -} - -/** - * Example preset for Redis-based caching service - */ -export const redisCachePreset: PresetConfig = { - // 'redis' is an extended storage type outside the built-in StorageOption - // enum — extension presets resolve it at runtime through the provider - // registry, so this is a genuine typed boundary (hence the double - // assertion: the literal has no overlap with the enum). - storage: 'redis' as unknown as StorageOption, - model: ModelPrecision.Q8, - features: ['core', 'cache', 'search'], - distributed: true, - role: DistributedRole.READER, - readOnly: true, - cache: { - hotCacheMaxSize: 50000, // Large Redis cache - autoTune: true - }, - description: 'Redis-backed caching layer', - category: PresetCategory.SERVICE -} - -/** - * Get the configuration registry - */ -export function getConfigRegistry(): ConfigurationRegistry { - return ConfigurationRegistry.getInstance() -} \ No newline at end of file diff --git a/src/config/index.ts b/src/config/index.ts index 7d7309a8..08e9ad28 100644 --- a/src/config/index.ts +++ b/src/config/index.ts @@ -35,32 +35,6 @@ export { createEmbeddingFunctionWithPrecision } from './zeroConfig.js' -// Strongly-typed presets and enums -export { - PresetName, - PresetCategory, - ModelPrecision, - StorageOption, - FeatureSet, - DistributedRole, - PresetConfig, - PRESET_CONFIGS, - getPreset, - isValidPreset, - getPresetsByCategory, - getAllPresetNames, - getPresetDescription, - presetToBrainyConfig -} from './distributedPresets.js' - -// Extensible configuration -export { - StorageProvider, - registerStorageAugmentation, - registerPresetAugmentation, - getConfigRegistry -} from './extensibleConfig.js' - /** * Main zero-config processor * This is what Brainy will call diff --git a/src/config/zeroConfig.ts b/src/config/zeroConfig.ts index b55b478c..e7e1420d 100644 --- a/src/config/zeroConfig.ts +++ b/src/config/zeroConfig.ts @@ -18,8 +18,10 @@ export interface BrainyZeroConfig { * - 'development': Optimized for development (memory storage, fp32, verbose logging) * - 'minimal': Minimal footprint (memory storage, q8, minimal features) * - 'zero': True zero config (all auto-detected) - * - 'writer': Write-only instance for distributed setups (no index loading) - * - 'reader': Read-only instance for distributed setups (no write operations) + * - 'writer': Writer process (read + write) — the single mutating process in + * Brainy's multi-process model + * - 'reader': Reader process (read-only, all mutations throw) — any number of + * these may run against the same on-disk store as the writer */ mode?: 'production' | 'development' | 'minimal' | 'zero' | 'writer' | 'reader' @@ -84,21 +86,15 @@ const PRESETS = { storage: 'auto' as const, features: 'minimal' as const, verbose: false, - // Writer-specific settings - distributed: true, - role: 'writer' as const, - writeOnly: true, - allowDirectReads: true // Allow deduplication checks + // The single mutating process: HybridMode (read + write). + mode: 'writer' as const }, reader: { storage: 'auto' as const, features: 'default' as const, verbose: false, - // Reader-specific settings - distributed: true, - role: 'reader' as const, - readOnly: true, - lazyLoadInReadOnlyMode: true // Optimize for search + // Read-only process: ReaderMode — every mutation throws. + mode: 'reader' as const } } @@ -263,44 +259,17 @@ export async function processZeroConfig(input?: string | BrainyZeroConfig): Prom ...config.advanced } - // Apply distributed preset settings if applicable + // Multi-process role: writer (read + write) or reader (read-only). This is the + // only signal Brainy needs — `mode` drives the operational mode (HybridMode vs + // ReaderMode) that gates every mutation path. if (config.mode === 'writer' || config.mode === 'reader') { - // The writer/reader presets each carry their own subset of the - // distributed flags beyond the base preset shape — widen to the union of - // those flags so both branches read uniformly. - const presetSettings: { - distributed: boolean - role: 'writer' | 'reader' - readOnly?: boolean - writeOnly?: boolean - allowDirectReads?: boolean - lazyLoadInReadOnlyMode?: boolean - } = PRESETS[config.mode] - - // Apply distributed-specific settings - finalConfig.distributed = presetSettings.distributed - finalConfig.readOnly = presetSettings.readOnly || false - finalConfig.writeOnly = presetSettings.writeOnly || false - finalConfig.allowDirectReads = presetSettings.allowDirectReads || false - finalConfig.lazyLoadInReadOnlyMode = presetSettings.lazyLoadInReadOnlyMode || false - - // Set distributed role in distributed config - if (finalConfig.distributed) { - finalConfig.distributed = { - enabled: true, - role: presetSettings.role - } - } - - // Log distributed mode if verbose + finalConfig.mode = config.mode + if (verbose) { - console.log(`📡 Distributed mode: ${config.mode.toUpperCase()}`) - console.log(` Role: ${presetSettings.role}`) - console.log(` Read-only: ${finalConfig.readOnly}`) - console.log(` Write-only: ${finalConfig.writeOnly}`) + console.log(`📡 Process role: ${config.mode.toUpperCase()}`) } } - + return finalConfig } diff --git a/src/coreTypes.ts b/src/coreTypes.ts index 1861df68..7c6b63ea 100644 --- a/src/coreTypes.ts +++ b/src/coreTypes.ts @@ -703,12 +703,6 @@ export interface StatisticsData { * Last updated timestamp */ lastUpdated: string - - /** - * Distributed configuration (stored in index folder for easy access) - * This is used for distributed Brainy instances coordination - */ - distributedConfig?: import('./types/distributedTypes.js').SharedConfig } /** diff --git a/src/distributed/cacheSync.ts b/src/distributed/cacheSync.ts deleted file mode 100644 index ddd6bd31..00000000 --- a/src/distributed/cacheSync.ts +++ /dev/null @@ -1,342 +0,0 @@ -/** - * Distributed Cache Synchronization - * Provides cache coherence across multiple Brainy instances - */ - -import { EventEmitter } from 'node:events' - -export interface CacheSyncConfig { - nodeId: string - syncInterval?: number - maxSyncBatchSize?: number - compressionEnabled?: boolean -} - -export interface CacheEntry { - key: string - value: any - version: number - timestamp: number - ttl?: number - nodeId: string -} - -export interface SyncMessage { - type: 'invalidate' | 'update' | 'delete' | 'batch' - entries: CacheEntry[] - source: string - timestamp: number -} - -/** - * Distributed Cache Synchronizer - */ -export class CacheSync extends EventEmitter { - private nodeId: string - private localCache: Map = new Map() - private versionVector: Map = new Map() - private syncQueue: SyncMessage[] = [] - private syncInterval: number - private maxSyncBatchSize: number - private syncTimer?: NodeJS.Timeout - private isRunning: boolean = false - - constructor(config: CacheSyncConfig) { - super() - - this.nodeId = config.nodeId - this.syncInterval = config.syncInterval || 1000 - this.maxSyncBatchSize = config.maxSyncBatchSize || 100 - } - - /** - * Start cache synchronization - */ - start(): void { - if (this.isRunning) return - - this.isRunning = true - this.startSyncTimer() - - this.emit('started', { nodeId: this.nodeId }) - } - - /** - * Stop cache synchronization - */ - stop(): void { - if (!this.isRunning) return - - this.isRunning = false - - if (this.syncTimer) { - clearInterval(this.syncTimer) - this.syncTimer = undefined - } - - this.emit('stopped', { nodeId: this.nodeId }) - } - - /** - * Get a value from cache - */ - get(key: string): any | undefined { - const entry = this.localCache.get(key) - - if (!entry) return undefined - - // Check TTL - if (entry.ttl && Date.now() - entry.timestamp > entry.ttl) { - this.localCache.delete(key) - return undefined - } - - return entry.value - } - - /** - * Set a value in cache and propagate - */ - set(key: string, value: any, ttl?: number): void { - const version = this.incrementVersion(key) - - const entry: CacheEntry = { - key, - value, - version, - timestamp: Date.now(), - ttl, - nodeId: this.nodeId - } - - this.localCache.set(key, entry) - - // Queue for sync - this.queueSync('update', [entry]) - } - - /** - * Delete a value from cache and propagate - */ - delete(key: string): boolean { - const existed = this.localCache.has(key) - - if (existed) { - const version = this.incrementVersion(key) - this.localCache.delete(key) - - // Queue deletion for sync - this.queueSync('delete', [{ - key, - value: null, - version, - timestamp: Date.now(), - nodeId: this.nodeId - }]) - } - - return existed - } - - /** - * Invalidate a cache entry across all nodes - */ - invalidate(key: string): void { - const version = this.incrementVersion(key) - this.localCache.delete(key) - - // Queue invalidation - this.queueSync('invalidate', [{ - key, - value: null, - version, - timestamp: Date.now(), - nodeId: this.nodeId - }]) - } - - /** - * Clear all cache entries - */ - clear(): void { - const entries: CacheEntry[] = [] - - for (const key of this.localCache.keys()) { - const version = this.incrementVersion(key) - entries.push({ - key, - value: null, - version, - timestamp: Date.now(), - nodeId: this.nodeId - }) - } - - this.localCache.clear() - - if (entries.length > 0) { - this.queueSync('delete', entries) - } - } - - /** - * Handle incoming sync message from another node - */ - handleSyncMessage(message: SyncMessage): void { - if (message.source === this.nodeId) return // Ignore own messages - - for (const entry of message.entries) { - this.handleRemoteEntry(message.type, entry) - } - - this.emit('synced', { - type: message.type, - entries: message.entries.length, - source: message.source - }) - } - - /** - * Handle a remote cache entry - */ - private handleRemoteEntry(type: 'invalidate' | 'update' | 'delete' | 'batch', entry: CacheEntry): void { - const localEntry = this.localCache.get(entry.key) - const localVersion = this.versionVector.get(entry.key) || 0 - - // Version vector check - only accept if remote version is newer - if (entry.version <= localVersion) { - return // Our version is newer or same, ignore - } - - // Update version vector - this.versionVector.set(entry.key, entry.version) - - switch (type) { - case 'update': - case 'batch': - // Update local cache with remote value - this.localCache.set(entry.key, entry) - break - - case 'delete': - case 'invalidate': - // Remove from local cache - this.localCache.delete(entry.key) - break - } - } - - /** - * Queue a sync message - */ - private queueSync(type: 'invalidate' | 'update' | 'delete' | 'batch', entries: CacheEntry[]): void { - const message: SyncMessage = { - type, - entries, - source: this.nodeId, - timestamp: Date.now() - } - - this.syncQueue.push(message) - - // If queue is getting large, sync immediately - if (this.syncQueue.length >= this.maxSyncBatchSize) { - this.performSync() - } - } - - /** - * Start sync timer - */ - private startSyncTimer(): void { - this.syncTimer = setInterval(() => { - this.performSync() - }, this.syncInterval) - } - - /** - * Perform sync operation - */ - private performSync(): void { - if (this.syncQueue.length === 0) return - - // Batch multiple messages if possible - const messages = this.syncQueue.splice(0, this.maxSyncBatchSize) - - if (messages.length === 1) { - // Single message - this.emit('sync', messages[0]) - } else { - // Batch multiple messages - const batchedEntries: CacheEntry[] = [] - for (const msg of messages) { - batchedEntries.push(...msg.entries) - } - - const batchMessage: SyncMessage = { - type: 'batch', - entries: batchedEntries, - source: this.nodeId, - timestamp: Date.now() - } - - this.emit('sync', batchMessage) - } - } - - /** - * Increment version for a key - */ - private incrementVersion(key: string): number { - const current = this.versionVector.get(key) || 0 - const next = current + 1 - this.versionVector.set(key, next) - return next - } - - /** - * Get cache statistics - */ - getStats(): { - entries: number - pendingSync: number - versionedKeys: number - memoryUsage: number - } { - // Estimate memory usage (rough approximation) - let memoryUsage = 0 - for (const entry of this.localCache.values()) { - memoryUsage += JSON.stringify(entry).length - } - - return { - entries: this.localCache.size, - pendingSync: this.syncQueue.length, - versionedKeys: this.versionVector.size, - memoryUsage - } - } - - /** - * Get cache entries for debugging - */ - getEntries(): CacheEntry[] { - return Array.from(this.localCache.values()) - } - - /** - * Merge cache state from another node (for recovery) - */ - mergeState(entries: CacheEntry[]): void { - for (const entry of entries) { - this.handleRemoteEntry('update', entry) - } - } -} - -/** - * Create a cache sync instance - */ -export function createCacheSync(config: CacheSyncConfig): CacheSync { - return new CacheSync(config) -} \ No newline at end of file diff --git a/src/distributed/configManager.ts b/src/distributed/configManager.ts deleted file mode 100644 index dc84f913..00000000 --- a/src/distributed/configManager.ts +++ /dev/null @@ -1,519 +0,0 @@ -/** - * Distributed Configuration Manager - * Manages shared configuration in S3 for distributed Brainy instances - */ - -import { v4 as uuidv4 } from '../universal/uuid.js' -import { - DistributedConfig, - SharedConfig, - InstanceInfo, - InstanceRole -} from '../types/distributedTypes.js' -import { StorageAdapter } from '../coreTypes.js' - -// Constants for config storage locations -const DISTRIBUTED_CONFIG_KEY = 'distributed_config' -const LEGACY_CONFIG_KEY = '_distributed_config' - -export class DistributedConfigManager { - private config: SharedConfig | null = null - private instanceId: string - private role: InstanceRole | undefined - private configPath: string - private heartbeatInterval: number - private configCheckInterval: number - private instanceTimeout: number - private storage: StorageAdapter - private heartbeatTimer?: NodeJS.Timeout - private configWatchTimer?: NodeJS.Timeout - private lastConfigVersion: number = 0 - private onConfigUpdate?: (config: SharedConfig) => void - private hasMigrated: boolean = false - - constructor( - storage: StorageAdapter, - distributedConfig?: DistributedConfig, - brainyMode?: { readOnly?: boolean; writeOnly?: boolean } - ) { - this.storage = storage - this.instanceId = distributedConfig?.instanceId || `instance-${uuidv4()}` - // Updated default path to use _system instead of _brainy - this.configPath = distributedConfig?.configPath || '_system/distributed_config.json' - this.heartbeatInterval = distributedConfig?.heartbeatInterval || 30000 - this.configCheckInterval = distributedConfig?.configCheckInterval || 10000 - this.instanceTimeout = distributedConfig?.instanceTimeout || 60000 - - // Set role from distributed config if provided - if (distributedConfig?.role) { - this.role = distributedConfig.role - } - // Infer role from Brainy's read/write mode if not explicitly set - else if (brainyMode) { - if (brainyMode.writeOnly) { - this.role = 'writer' - } else if (brainyMode.readOnly) { - this.role = 'reader' - } - // If neither readOnly nor writeOnly, role must be explicitly set - } - } - - /** - * Initialize the distributed configuration - */ - async initialize(): Promise { - // Load or create configuration - this.config = await this.loadOrCreateConfig() - - // Determine role if not explicitly set - if (!this.role) { - this.role = await this.determineRole() - } - - // Register this instance - await this.registerInstance() - - // Start heartbeat and config watching - this.startHeartbeat() - this.startConfigWatch() - - return this.config - } - - /** - * Load existing config or create new one - */ - private async loadOrCreateConfig(): Promise { - // First, try to load from the new location in index folder - try { - const configData = await this.storage.getStatistics() - if (configData && configData.distributedConfig) { - this.lastConfigVersion = configData.distributedConfig.version - return configData.distributedConfig as SharedConfig - } - } catch (error) { - // Config doesn't exist in new location yet - } - - // Check if we need to migrate from old location - if (!this.hasMigrated) { - const migrated = await this.migrateConfigFromLegacyLocation() - if (migrated) { - return migrated - } - } - - // Legacy fallback - try old location - try { - const configData = await this.storage.getMetadata(LEGACY_CONFIG_KEY) - if (configData) { - // Migrate to new location - const config = configData as unknown as SharedConfig - await this.migrateConfig(config) - this.lastConfigVersion = config.version - return config - } - } catch (error) { - // Config doesn't exist yet - } - - // Create default config - const newConfig: SharedConfig = { - version: 1, - updated: new Date().toISOString(), - settings: { - partitionStrategy: 'hash', - partitionCount: 100, - embeddingModel: 'text-embedding-ada-002', - dimensions: 1536, - distanceMetric: 'cosine', - hnswParams: { - M: 16, - efConstruction: 200 - } - }, - instances: {} - } - - await this.saveConfig(newConfig) - return newConfig - } - - /** - * Determine role based on configuration - * IMPORTANT: Role must be explicitly set - no automatic assignment based on order - */ - private async determineRole(): Promise { - // Check environment variable first - if (process.env.BRAINY_ROLE) { - const role = process.env.BRAINY_ROLE.toLowerCase() - if (role === 'writer' || role === 'reader' || role === 'hybrid') { - return role as InstanceRole - } - throw new Error(`Invalid BRAINY_ROLE: ${process.env.BRAINY_ROLE}. Must be 'writer', 'reader', or 'hybrid'`) - } - - // Check if explicitly passed in distributed config - if (this.role) { - return this.role - } - - // DO NOT auto-assign roles based on deployment order or existing instances - // This is dangerous and can lead to data corruption or loss - throw new Error( - 'Distributed mode requires explicit role configuration. ' + - 'Set BRAINY_ROLE environment variable or pass role in distributed config. ' + - 'Valid roles: "writer", "reader", "hybrid"' - ) - } - - /** - * Check if an instance is still alive - */ - private isInstanceAlive(instance: InstanceInfo): boolean { - const lastSeen = new Date(instance.lastHeartbeat).getTime() - const now = Date.now() - return (now - lastSeen) < this.instanceTimeout - } - - /** - * Register this instance in the shared config - */ - private async registerInstance(): Promise { - if (!this.config) return - - // Role must be set by this point - if (!this.role) { - throw new Error('Cannot register instance without a role') - } - - const instanceInfo: InstanceInfo = { - role: this.role, - status: 'active', - lastHeartbeat: new Date().toISOString(), - metrics: { - memoryUsage: process.memoryUsage().heapUsed - } - } - - // Add endpoint if available - if (process.env.SERVICE_ENDPOINT) { - instanceInfo.endpoint = process.env.SERVICE_ENDPOINT - } - - this.config.instances[this.instanceId] = instanceInfo - await this.saveConfig(this.config) - } - - /** - * Migrate config from legacy location to new location - */ - private async migrateConfigFromLegacyLocation(): Promise { - try { - // Try to load from old location - const legacyConfig = await this.storage.getMetadata(LEGACY_CONFIG_KEY) - if (legacyConfig) { - console.log('Migrating distributed config from legacy location to index folder...') - - const config = legacyConfig as unknown as SharedConfig - // Save to new location - await this.migrateConfig(config) - - // Delete from old location (optional - we can keep it for rollback) - // await this.storage.deleteMetadata(LEGACY_CONFIG_KEY) - - this.hasMigrated = true - this.lastConfigVersion = config.version - return config - } - } catch (error) { - console.error('Error during config migration:', error) - } - - this.hasMigrated = true - return null - } - - /** - * Migrate config to new location in index folder - */ - private async migrateConfig(config: SharedConfig): Promise { - // Get existing statistics or create new - let stats = await this.storage.getStatistics() - if (!stats) { - stats = { - nounCount: {}, - verbCount: {}, - metadataCount: {}, - hnswIndexSize: 0, - lastUpdated: new Date().toISOString() - } - } - - // Add distributed config to statistics - stats.distributedConfig = config - - // Save updated statistics - await this.storage.saveStatistics(stats) - } - - /** - * Save configuration with version increment - */ - private async saveConfig(config: SharedConfig): Promise { - config.version++ - config.updated = new Date().toISOString() - this.lastConfigVersion = config.version - - // Save to new location in index folder along with statistics - let stats = await this.storage.getStatistics() - if (!stats) { - stats = { - nounCount: {}, - verbCount: {}, - metadataCount: {}, - hnswIndexSize: 0, - lastUpdated: new Date().toISOString() - } - } - - // Update distributed config in statistics - stats.distributedConfig = config - - // Save updated statistics - await this.storage.saveStatistics(stats) - - this.config = config - } - - /** - * Start heartbeat to keep instance alive in config - */ - private startHeartbeat(): void { - this.heartbeatTimer = setInterval(async () => { - await this.updateHeartbeat() - }, this.heartbeatInterval) - } - - /** - * Update heartbeat and clean stale instances - */ - private async updateHeartbeat(): Promise { - if (!this.config) return - - // Reload config to get latest state - try { - const latestConfig = await this.loadConfig() - if (latestConfig) { - this.config = latestConfig - } - } catch (error) { - console.error('Failed to reload config:', error) - } - - // Update our heartbeat - if (this.config.instances[this.instanceId]) { - this.config.instances[this.instanceId].lastHeartbeat = new Date().toISOString() - this.config.instances[this.instanceId].status = 'active' - - // Update metrics if available - this.config.instances[this.instanceId].metrics = { - memoryUsage: process.memoryUsage().heapUsed - } - } else { - // Re-register if we were removed - await this.registerInstance() - return - } - - // Clean up stale instances - const now = Date.now() - let hasChanges = false - - for (const [id, instance] of Object.entries(this.config.instances)) { - if (id === this.instanceId) continue - - const lastSeen = new Date(instance.lastHeartbeat).getTime() - if (now - lastSeen > this.instanceTimeout) { - delete this.config.instances[id] - hasChanges = true - } - } - - // Save if there were changes - if (hasChanges) { - await this.saveConfig(this.config) - } else { - // Just update our heartbeat without version increment - // Get existing statistics - let stats = await this.storage.getStatistics() - if (!stats) { - stats = { - nounCount: {}, - verbCount: {}, - metadataCount: {}, - hnswIndexSize: 0, - lastUpdated: new Date().toISOString() - } - } - - // Update distributed config in statistics without version increment - stats.distributedConfig = this.config - - // Save updated statistics - await this.storage.saveStatistics(stats) - } - } - - /** - * Start watching for config changes - */ - private startConfigWatch(): void { - this.configWatchTimer = setInterval(async () => { - await this.checkForConfigUpdates() - }, this.configCheckInterval) - } - - /** - * Check for configuration updates - */ - private async checkForConfigUpdates(): Promise { - try { - const latestConfig = await this.loadConfig() - if (!latestConfig) return - - if (latestConfig.version > this.lastConfigVersion) { - this.config = latestConfig - this.lastConfigVersion = latestConfig.version - - // Notify listeners of config update - if (this.onConfigUpdate) { - this.onConfigUpdate(latestConfig) - } - } - } catch (error) { - console.error('Failed to check config updates:', error) - } - } - - /** - * Load configuration from storage - */ - private async loadConfig(): Promise { - try { - // Try new location first - const stats = await this.storage.getStatistics() - if (stats && stats.distributedConfig) { - return stats.distributedConfig as SharedConfig - } - - // Fallback to legacy location if not migrated yet - if (!this.hasMigrated) { - const configData = await this.storage.getMetadata(LEGACY_CONFIG_KEY) - if (configData) { - // Trigger migration on next save - return configData as unknown as SharedConfig - } - } - } catch (error) { - console.error('Failed to load config:', error) - } - return null - } - - /** - * Get current configuration - */ - getConfig(): SharedConfig | null { - return this.config - } - - /** - * Get instance role - */ - getRole(): InstanceRole { - if (!this.role) { - throw new Error('Role not initialized') - } - return this.role - } - - /** - * Get instance ID - */ - getInstanceId(): string { - return this.instanceId - } - - /** - * Set config update callback - */ - setOnConfigUpdate(callback: (config: SharedConfig) => void): void { - this.onConfigUpdate = callback - } - - /** - * Get all active instances of a specific role - */ - getInstancesByRole(role: InstanceRole): InstanceInfo[] { - if (!this.config) return [] - - return Object.entries(this.config.instances) - .filter(([_, instance]) => - instance.role === role && - this.isInstanceAlive(instance) - ) - .map(([_, instance]) => instance) - } - - /** - * Update instance metrics - */ - async updateMetrics(metrics: Partial): Promise { - if (!this.config || !this.config.instances[this.instanceId]) return - - this.config.instances[this.instanceId].metrics = { - ...this.config.instances[this.instanceId].metrics, - ...metrics - } - - // Don't increment version for metric updates - // Get existing statistics - let stats = await this.storage.getStatistics() - if (!stats) { - stats = { - nounCount: {}, - verbCount: {}, - metadataCount: {}, - hnswIndexSize: 0, - lastUpdated: new Date().toISOString() - } - } - - // Update distributed config in statistics without version increment - stats.distributedConfig = this.config - - // Save updated statistics - await this.storage.saveStatistics(stats) - } - - /** - * Cleanup resources - */ - async cleanup(): Promise { - // Stop timers - if (this.heartbeatTimer) { - clearInterval(this.heartbeatTimer) - } - if (this.configWatchTimer) { - clearInterval(this.configWatchTimer) - } - - // Mark instance as inactive - if (this.config && this.config.instances[this.instanceId]) { - this.config.instances[this.instanceId].status = 'inactive' - await this.saveConfig(this.config) - } - } -} \ No newline at end of file diff --git a/src/distributed/coordinator.ts b/src/distributed/coordinator.ts deleted file mode 100644 index 5aff6ebc..00000000 --- a/src/distributed/coordinator.ts +++ /dev/null @@ -1,686 +0,0 @@ -/** - * Distributed Coordinator for Brainy 3.0 - * Provides leader election, consensus, and coordination for distributed instances - */ - -import { EventEmitter } from 'node:events' -import { NetworkTransport, NetworkMessage } from './networkTransport.js' -import { createHash } from 'node:crypto' - -export interface NodeInfo { - id: string - address: string - port: number - lastSeen: number - status: 'active' | 'inactive' | 'suspected' - state?: 'follower' | 'candidate' | 'leader' - lastHeartbeat?: number -} - -export interface CoordinatorConfig { - nodeId?: string - address?: string - port?: number - heartbeatInterval?: number - electionTimeout?: number - nodes?: string[] -} - -export interface ConsensusState { - term: number - votedFor: string | null - leader: string | null - state: 'follower' | 'candidate' | 'leader' -} - -export interface RaftMessage { - type: 'requestVote' | 'voteResponse' | 'appendEntries' | 'appendResponse' - term: number - from: string - to?: string - data?: any -} - -/** - * Distributed Coordinator implementing Raft-like consensus - */ -export class DistributedCoordinator extends EventEmitter { - private nodeId: string - private nodes: Map = new Map() - private consensusState: ConsensusState - private heartbeatInterval: number - private electionTimeout: number - private electionTimer?: NodeJS.Timeout - private heartbeatTimer?: NodeJS.Timeout - private isRunning: boolean = false - private networkTransport?: NetworkTransport - private votesReceived: Set = new Set() - private currentTerm: number = 0 - private lastLogIndex: number = -1 - private lastLogTerm: number = 0 - private logEntries: Array<{ term: number; index: number; data: any }> = [] - private transport: any = null // For migration proposals - private pendingMigrations = new Map() - private committedMigrations = new Set() - - constructor(config: CoordinatorConfig = {}) { - super() - - // Generate node ID if not provided - this.nodeId = config.nodeId || this.generateNodeId() - - // Configuration - this.heartbeatInterval = config.heartbeatInterval || 1000 - this.electionTimeout = config.electionTimeout || 5000 - - // Initialize consensus state - this.consensusState = { - term: 0, - votedFor: null, - leader: null, - state: 'follower' - } - - // Register this node - this.nodes.set(this.nodeId, { - id: this.nodeId, - address: config.address || 'localhost', - port: config.port || 3000, - lastSeen: Date.now(), - status: 'active', - state: 'follower', - lastHeartbeat: Date.now() - }) - - // Register other nodes if provided - if (config.nodes) { - this.registerNodes(config.nodes) - } - } - - /** - * Start the coordinator - */ - async start(networkTransport?: NetworkTransport): Promise { - if (this.isRunning) return - - this.isRunning = true - this.networkTransport = networkTransport - - // Setup network message handlers if transport is provided - if (this.networkTransport) { - this.setupNetworkHandlers() - } - - this.emit('started', { nodeId: this.nodeId }) - - // Start as follower - this.becomeFollower() - } - - /** - * Setup network message handlers - */ - private setupNetworkHandlers(): void { - if (!this.networkTransport) return - - // Register handlers through the transport's public onMessage(), which - // writes to the same messageHandlers map this code used to reach into. - const transport = this.networkTransport - - // Handle vote requests - transport.onMessage('requestVote', async (msg: NetworkMessage) => { - const response = await this.handleVoteRequest(msg) - // Send response back - if (this.networkTransport) { - await this.networkTransport.sendToNode(msg.from, 'voteResponse', response) - } - return response - }) - - // Handle vote responses - transport.onMessage('voteResponse', async (msg: NetworkMessage) => { - this.handleVoteResponse(msg) - }) - - // Handle heartbeats/append entries - transport.onMessage('appendEntries', async (msg: NetworkMessage) => { - const response = await this.handleAppendEntries(msg) - // Send response back - if (this.networkTransport) { - await this.networkTransport.sendToNode(msg.from, 'appendResponse', response) - } - return response - }) - - // Handle append responses - transport.onMessage('appendResponse', async (msg: NetworkMessage) => { - this.handleAppendResponse(msg) - }) - } - - /** - * Stop the coordinator - */ - async stop(): Promise { - if (!this.isRunning) return - - this.isRunning = false - - if (this.electionTimer) { - clearTimeout(this.electionTimer) - this.electionTimer = undefined - } - - if (this.heartbeatTimer) { - clearInterval(this.heartbeatTimer) - this.heartbeatTimer = undefined - } - - this.emit('stopped') - } - - /** - * Register additional nodes - */ - registerNodes(nodes: string[]): void { - for (const node of nodes) { - const [address, port] = node.split(':') - const nodeId = this.generateNodeId(node) - - if (!this.nodes.has(nodeId)) { - this.nodes.set(nodeId, { - id: nodeId, - address, - port: parseInt(port || '3000'), - lastSeen: Date.now(), - status: 'active', - state: 'follower' - }) - } - } - } - - /** - * Become a follower - */ - private becomeFollower(): void { - this.consensusState.state = 'follower' - - const node = this.nodes.get(this.nodeId) - if (node) { - node.state = 'follower' - } - - this.resetElectionTimeout() - this.emit('stateChange', 'follower') - } - - /** - * Become a candidate and start election - */ - private async becomeCandidate(): Promise { - this.consensusState.state = 'candidate' - this.currentTerm++ - this.consensusState.term = this.currentTerm - this.consensusState.votedFor = this.nodeId - this.votesReceived = new Set([this.nodeId]) - - const node = this.nodes.get(this.nodeId) - if (node) { - node.state = 'candidate' - } - - this.emit('stateChange', 'candidate') - - // Request votes from all other nodes - await this.requestVotes() - - // Reset election timeout - this.resetElectionTimeout() - } - - /** - * Become the leader - */ - private becomeLeader(): void { - this.consensusState.state = 'leader' - this.consensusState.leader = this.nodeId - - const node = this.nodes.get(this.nodeId) - if (node) { - node.state = 'leader' - } - - // Stop election timer as leader - if (this.electionTimer) { - clearTimeout(this.electionTimer) - this.electionTimer = undefined - } - - this.emit('stateChange', 'leader') - this.emit('leaderElected', this.nodeId) - - // Start sending heartbeats - this.startHeartbeat() - } - - /** - * Request votes from all nodes - */ - private async requestVotes(): Promise { - if (!this.networkTransport) { - // Simulate vote for testing - this.checkVoteMajority() - return - } - - const voteRequest = { - type: 'requestVote' as const, - term: this.currentTerm, - candidateId: this.nodeId, - lastLogIndex: this.getLastLogIndex(), - lastLogTerm: this.getLastLogTerm() - } - - // Send vote requests to all other nodes - for (const [nodeId] of this.nodes) { - if (nodeId !== this.nodeId) { - try { - await this.networkTransport.sendToNode(nodeId, 'requestVote', voteRequest) - } catch (err) { - console.error(`Failed to request vote from ${nodeId}:`, err) - } - } - } - } - - /** - * Handle vote request from another node - */ - private async handleVoteRequest(msg: NetworkMessage): Promise { - const { term, candidateId, lastLogIndex, lastLogTerm } = msg.data - - // Update term if necessary - if (term > this.currentTerm) { - this.currentTerm = term - this.consensusState.term = term - this.consensusState.votedFor = null - this.becomeFollower() - } - - // Grant vote if conditions are met - let voteGranted = false - if (term >= this.currentTerm && - (!this.consensusState.votedFor || this.consensusState.votedFor === candidateId) && - this.isLogUpToDate(lastLogIndex, lastLogTerm)) { - this.consensusState.votedFor = candidateId - voteGranted = true - this.resetElectionTimeout() - } - - return { - type: 'voteResponse', - term: this.currentTerm, - voteGranted - } - } - - /** - * Handle vote response - */ - private handleVoteResponse(msg: NetworkMessage): void { - const { term, voteGranted } = msg.data - - // Ignore old responses - if (term < this.currentTerm) return - - // Update term if necessary - if (term > this.currentTerm) { - this.currentTerm = term - this.consensusState.term = term - this.becomeFollower() - return - } - - // Count vote if granted - if (voteGranted && this.consensusState.state === 'candidate') { - this.votesReceived.add(msg.from) - this.checkVoteMajority() - } - } - - /** - * Check if we have majority votes - */ - private checkVoteMajority(): void { - const majority = Math.floor(this.nodes.size / 2) + 1 - if (this.votesReceived.size >= majority) { - this.becomeLeader() - } - } - - /** - * Handle append entries (heartbeat) from leader - */ - private async handleAppendEntries(msg: NetworkMessage): Promise { - const { term, leaderId, prevLogIndex, prevLogTerm, entries, leaderCommit } = msg.data - - // Update term if necessary - if (term > this.currentTerm) { - this.currentTerm = term - this.consensusState.term = term - this.consensusState.votedFor = null - this.becomeFollower() - } - - // Reset election timeout when receiving valid heartbeat - if (term >= this.currentTerm) { - this.consensusState.leader = leaderId - this.resetElectionTimeout() - - // Update leader node's last heartbeat - const leaderNode = this.nodes.get(leaderId) - if (leaderNode) { - leaderNode.lastHeartbeat = Date.now() - leaderNode.lastSeen = Date.now() - } - } - - // Check log consistency - let success = false - if (term >= this.currentTerm) { - if (this.checkLogConsistency(prevLogIndex, prevLogTerm)) { - // Append new entries if any - if (entries && entries.length > 0) { - this.appendLogEntries(prevLogIndex, entries) - } - success = true - } - } - - return { - type: 'appendResponse', - term: this.currentTerm, - success - } - } - - /** - * Handle append response from follower - */ - private handleAppendResponse(msg: NetworkMessage): void { - const { term, success } = msg.data - - // Update term if necessary - if (term > this.currentTerm) { - this.currentTerm = term - this.consensusState.term = term - this.becomeFollower() - } - - // Process successful append - if (success && this.consensusState.state === 'leader') { - // Update follower's match index - // In a real implementation, this would track replication progress - } - } - - /** - * Send heartbeat to followers - */ - private async sendHeartbeat(): Promise { - if (!this.networkTransport) { - // Fallback for testing - await new Promise(resolve => setTimeout(resolve, 10)) - return - } - - const heartbeat = { - type: 'appendEntries' as const, - term: this.currentTerm, - leaderId: this.nodeId, - prevLogIndex: this.getLastLogIndex(), - prevLogTerm: this.getLastLogTerm(), - entries: [], - leaderCommit: 0 - } - - // Send heartbeat to all followers - for (const [nodeId] of this.nodes) { - if (nodeId !== this.nodeId) { - try { - await this.networkTransport.sendToNode(nodeId, 'appendEntries', heartbeat) - } catch (err) { - // Node might be down, mark as suspected - const node = this.nodes.get(nodeId) - if (node) { - node.status = 'suspected' - } - } - } - } - } - - /** - * Start heartbeat timer as leader - */ - private startHeartbeat(): void { - if (this.heartbeatTimer) { - clearInterval(this.heartbeatTimer) - } - - this.heartbeatTimer = setInterval(() => { - this.sendHeartbeat() - }, this.heartbeatInterval) - - // Send immediate heartbeat - this.sendHeartbeat() - } - - /** - * Reset election timeout - */ - private resetElectionTimeout(): void { - if (this.electionTimer) { - clearTimeout(this.electionTimer) - } - - // Randomize timeout to prevent split votes - const timeout = this.electionTimeout + Math.random() * this.electionTimeout - - this.electionTimer = setTimeout(() => { - if (this.consensusState.state === 'follower') { - this.becomeCandidate() - } - }, timeout) - } - - /** - * Check if log is up to date - */ - private isLogUpToDate(lastLogIndex: number, lastLogTerm: number): boolean { - const myLastLogTerm = this.getLastLogTerm() - const myLastLogIndex = this.getLastLogIndex() - - if (lastLogTerm > myLastLogTerm) return true - if (lastLogTerm < myLastLogTerm) return false - return lastLogIndex >= myLastLogIndex - } - - /** - * Check log consistency - */ - private checkLogConsistency(prevLogIndex: number, prevLogTerm: number): boolean { - if (prevLogIndex === -1) return true - - if (prevLogIndex >= this.logEntries.length) return false - - const entry = this.logEntries[prevLogIndex] - return entry && entry.term === prevLogTerm - } - - /** - * Append log entries - */ - private appendLogEntries(prevLogIndex: number, entries: any[]): void { - // Remove conflicting entries - this.logEntries = this.logEntries.slice(0, prevLogIndex + 1) - - // Append new entries - for (const entry of entries) { - this.logEntries.push({ - term: entry.term, - index: this.logEntries.length, - data: entry.data - }) - } - - this.lastLogIndex = this.logEntries.length - 1 - if (this.lastLogIndex >= 0) { - this.lastLogTerm = this.logEntries[this.lastLogIndex].term - } - } - - /** - * Get last log index - */ - private getLastLogIndex(): number { - return this.lastLogIndex - } - - /** - * Get last log term - */ - private getLastLogTerm(): number { - return this.lastLogTerm - } - - /** - * Generate a unique node ID - */ - private generateNodeId(seed?: string): string { - const source = seed || `${process.pid}-${Date.now()}-${Math.random()}` - return createHash('sha256').update(source).digest('hex').substring(0, 16) - } - - /** - * Get current leader - */ - getLeader(): string | null { - return this.consensusState.leader - } - - /** - * Propose a shard migration to the cluster - */ - async proposeMigration(migration: { - shardId: string - fromNode: string - toNode: string - migrationId: string - }): Promise { - if (!this.isLeader()) { - throw new Error('Only leader can propose migrations') - } - - // Broadcast migration proposal to all nodes - const message = { - type: 'migration-proposal', - migration, - timestamp: Date.now() - } - - await this.transport.broadcast('migration', message) - - // Store migration as pending - this.pendingMigrations.set(migration.migrationId, { - ...migration, - status: 'pending' - }) - } - - /** - * Get migration status - */ - async getMigrationStatus(migrationId: string): Promise<'pending' | 'committed' | 'rejected'> { - const migration = this.pendingMigrations.get(migrationId) - - if (!migration) { - // Check if it was committed - return this.committedMigrations.has(migrationId) ? 'committed' : 'rejected' - } - - return migration.status || 'pending' - } - - /** - * Check if this node is the leader - */ - isLeader(): boolean { - return this.consensusState.state === 'leader' - } - - /** - * Get all nodes in the cluster - */ - getNodes(): NodeInfo[] { - return Array.from(this.nodes.values()) - } - - /** - * Get cluster health status - */ - getHealth(): { healthy: boolean; leader: string | null; nodes: number; activeNodes: number } { - const now = Date.now() - const activeNodes = Array.from(this.nodes.values()).filter( - node => now - node.lastSeen < this.electionTimeout - ).length - - return { - healthy: this.consensusState.leader !== null && activeNodes > this.nodes.size / 2, - leader: this.consensusState.leader, - nodes: this.nodes.size, - activeNodes - } - } - - /** - * Propose a command to the cluster - */ - async proposeCommand(command: any): Promise { - if (this.consensusState.state !== 'leader') { - throw new Error(`Not the leader. Current leader: ${this.consensusState.leader}`) - } - - // Add to log - const entry = { - term: this.currentTerm, - index: this.logEntries.length, - data: command - } - - this.logEntries.push(entry) - this.lastLogIndex = entry.index - this.lastLogTerm = entry.term - - // Replicate to followers - await this.sendHeartbeat() - - this.emit('commandProposed', command) - } - - /** - * Get current state - */ - getState(): ConsensusState { - return { ...this.consensusState } - } -} - -/** - * Create a coordinator instance - */ -export function createCoordinator(config?: CoordinatorConfig): DistributedCoordinator { - return new DistributedCoordinator(config) -} \ No newline at end of file diff --git a/src/distributed/domainDetector.ts b/src/distributed/domainDetector.ts deleted file mode 100644 index 981683d0..00000000 --- a/src/distributed/domainDetector.ts +++ /dev/null @@ -1,323 +0,0 @@ -/** - * Domain Detector - * Automatically detects and manages data domains for logical separation - */ - -import { DomainMetadata } from '../types/distributedTypes.js' - -export interface DomainPattern { - domain: string - patterns: { - fields?: string[] - keywords?: string[] - regex?: RegExp - } - priority?: number -} - -export class DomainDetector { - private domainPatterns: DomainPattern[] = [ - { - domain: 'medical', - patterns: { - fields: ['symptoms', 'diagnosis', 'treatment', 'medication', 'patient'], - keywords: ['medical', 'health', 'disease', 'symptom', 'treatment', 'doctor', 'patient'] - }, - priority: 1 - }, - { - domain: 'legal', - patterns: { - fields: ['contract', 'clause', 'litigation', 'statute', 'jurisdiction'], - keywords: ['legal', 'law', 'contract', 'court', 'attorney', 'litigation', 'statute'] - }, - priority: 1 - }, - { - domain: 'product', - patterns: { - fields: ['price', 'sku', 'inventory', 'category', 'brand'], - keywords: ['product', 'price', 'sale', 'inventory', 'catalog', 'item', 'sku'] - }, - priority: 1 - }, - { - domain: 'customer', - patterns: { - fields: ['customerId', 'email', 'phone', 'address', 'orders'], - keywords: ['customer', 'client', 'user', 'account', 'profile', 'contact'] - }, - priority: 1 - }, - { - domain: 'financial', - patterns: { - fields: ['amount', 'currency', 'transaction', 'balance', 'account'], - keywords: ['financial', 'money', 'payment', 'transaction', 'bank', 'credit', 'debit'] - }, - priority: 1 - }, - { - domain: 'technical', - patterns: { - fields: ['code', 'function', 'error', 'stack', 'api'], - keywords: ['code', 'software', 'api', 'error', 'debug', 'function', 'class', 'method'] - }, - priority: 2 - } - ] - - private customPatterns: DomainPattern[] = [] - private domainStats: Map = new Map() - - /** - * Detect domain from data object - * @param data - The data object to analyze - * @returns The detected domain and metadata - */ - detectDomain(data: any): DomainMetadata { - if (!data || typeof data !== 'object') { - return { domain: 'general' } - } - - // Check for explicit domain field - if (data.domain && typeof data.domain === 'string') { - this.updateStats(data.domain) - return { - domain: data.domain, - domainMetadata: this.extractDomainMetadata(data, data.domain) - } - } - - // Score each domain pattern - const scores = new Map() - - // Check custom patterns first (higher priority) - for (const pattern of this.customPatterns) { - const score = this.scorePattern(data, pattern) - if (score > 0) { - scores.set(pattern.domain, score * (pattern.priority || 1)) - } - } - - // Check default patterns - for (const pattern of this.domainPatterns) { - const score = this.scorePattern(data, pattern) - if (score > 0) { - const currentScore = scores.get(pattern.domain) || 0 - scores.set(pattern.domain, currentScore + score * (pattern.priority || 1)) - } - } - - // Find highest scoring domain - let bestDomain = 'general' - let bestScore = 0 - - for (const [domain, score] of scores.entries()) { - if (score > bestScore) { - bestDomain = domain - bestScore = score - } - } - - this.updateStats(bestDomain) - - return { - domain: bestDomain, - domainMetadata: this.extractDomainMetadata(data, bestDomain) - } - } - - /** - * Score a data object against a domain pattern - */ - private scorePattern(data: any, pattern: DomainPattern): number { - let score = 0 - - // Check field matches - if (pattern.patterns.fields) { - const dataKeys = Object.keys(data) - for (const field of pattern.patterns.fields) { - if (dataKeys.some(key => key.toLowerCase().includes(field.toLowerCase()))) { - score += 2 // Field match is strong signal - } - } - } - - // Check keyword matches in values - if (pattern.patterns.keywords) { - const dataStr = JSON.stringify(data).toLowerCase() - for (const keyword of pattern.patterns.keywords) { - if (dataStr.includes(keyword.toLowerCase())) { - score += 1 - } - } - } - - // Check regex patterns - if (pattern.patterns.regex) { - const dataStr = JSON.stringify(data) - if (pattern.patterns.regex.test(dataStr)) { - score += 3 // Regex match is very specific - } - } - - return score - } - - /** - * Extract domain-specific metadata - */ - private extractDomainMetadata(data: any, domain: string): Record { - const metadata: Record = {} - - switch (domain) { - case 'medical': - if (data.patientId) metadata.patientId = data.patientId - if (data.condition) metadata.condition = data.condition - if (data.severity) metadata.severity = data.severity - break - - case 'legal': - if (data.caseId) metadata.caseId = data.caseId - if (data.jurisdiction) metadata.jurisdiction = data.jurisdiction - if (data.documentType) metadata.documentType = data.documentType - break - - case 'product': - if (data.sku) metadata.sku = data.sku - if (data.category) metadata.category = data.category - if (data.brand) metadata.brand = data.brand - if (data.price) metadata.priceRange = this.getPriceRange(data.price) - break - - case 'customer': - if (data.customerId) metadata.customerId = data.customerId - if (data.segment) metadata.segment = data.segment - if (data.lifetime_value) metadata.valueCategory = this.getValueCategory(data.lifetime_value) - break - - case 'financial': - if (data.accountId) metadata.accountId = data.accountId - if (data.transactionType) metadata.transactionType = data.transactionType - if (data.amount) metadata.amountRange = this.getAmountRange(data.amount) - break - - case 'technical': - if (data.service) metadata.service = data.service - if (data.environment) metadata.environment = data.environment - if (data.severity) metadata.severity = data.severity - break - } - - // Add detection confidence - metadata.detectionConfidence = this.calculateConfidence(data, domain) - - return metadata - } - - /** - * Calculate detection confidence - */ - private calculateConfidence(data: any, domain: string): 'high' | 'medium' | 'low' { - // If domain was explicitly specified - if (data.domain === domain) return 'high' - - // Check how many patterns matched - const pattern = [...this.customPatterns, ...this.domainPatterns] - .find(p => p.domain === domain) - - if (!pattern) return 'low' - - const score = this.scorePattern(data, pattern) - if (score >= 5) return 'high' - if (score >= 2) return 'medium' - return 'low' - } - - /** - * Categorize price ranges - */ - private getPriceRange(price: number): string { - if (price < 10) return 'low' - if (price < 100) return 'medium' - if (price < 1000) return 'high' - return 'premium' - } - - /** - * Categorize customer value - */ - private getValueCategory(value: number): string { - if (value < 100) return 'low' - if (value < 1000) return 'medium' - if (value < 10000) return 'high' - return 'vip' - } - - /** - * Categorize amount ranges - */ - private getAmountRange(amount: number): string { - if (amount < 100) return 'micro' - if (amount < 1000) return 'small' - if (amount < 10000) return 'medium' - if (amount < 100000) return 'large' - return 'enterprise' - } - - /** - * Add custom domain pattern - * @param pattern - Custom domain pattern to add - */ - addCustomPattern(pattern: DomainPattern): void { - // Remove existing pattern for same domain if exists - this.customPatterns = this.customPatterns.filter(p => p.domain !== pattern.domain) - this.customPatterns.push(pattern) - } - - /** - * Remove custom domain pattern - * @param domain - Domain to remove pattern for - */ - removeCustomPattern(domain: string): void { - this.customPatterns = this.customPatterns.filter(p => p.domain !== domain) - } - - /** - * Update domain statistics - */ - private updateStats(domain: string): void { - const count = this.domainStats.get(domain) || 0 - this.domainStats.set(domain, count + 1) - } - - /** - * Get domain statistics - * @returns Map of domain to count - */ - getDomainStats(): Map { - return new Map(this.domainStats) - } - - /** - * Clear domain statistics - */ - clearStats(): void { - this.domainStats.clear() - } - - /** - * Get all configured domains - * @returns Array of domain names - */ - getConfiguredDomains(): string[] { - const domains = new Set() - - for (const pattern of [...this.domainPatterns, ...this.customPatterns]) { - domains.add(pattern.domain) - } - - return Array.from(domains).sort() - } -} \ No newline at end of file diff --git a/src/distributed/hashPartitioner.ts b/src/distributed/hashPartitioner.ts deleted file mode 100644 index 3b8e26ed..00000000 --- a/src/distributed/hashPartitioner.ts +++ /dev/null @@ -1,170 +0,0 @@ -/** - * Hash-based Partitioner - * Provides deterministic partitioning for distributed writes - */ - -import { getPartitionHash } from '../utils/crypto.js' -import { SharedConfig } from '../types/distributedTypes.js' - -export class HashPartitioner { - private partitionCount: number - private partitionPrefix: string = 'vectors/p' - - constructor(config: SharedConfig) { - this.partitionCount = config.settings.partitionCount || 100 - } - - /** - * Get partition for a given vector ID using deterministic hashing - * @param vectorId - The unique identifier of the vector - * @returns The partition path - */ - getPartition(vectorId: string): string { - const hash = this.hashString(vectorId) - const partitionIndex = hash % this.partitionCount - return `${this.partitionPrefix}${partitionIndex.toString().padStart(3, '0')}` - } - - /** - * Get partition with domain metadata (domain stored as metadata, not in path) - * @param vectorId - The unique identifier of the vector - * @param domain - The domain identifier (for metadata only) - * @returns The partition path - */ - getPartitionWithDomain(vectorId: string, domain?: string): string { - // Domain doesn't affect partitioning - it's just metadata - return this.getPartition(vectorId) - } - - /** - * Get all partition paths - * @returns Array of all partition paths - */ - getAllPartitions(): string[] { - const partitions: string[] = [] - for (let i = 0; i < this.partitionCount; i++) { - partitions.push(`${this.partitionPrefix}${i.toString().padStart(3, '0')}`) - } - return partitions - } - - /** - * Get partition index from partition path - * @param partitionPath - The partition path - * @returns The partition index - */ - getPartitionIndex(partitionPath: string): number { - const match = partitionPath.match(/p(\d+)$/) - if (match) { - return parseInt(match[1], 10) - } - throw new Error(`Invalid partition path: ${partitionPath}`) - } - - /** - * Hash a string to a number for consistent partitioning - * @param str - The string to hash - * @returns A positive integer hash - */ - private hashString(str: string): number { - // Use our cross-platform hash function - return getPartitionHash(str) - } - - /** - * Get partitions for batch operations - * Groups vector IDs by their target partition - * @param vectorIds - Array of vector IDs - * @returns Map of partition to vector IDs - */ - getPartitionsForBatch(vectorIds: string[]): Map { - const partitionMap = new Map() - - for (const id of vectorIds) { - const partition = this.getPartition(id) - if (!partitionMap.has(partition)) { - partitionMap.set(partition, []) - } - partitionMap.get(partition)!.push(id) - } - - return partitionMap - } -} - -/** - * Affinity-based Partitioner - * Extends HashPartitioner to prefer certain partitions for a writer - * while maintaining correctness - */ -export class AffinityPartitioner extends HashPartitioner { - private preferredPartitions: Set - private instanceId: string - - constructor(config: SharedConfig, instanceId: string) { - super(config) - this.instanceId = instanceId - this.preferredPartitions = this.calculatePreferredPartitions(config) - } - - /** - * Calculate preferred partitions for this instance - */ - private calculatePreferredPartitions(config: SharedConfig): Set { - const partitionCount = config.settings.partitionCount || 100 - const writers = Object.entries(config.instances) - .filter(([_, inst]) => inst.role === 'writer') - .map(([id, _]) => id) - .sort() // Ensure consistent ordering - - const writerIndex = writers.indexOf(this.instanceId) - if (writerIndex === -1) { - // Not a writer or not found, no preferences - return new Set() - } - - const writerCount = writers.length - const partitionsPerWriter = Math.ceil(partitionCount / writerCount) - - const preferred = new Set() - const start = writerIndex * partitionsPerWriter - const end = Math.min(start + partitionsPerWriter, partitionCount) - - for (let i = start; i < end; i++) { - preferred.add(i) - } - - return preferred - } - - /** - * Check if a partition is preferred for this instance - * @param partitionPath - The partition path - * @returns Whether this partition is preferred - */ - isPreferredPartition(partitionPath: string): boolean { - try { - const index = this.getPartitionIndex(partitionPath) - return this.preferredPartitions.has(index) - } catch { - return false - } - } - - /** - * Get all preferred partitions for this instance - * @returns Array of preferred partition paths - */ - getPreferredPartitions(): string[] { - return Array.from(this.preferredPartitions) - .map(index => `vectors/p${index.toString().padStart(3, '0')}`) - } - - /** - * Update preferred partitions based on new config - * @param config - The updated shared configuration - */ - updatePreferences(config: SharedConfig): void { - this.preferredPartitions = this.calculatePreferredPartitions(config) - } -} \ No newline at end of file diff --git a/src/distributed/healthMonitor.ts b/src/distributed/healthMonitor.ts deleted file mode 100644 index e35e3e54..00000000 --- a/src/distributed/healthMonitor.ts +++ /dev/null @@ -1,301 +0,0 @@ -/** - * Health Monitor - * Monitors and reports instance health in distributed deployments - */ - -import { DistributedConfigManager } from './configManager.js' -import { InstanceInfo } from '../types/distributedTypes.js' - -export interface HealthMetrics { - vectorCount: number - cacheHitRate: number - memoryUsage: number - cpuUsage?: number - requestsPerSecond?: number - averageLatency?: number - errorRate?: number -} - -export interface HealthStatus { - status: 'healthy' | 'degraded' | 'unhealthy' - instanceId: string - role: string - uptime: number - lastCheck: string - metrics: HealthMetrics - warnings?: string[] - errors?: string[] -} - -export class HealthMonitor { - private configManager: DistributedConfigManager - private startTime: number - private requestCount: number = 0 - private errorCount: number = 0 - private totalLatency: number = 0 - private cacheHits: number = 0 - private cacheMisses: number = 0 - private vectorCount: number = 0 - private checkInterval: number = 30000 // 30 seconds - private healthCheckTimer?: NodeJS.Timeout - private metricsWindow: number[] = [] // Sliding window for RPS calculation - private latencyWindow: number[] = [] // Sliding window for latency - private windowSize: number = 60000 // 1 minute window - - constructor(configManager: DistributedConfigManager) { - this.configManager = configManager - this.startTime = Date.now() - } - - /** - * Start health monitoring - */ - start(): void { - // Initial health update - this.updateHealth() - - // Schedule periodic health checks - this.healthCheckTimer = setInterval(() => { - this.updateHealth() - }, this.checkInterval) - } - - /** - * Stop health monitoring - */ - stop(): void { - if (this.healthCheckTimer) { - clearInterval(this.healthCheckTimer) - this.healthCheckTimer = undefined - } - } - - /** - * Update health status and metrics - */ - private async updateHealth(): Promise { - const metrics = this.collectMetrics() - - // Update config with latest metrics - await this.configManager.updateMetrics({ - vectorCount: metrics.vectorCount, - cacheHitRate: metrics.cacheHitRate, - memoryUsage: metrics.memoryUsage, - cpuUsage: metrics.cpuUsage - }) - - // Clean sliding windows - this.cleanWindows() - } - - /** - * Collect current metrics - */ - private collectMetrics(): HealthMetrics { - const memUsage = process.memoryUsage() - - return { - vectorCount: this.vectorCount, - cacheHitRate: this.calculateCacheHitRate(), - memoryUsage: memUsage.heapUsed, - cpuUsage: this.getCPUUsage(), - requestsPerSecond: this.calculateRPS(), - averageLatency: this.calculateAverageLatency(), - errorRate: this.calculateErrorRate() - } - } - - /** - * Calculate cache hit rate - */ - private calculateCacheHitRate(): number { - const total = this.cacheHits + this.cacheMisses - if (total === 0) return 0 - return this.cacheHits / total - } - - /** - * Calculate requests per second - */ - private calculateRPS(): number { - const now = Date.now() - const recentRequests = this.metricsWindow.filter( - timestamp => now - timestamp < this.windowSize - ) - return recentRequests.length / (this.windowSize / 1000) - } - - /** - * Calculate average latency - */ - private calculateAverageLatency(): number { - if (this.latencyWindow.length === 0) return 0 - const sum = this.latencyWindow.reduce((a, b) => a + b, 0) - return sum / this.latencyWindow.length - } - - /** - * Calculate error rate - */ - private calculateErrorRate(): number { - if (this.requestCount === 0) return 0 - return this.errorCount / this.requestCount - } - - /** - * Get CPU usage (simplified) - */ - private getCPUUsage(): number { - // Simplified CPU usage based on process time - const usage = process.cpuUsage() - const total = usage.user + usage.system - const seconds = (Date.now() - this.startTime) / 1000 - return Math.min(100, (total / 1000000 / seconds) * 100) - } - - /** - * Clean old entries from sliding windows - */ - private cleanWindows(): void { - const now = Date.now() - const cutoff = now - this.windowSize - - this.metricsWindow = this.metricsWindow.filter(t => t > cutoff) - - // Keep only recent latency measurements - if (this.latencyWindow.length > 100) { - this.latencyWindow = this.latencyWindow.slice(-100) - } - } - - /** - * Record a request - * @param latency - Request latency in milliseconds - * @param error - Whether the request resulted in an error - */ - recordRequest(latency: number, error: boolean = false): void { - this.requestCount++ - this.metricsWindow.push(Date.now()) - this.latencyWindow.push(latency) - - if (error) { - this.errorCount++ - } - } - - /** - * Record cache access - * @param hit - Whether it was a cache hit - */ - recordCacheAccess(hit: boolean): void { - if (hit) { - this.cacheHits++ - } else { - this.cacheMisses++ - } - } - - /** - * Update vector count - * @param count - New vector count - */ - updateVectorCount(count: number): void { - this.vectorCount = count - } - - /** - * Get current health status - * @returns Health status object - */ - getHealthStatus(): HealthStatus { - const metrics = this.collectMetrics() - const uptime = Date.now() - this.startTime - const warnings: string[] = [] - const errors: string[] = [] - - // Check for warnings - if (metrics.memoryUsage > 1024 * 1024 * 1024) { // > 1GB - warnings.push('High memory usage detected') - } - - if (metrics.cacheHitRate < 0.5) { - warnings.push('Low cache hit rate') - } - - if (metrics.errorRate && metrics.errorRate > 0.05) { - warnings.push('High error rate detected') - } - - if (metrics.averageLatency && metrics.averageLatency > 1000) { - warnings.push('High latency detected') - } - - // Check for errors - if (metrics.memoryUsage > 2 * 1024 * 1024 * 1024) { // > 2GB - errors.push('Critical memory usage') - } - - if (metrics.errorRate && metrics.errorRate > 0.2) { - errors.push('Critical error rate') - } - - // Determine overall status - let status: 'healthy' | 'degraded' | 'unhealthy' = 'healthy' - if (errors.length > 0) { - status = 'unhealthy' - } else if (warnings.length > 0) { - status = 'degraded' - } - - return { - status, - instanceId: this.configManager.getInstanceId(), - role: this.configManager.getRole(), - uptime, - lastCheck: new Date().toISOString(), - metrics, - warnings: warnings.length > 0 ? warnings : undefined, - errors: errors.length > 0 ? errors : undefined - } - } - - /** - * Get health check endpoint data - * @returns JSON-serializable health data - */ - getHealthEndpointData(): Record { - const status = this.getHealthStatus() - - return { - status: status.status, - instanceId: status.instanceId, - role: status.role, - uptime: Math.floor(status.uptime / 1000), // Convert to seconds - lastCheck: status.lastCheck, - metrics: { - vectorCount: status.metrics.vectorCount, - cacheHitRate: Math.round(status.metrics.cacheHitRate * 100) / 100, - memoryUsageMB: Math.round(status.metrics.memoryUsage / 1024 / 1024), - cpuUsagePercent: Math.round(status.metrics.cpuUsage || 0), - requestsPerSecond: Math.round(status.metrics.requestsPerSecond || 0), - averageLatencyMs: Math.round(status.metrics.averageLatency || 0), - errorRate: Math.round((status.metrics.errorRate || 0) * 100) / 100 - }, - warnings: status.warnings, - errors: status.errors - } - } - - /** - * Reset metrics (useful for testing) - */ - resetMetrics(): void { - this.requestCount = 0 - this.errorCount = 0 - this.totalLatency = 0 - this.cacheHits = 0 - this.cacheMisses = 0 - this.metricsWindow = [] - this.latencyWindow = [] - } -} \ No newline at end of file diff --git a/src/distributed/httpTransport.ts b/src/distributed/httpTransport.ts deleted file mode 100644 index 50de7778..00000000 --- a/src/distributed/httpTransport.ts +++ /dev/null @@ -1,547 +0,0 @@ -/** - * HTTP + SSE Transport for Zero-Config Distributed Brainy - * Simple, reliable, works everywhere - no WebSocket complexity! - * REAL PRODUCTION CODE - Handles millions of operations - */ - -import * as http from 'node:http' -import * as https from 'node:https' -import { EventEmitter } from 'node:events' -import * as net from 'node:net' -import { URL } from 'node:url' - -export interface TransportMessage { - id: string - method: string - params: any - timestamp: number - from: string - to?: string -} - -export interface TransportResponse { - id: string - result?: any - error?: { - code: number - message: string - data?: any - } - timestamp: number -} - -export interface SSEClient { - id: string - response: http.ServerResponse - lastPing: number -} - -export class HTTPTransport extends EventEmitter { - private server: http.Server | null = null - private port: number = 0 - private nodeId: string - private endpoints: Map = new Map() // nodeId -> endpoint - private pendingRequests: Map void - reject: (error: any) => void - timeout: NodeJS.Timeout - }> = new Map() - private sseClients: Map = new Map() - private messageHandlers: Map Promise> = new Map() - private isRunning: boolean = false - private readonly REQUEST_TIMEOUT = 30000 // 30 seconds - private readonly SSE_HEARTBEAT_INTERVAL = 15000 // 15 seconds - private sseHeartbeatTimer: NodeJS.Timeout | null = null - - constructor(nodeId: string) { - super() - this.nodeId = nodeId - } - - /** - * Start HTTP server with automatic port selection - */ - async start(): Promise { - if (this.isRunning) return this.port - - // Create HTTP server with all handlers - this.server = http.createServer(async (req, res) => { - // Enable CORS for browser compatibility - res.setHeader('Access-Control-Allow-Origin', '*') - res.setHeader('Access-Control-Allow-Methods', 'GET, POST, OPTIONS') - res.setHeader('Access-Control-Allow-Headers', 'Content-Type') - - if (req.method === 'OPTIONS') { - res.writeHead(204) - res.end() - return - } - - const url = new URL(req.url || '/', `http://${req.headers.host}`) - - try { - // Route requests - if (url.pathname === '/health') { - await this.handleHealth(req, res) - } else if (url.pathname === '/rpc') { - await this.handleRPC(req, res) - } else if (url.pathname === '/events') { - await this.handleSSE(req, res) - } else if (url.pathname.startsWith('/stream/')) { - await this.handleStream(req, res, url) - } else { - res.writeHead(404) - res.end('Not Found') - } - } catch (err) { - console.error(`[${this.nodeId}] Request error:`, err) - res.writeHead(500) - res.end('Internal Server Error') - } - }) - - // Find available port - this.port = await this.findAvailablePort() - - // Start server - await new Promise((resolve, reject) => { - this.server!.listen(this.port, () => { - console.log(`[${this.nodeId}] HTTP transport listening on port ${this.port}`) - resolve() - }) - this.server!.on('error', reject) - }) - - this.isRunning = true - - // Start SSE heartbeat - this.startSSEHeartbeat() - - this.emit('started', this.port) - - return this.port - } - - /** - * Stop HTTP server - */ - async stop(): Promise { - if (!this.isRunning) return - - this.isRunning = false - - // Stop SSE heartbeat - if (this.sseHeartbeatTimer) { - clearInterval(this.sseHeartbeatTimer) - this.sseHeartbeatTimer = null - } - - // Close all SSE connections - for (const client of this.sseClients.values()) { - client.response.end() - } - this.sseClients.clear() - - // Cancel pending requests - for (const pending of this.pendingRequests.values()) { - clearTimeout(pending.timeout) - pending.reject(new Error('Transport shutting down')) - } - this.pendingRequests.clear() - - // Close server - if (this.server) { - await new Promise(resolve => { - this.server!.close(() => resolve()) - }) - this.server = null - } - - this.emit('stopped') - } - - /** - * Register a node endpoint - */ - registerEndpoint(nodeId: string, endpoint: string): void { - this.endpoints.set(nodeId, endpoint) - this.emit('endpointRegistered', { nodeId, endpoint }) - } - - /** - * Register RPC method handler - */ - registerHandler(method: string, handler: (params: any, from: string) => Promise): void { - this.messageHandlers.set(method, handler) - } - - /** - * Call RPC method on remote node - */ - async call(nodeId: string, method: string, params: any): Promise { - const endpoint = this.endpoints.get(nodeId) - if (!endpoint) { - throw new Error(`No endpoint registered for node ${nodeId}`) - } - - const message: TransportMessage = { - id: this.generateId(), - method, - params, - timestamp: Date.now(), - from: this.nodeId, - to: nodeId - } - - // Send HTTP request - const response = await this.sendHTTPRequest(endpoint, '/rpc', message) - - if (response.error) { - throw new Error(response.error.message) - } - - return response.result - } - - /** - * Broadcast to all SSE clients - */ - broadcast(event: string, data: any): void { - const message = JSON.stringify({ event, data, timestamp: Date.now() }) - - for (const [clientId, client] of this.sseClients.entries()) { - try { - client.response.write(`data: ${message}\n\n`) - } catch (err) { - // Client disconnected - console.debug(`[${this.nodeId}] SSE client ${clientId} disconnected`) - this.sseClients.delete(clientId) - } - } - } - - /** - * Handle health check - */ - private async handleHealth(req: http.IncomingMessage, res: http.ServerResponse): Promise { - res.writeHead(200, { 'Content-Type': 'application/json' }) - res.end(JSON.stringify({ - status: 'healthy', - nodeId: this.nodeId, - uptime: process.uptime(), - memory: process.memoryUsage(), - connections: this.sseClients.size - })) - } - - /** - * Handle RPC requests - */ - private async handleRPC(req: http.IncomingMessage, res: http.ServerResponse): Promise { - if (req.method !== 'POST') { - res.writeHead(405) - res.end('Method Not Allowed') - return - } - - // Read body - const body = await this.readBody(req) - let message: TransportMessage - - try { - message = JSON.parse(body) - } catch (err) { - res.writeHead(400) - res.end('Invalid JSON') - return - } - - // Handle the RPC call - const response: TransportResponse = { - id: message.id, - timestamp: Date.now() - } - - try { - const handler = this.messageHandlers.get(message.method) - if (!handler) { - throw new Error(`Method ${message.method} not found`) - } - - response.result = await handler(message.params, message.from) - } catch (err: any) { - response.error = { - code: -32603, - message: err.message, - data: err.stack - } - } - - res.writeHead(200, { 'Content-Type': 'application/json' }) - res.end(JSON.stringify(response)) - } - - /** - * Handle SSE connections for real-time updates - */ - private async handleSSE(req: http.IncomingMessage, res: http.ServerResponse): Promise { - // Setup SSE headers - res.writeHead(200, { - 'Content-Type': 'text/event-stream', - 'Cache-Control': 'no-cache', - 'Connection': 'keep-alive', - 'X-Accel-Buffering': 'no' // Disable Nginx buffering - }) - - // Send initial connection event - const clientId = this.generateId() - res.write(`data: ${JSON.stringify({ - event: 'connected', - clientId, - nodeId: this.nodeId - })}\n\n`) - - // Store client - this.sseClients.set(clientId, { - id: clientId, - response: res, - lastPing: Date.now() - }) - - // Handle client disconnect - req.on('close', () => { - this.sseClients.delete(clientId) - this.emit('sseDisconnected', clientId) - }) - - this.emit('sseConnected', clientId) - } - - /** - * Handle streaming data (for shard migration) - */ - private async handleStream( - req: http.IncomingMessage, - res: http.ServerResponse, - url: URL - ): Promise { - const streamId = url.pathname.split('/').pop() - - if (req.method === 'POST') { - // Receiving stream - await this.handleStreamUpload(req, res, streamId!) - } else if (req.method === 'GET') { - // Sending stream - await this.handleStreamDownload(req, res, streamId!) - } else { - res.writeHead(405) - res.end('Method Not Allowed') - } - } - - /** - * Handle stream upload (receiving data) - */ - private async handleStreamUpload( - req: http.IncomingMessage, - res: http.ServerResponse, - streamId: string - ): Promise { - const chunks: Buffer[] = [] - let totalSize = 0 - - req.on('data', (chunk: Buffer) => { - chunks.push(chunk) - totalSize += chunk.length - - // Emit progress - this.emit('streamProgress', { - streamId, - type: 'upload', - bytes: totalSize - }) - }) - - req.on('end', () => { - const data = Buffer.concat(chunks) - - // Process the streamed data - this.emit('streamComplete', { - streamId, - type: 'upload', - data, - size: totalSize - }) - - res.writeHead(200, { 'Content-Type': 'application/json' }) - res.end(JSON.stringify({ - success: true, - streamId, - size: totalSize - })) - }) - - req.on('error', (err) => { - console.error(`[${this.nodeId}] Stream upload error:`, err) - res.writeHead(500) - res.end('Stream Error') - }) - } - - /** - * Handle stream download (sending data) - */ - private async handleStreamDownload( - req: http.IncomingMessage, - res: http.ServerResponse, - streamId: string - ): Promise { - // Stream download not yet implemented - // Return error response instead of fake data - res.writeHead(501, { - 'Content-Type': 'application/json' - }) - - res.end(JSON.stringify({ - error: 'Stream download not implemented', - message: 'This feature is not yet available in the current version', - streamId - })) - - this.emit('streamError', { - streamId, - type: 'download', - error: 'Not implemented' - }) - } - - /** - * Send HTTP request to another node - */ - private async sendHTTPRequest( - endpoint: string, - path: string, - data: any - ): Promise { - return new Promise((resolve, reject) => { - const url = new URL(path, endpoint) - const options = { - method: 'POST', - headers: { - 'Content-Type': 'application/json' - } - } - - const proto = url.protocol === 'https:' ? https : http - const req = proto.request(url, options, (res) => { - let body = '' - - res.on('data', chunk => body += chunk) - res.on('end', () => { - try { - const response = JSON.parse(body) - resolve(response) - } catch (err) { - reject(new Error(`Invalid response: ${body}`)) - } - }) - }) - - req.on('error', reject) - req.on('timeout', () => { - req.destroy() - reject(new Error('Request timeout')) - }) - - req.setTimeout(this.REQUEST_TIMEOUT) - req.write(JSON.stringify(data)) - req.end() - }) - } - - /** - * Read request body - */ - private readBody(req: http.IncomingMessage): Promise { - return new Promise((resolve, reject) => { - let body = '' - req.on('data', chunk => body += chunk) - req.on('end', () => resolve(body)) - req.on('error', reject) - }) - } - - /** - * Find an available port - */ - private async findAvailablePort(startPort: number = 7947): Promise { - const checkPort = (port: number): Promise => { - return new Promise(resolve => { - const server = net.createServer() - server.once('error', () => resolve(false)) - server.once('listening', () => { - server.close() - resolve(true) - }) - server.listen(port) - }) - } - - // Try preferred port first - if (await checkPort(startPort)) { - return startPort - } - - // Find random available port - return new Promise((resolve, reject) => { - const server = net.createServer() - server.once('error', reject) - server.once('listening', () => { - const port = (server.address() as net.AddressInfo).port - server.close(() => resolve(port)) - }) - server.listen(0) // 0 = random available port - }) - } - - /** - * Start SSE heartbeat to keep connections alive - */ - private startSSEHeartbeat(): void { - this.sseHeartbeatTimer = setInterval(() => { - const now = Date.now() - const ping = JSON.stringify({ event: 'ping', timestamp: now }) - - for (const [clientId, client] of this.sseClients.entries()) { - try { - client.response.write(`: ping\n\n`) // SSE comment for keep-alive - client.lastPing = now - } catch (err) { - // Client disconnected - this.sseClients.delete(clientId) - } - } - }, this.SSE_HEARTBEAT_INTERVAL) - } - - /** - * Generate unique ID - */ - private generateId(): string { - return `${Date.now()}-${Math.random().toString(36).substring(2, 11)}` - } - - /** - * Get connected nodes count - */ - getConnectionCount(): number { - return this.endpoints.size - } - - /** - * Get SSE client count - */ - getSSEClientCount(): number { - return this.sseClients.size - } -} \ No newline at end of file diff --git a/src/distributed/index.ts b/src/distributed/index.ts deleted file mode 100644 index 9eb39dba..00000000 --- a/src/distributed/index.ts +++ /dev/null @@ -1,54 +0,0 @@ -/** - * Distributed module exports - */ - -export { DistributedConfigManager } from './configManager.js' -export { HashPartitioner, AffinityPartitioner } from './hashPartitioner.js' -export { - BaseOperationalMode, - ReaderMode, - WriterMode, - HybridMode, - OperationalModeFactory -} from './operationalModes.js' -export { DomainDetector } from './domainDetector.js' -export { HealthMonitor } from './healthMonitor.js' - -// New distributed scaling components -export { DistributedCoordinator, createCoordinator } from './coordinator.js' -export { ShardManager, createShardManager } from './shardManager.js' -export { CacheSync, createCacheSync } from './cacheSync.js' -export { ReadWriteSeparation, createReadWriteSeparation } from './readWriteSeparation.js' - -export type { - HealthMetrics, - HealthStatus -} from './healthMonitor.js' - -export type { - DomainPattern -} from './domainDetector.js' - -export type { - NodeInfo, - CoordinatorConfig, - ConsensusState -} from './coordinator.js' - -export type { - ShardConfig, - Shard, - ShardAssignment -} from './shardManager.js' - -export type { - CacheSyncConfig, - CacheEntry, - SyncMessage -} from './cacheSync.js' - -export type { - ReplicationConfig, - WriteOperation, - ReplicationLog -} from './readWriteSeparation.js' \ No newline at end of file diff --git a/src/distributed/networkTransport.ts b/src/distributed/networkTransport.ts deleted file mode 100644 index 9f16be7a..00000000 --- a/src/distributed/networkTransport.ts +++ /dev/null @@ -1,739 +0,0 @@ -/** - * Network Transport Layer for Distributed Brainy - * Uses WebSocket + HTTP for maximum compatibility - */ - -import * as http from 'node:http' -import { EventEmitter } from 'node:events' -import { WebSocket } from 'ws' - -// Use dynamic imports for Node.js specific modules -let WebSocketServer: any - -// Default ports -const HTTP_PORT = process.env.BRAINY_HTTP_PORT || 7947 -const WS_PORT = process.env.BRAINY_WS_PORT || 7948 - -export interface NetworkMessage { - type: string - from: string - to?: string - data: any - timestamp: number - id: string -} - -export interface NodeEndpoint { - nodeId: string - host: string - httpPort: number - wsPort: number - lastSeen: number -} - -export interface NetworkConfig { - nodeId?: string - host?: string - httpPort?: number - wsPort?: number - seeds?: string[] // Known node addresses for bootstrap - discoveryMethod?: 'seeds' | 'dns' | 'kubernetes' | 'auto' - enableUDP?: boolean // Optional UDP discovery for LAN -} - -/** - * Production-ready network transport - */ -export class NetworkTransport extends EventEmitter { - private nodeId: string - private config: NetworkConfig - private httpServer?: http.Server - private wsServer: any - private peers: Map = new Map() - private connections: Map = new Map() - private messageHandlers: Map Promise> = new Map() - private responseHandlers: Map void> = new Map() - private isRunning = false - - constructor(config: NetworkConfig) { - super() - this.nodeId = config.nodeId || this.generateNodeId() - this.config = { - host: '0.0.0.0', - httpPort: Number(HTTP_PORT), - wsPort: Number(WS_PORT), - discoveryMethod: 'auto', - enableUDP: false, // Disabled by default for cloud compatibility - ...config - } - } - - /** - * Start network transport - */ - async start(): Promise { - if (this.isRunning) return - - const ws = await import('ws') - // The ws package has exported its server class as `WebSocketServer` - // (v8+), `Server` (older releases) and via the default export — and the - // available type declarations disagree on which of those are value - // exports, so probe all three through an optional-keyed module view. - const wsModule = ws as typeof ws & { WebSocketServer?: unknown; Server?: unknown } - WebSocketServer = wsModule.WebSocketServer || wsModule.Server || ws.default - - await this.startHTTPServer() - await this.startWebSocketServer() - await this.discoverPeers() - - this.isRunning = true - this.emit('started', { nodeId: this.nodeId }) - - // Start heartbeat - this.startHeartbeat() - } - - /** - * Stop network transport - */ - async stop(): Promise { - if (!this.isRunning) return - - this.isRunning = false - - // Close all connections - for (const ws of this.connections.values()) { - ws.close() - } - this.connections.clear() - - // Stop servers - if (this.httpServer) { - await new Promise((resolve) => { - this.httpServer!.close(() => resolve()) - }) - } - - if (this.wsServer) { - await new Promise((resolve) => { - this.wsServer.close(() => resolve()) - }) - } - - this.emit('stopped') - } - - /** - * Start HTTP server for REST API and health checks - */ - private async startHTTPServer(): Promise { - return new Promise((resolve, reject) => { - this.httpServer = http.createServer(async (req, res) => { - // Enable CORS - res.setHeader('Access-Control-Allow-Origin', '*') - res.setHeader('Access-Control-Allow-Methods', 'GET, POST, OPTIONS') - res.setHeader('Access-Control-Allow-Headers', 'Content-Type') - - if (req.method === 'OPTIONS') { - res.writeHead(200) - res.end() - return - } - - if (req.url === '/health') { - // Health check endpoint - res.writeHead(200, { 'Content-Type': 'application/json' }) - res.end(JSON.stringify({ - status: 'healthy', - nodeId: this.nodeId, - peers: Array.from(this.peers.keys()), - connections: Array.from(this.connections.keys()) - })) - return - } - - if (req.url === '/peers') { - // Peer discovery endpoint - res.writeHead(200, { 'Content-Type': 'application/json' }) - res.end(JSON.stringify({ - nodeId: this.nodeId, - endpoint: { - host: this.config.host, - httpPort: this.config.httpPort, - wsPort: this.config.wsPort - }, - peers: Array.from(this.peers.values()) - })) - return - } - - if (req.method === 'POST' && req.url === '/message') { - // Message handling endpoint - let body = '' - - req.on('data', (chunk) => { - body += chunk.toString() - }) - - req.on('end', async () => { - try { - const message: NetworkMessage = JSON.parse(body) - - // Handle message - const handler = this.messageHandlers.get(message.type) - if (handler) { - const response = await handler(message) - res.writeHead(200, { 'Content-Type': 'application/json' }) - res.end(JSON.stringify({ success: true, data: response })) - } else { - res.writeHead(404) - res.end(JSON.stringify({ success: false, error: 'Unknown message type' })) - } - } catch (err: any) { - res.writeHead(500) - res.end(JSON.stringify({ success: false, error: err.message })) - } - }) - } else { - res.writeHead(404) - res.end(JSON.stringify({ error: 'Not found' })) - } - }) - - this.httpServer.listen(this.config.httpPort, '0.0.0.0', () => { - console.log(`[Network] HTTP server listening on :${this.config.httpPort}`) - resolve() - }) - - this.httpServer.on('error', reject) - }) - } - - /** - * Start WebSocket server for real-time communication - */ - private async startWebSocketServer(): Promise { - if (!WebSocketServer) { - console.warn('[Network] WebSocket not available in this environment') - return - } - - this.wsServer = new WebSocketServer({ port: this.config.wsPort }) - - this.wsServer.on('connection', (ws: WebSocket) => { - let peerId: string | null = null - - ws.on('message', async (data: Buffer | string) => { - try { - const message: NetworkMessage = JSON.parse(data.toString()) - - // Handle handshake - if (message.type === 'HANDSHAKE') { - peerId = message.from - this.connections.set(peerId, ws) - - // Add to peers - const endpoint: NodeEndpoint = { - nodeId: peerId, - host: message.data.host || 'unknown', - httpPort: message.data.httpPort || 0, - wsPort: message.data.wsPort || 0, - lastSeen: Date.now() - } - this.peers.set(peerId, endpoint) - - // Send handshake response - ws.send(JSON.stringify({ - type: 'HANDSHAKE_ACK', - from: this.nodeId, - to: peerId, - data: { - nodeId: this.nodeId, - host: this.config.host, - httpPort: this.config.httpPort, - wsPort: this.config.wsPort - }, - timestamp: Date.now(), - id: this.generateMessageId() - })) - - this.emit('peerConnected', peerId) - return - } - - // Handle response messages - if (message.type.endsWith('_RESPONSE')) { - const handler = this.responseHandlers.get(message.id) - if (handler) { - handler(message.data) - this.responseHandlers.delete(message.id) - } - return - } - - // Handle regular messages - const handler = this.messageHandlers.get(message.type) - if (handler) { - const response = await handler(message) - if (response !== undefined) { - ws.send(JSON.stringify({ - type: `${message.type}_RESPONSE`, - from: this.nodeId, - to: message.from, - data: response, - timestamp: Date.now(), - id: message.id - })) - } - } - } catch (err) { - console.error('[Network] WebSocket message error:', err) - } - }) - - ws.on('close', () => { - if (peerId) { - this.connections.delete(peerId) - this.emit('peerDisconnected', peerId) - } - }) - - ws.on('error', (err: Error) => { - console.error(`[Network] WebSocket error:`, err.message) - }) - }) - - console.log(`[Network] WebSocket server listening on :${this.config.wsPort}`) - } - - /** - * Discover peers based on configuration - */ - private async discoverPeers(): Promise { - const method = this.config.discoveryMethod - - if (method === 'seeds' || (method === 'auto' && this.config.seeds)) { - // Use seed nodes - await this.connectToSeeds() - } else if (method === 'dns' || (method === 'auto' && process.env.BRAINY_DNS)) { - // Use DNS discovery - await this.discoverViaDNS() - } else if (method === 'kubernetes' || (method === 'auto' && process.env.KUBERNETES_SERVICE_HOST)) { - // Use Kubernetes service discovery - await this.discoverViaKubernetes() - } - - // Fallback: try localhost for development - if (this.peers.size === 0 && process.env.NODE_ENV !== 'production') { - await this.connectToNode('localhost', this.config.httpPort!) - } - } - - /** - * Connect to seed nodes - */ - private async connectToSeeds(): Promise { - if (!this.config.seeds) return - - for (const seed of this.config.seeds) { - try { - // Parse seed address (host:port or just host) - const [host, port] = seed.split(':') - await this.connectToNode(host, Number(port) || this.config.httpPort!) - } catch (err) { - console.error(`[Network] Failed to connect to seed ${seed}:`, err) - } - } - } - - /** - * Discover peers via DNS - */ - private async discoverViaDNS(): Promise { - const dnsName = process.env.BRAINY_DNS || 'brainy-cluster.local' - - try { - const dns = await import('dns') - const addresses = await new Promise((resolve, reject) => { - dns.resolve4(dnsName, (err, addresses) => { - if (err) reject(err) - else resolve(addresses || []) - }) - }) - - for (const address of addresses) { - await this.connectToNode(address, this.config.httpPort!) - } - } catch (err) { - console.log(`[Network] DNS discovery failed for ${dnsName}:`, err) - } - } - - /** - * Discover peers via Kubernetes - */ - private async discoverViaKubernetes(): Promise { - const serviceName = process.env.BRAINY_SERVICE || 'brainy' - const namespace = process.env.BRAINY_NAMESPACE || 'default' - const apiServer = 'https://kubernetes.default.svc' - const token = process.env.KUBERNETES_TOKEN || '' - - try { - // Query Kubernetes API for pod endpoints - const https = await import('node:https') - const response = await new Promise((resolve, reject) => { - https.get( - `${apiServer}/api/v1/namespaces/${namespace}/endpoints/${serviceName}`, - { - headers: { - 'Authorization': `Bearer ${token}` - } - }, - (res) => { - let data = '' - res.on('data', (chunk) => data += chunk) - res.on('end', () => resolve(JSON.parse(data))) - } - ).on('error', reject) - }) - - // Connect to each pod - if (response.subsets) { - for (const subset of response.subsets) { - for (const address of subset.addresses || []) { - await this.connectToNode(address.ip, this.config.httpPort!) - } - } - } - } catch (err) { - console.log('[Network] Kubernetes discovery failed:', err) - } - } - - /** - * Connect to a specific node - */ - private async connectToNode(host: string, httpPort: number): Promise { - try { - // First, get node info via HTTP - const nodeInfo = await this.getNodeInfo(host, httpPort) - - if (nodeInfo.nodeId === this.nodeId) { - return // Don't connect to self - } - - // Add to peers - const endpoint: NodeEndpoint = { - nodeId: nodeInfo.nodeId, - host, - httpPort, - wsPort: nodeInfo.endpoint.wsPort, - lastSeen: Date.now() - } - this.peers.set(nodeInfo.nodeId, endpoint) - - // Connect via WebSocket - await this.connectWebSocket(endpoint) - - // Get their peer list - for (const peer of nodeInfo.peers || []) { - if (!this.peers.has(peer.nodeId) && peer.nodeId !== this.nodeId) { - this.peers.set(peer.nodeId, peer) - // Optionally connect to them too - if (this.peers.size < 10) { // Limit connections - await this.connectWebSocket(peer) - } - } - } - } catch (err) { - // Node might be down or not ready - console.debug(`[Network] Could not connect to ${host}:${httpPort}`) - } - } - - /** - * Get node information via HTTP - */ - private async getNodeInfo(host: string, port: number): Promise { - return new Promise((resolve, reject) => { - const req = http.get(`http://${host}:${port}/peers`, (res) => { - let data = '' - res.on('data', (chunk) => data += chunk) - res.on('end', () => { - try { - resolve(JSON.parse(data)) - } catch (err) { - reject(err) - } - }) - }) - - req.on('error', reject) - req.setTimeout(2000, () => { - req.destroy() - reject(new Error('Timeout')) - }) - }) - } - - /** - * Connect to peer via WebSocket - */ - private async connectWebSocket(endpoint: NodeEndpoint): Promise { - if (this.connections.has(endpoint.nodeId)) return - - try { - const ws = new WebSocket(`ws://${endpoint.host}:${endpoint.wsPort}`) - - ws.on('open', () => { - // Send handshake - ws.send(JSON.stringify({ - type: 'HANDSHAKE', - from: this.nodeId, - data: { - nodeId: this.nodeId, - host: this.config.host, - httpPort: this.config.httpPort, - wsPort: this.config.wsPort - }, - timestamp: Date.now(), - id: this.generateMessageId() - })) - - this.connections.set(endpoint.nodeId, ws) - }) - - ws.on('message', async (data: Buffer | string) => { - try { - const message: NetworkMessage = JSON.parse(data.toString()) - - // Handle responses - if (message.type.endsWith('_RESPONSE')) { - const handler = this.responseHandlers.get(message.id) - if (handler) { - handler(message.data) - this.responseHandlers.delete(message.id) - } - return - } - - // Handle messages - const handler = this.messageHandlers.get(message.type) - if (handler) { - await handler(message) - } - } catch (err) { - console.error('[Network] Message handling error:', err) - } - }) - - ws.on('close', () => { - this.connections.delete(endpoint.nodeId) - }) - - ws.on('error', (err: Error) => { - console.debug(`[Network] WebSocket error with ${endpoint.nodeId}:`, err.message) - }) - } catch (err) { - console.debug(`[Network] Failed to connect to ${endpoint.nodeId}`) - } - } - - /** - * Start heartbeat to maintain connections - */ - private startHeartbeat(): void { - setInterval(() => { - if (!this.isRunning) return - - // Send heartbeat to all connected peers - for (const [nodeId, ws] of this.connections.entries()) { - if (ws.readyState === WebSocket.OPEN) { - ws.send(JSON.stringify({ - type: 'HEARTBEAT', - from: this.nodeId, - to: nodeId, - timestamp: Date.now(), - id: this.generateMessageId() - })) - } else { - // Connection lost, remove it - this.connections.delete(nodeId) - } - } - - // Clean up stale peers - const now = Date.now() - for (const [nodeId, peer] of this.peers.entries()) { - if (now - peer.lastSeen > 60000) { // 60 seconds - this.peers.delete(nodeId) - } - } - }, 10000) // Every 10 seconds - } - - /** - * Send message to specific node - */ - async sendToNode(nodeId: string, type: string, data: any): Promise { - const ws = this.connections.get(nodeId) - - if (ws && ws.readyState === WebSocket.OPEN) { - // Use WebSocket - return new Promise((resolve, reject) => { - const messageId = this.generateMessageId() - - const timeout = setTimeout(() => { - this.responseHandlers.delete(messageId) - reject(new Error(`Timeout waiting for response from ${nodeId}`)) - }, 5000) - - this.responseHandlers.set(messageId, (response) => { - clearTimeout(timeout) - resolve(response) - }) - - ws.send(JSON.stringify({ - type, - from: this.nodeId, - to: nodeId, - data, - timestamp: Date.now(), - id: messageId - })) - }) - } else { - // Use HTTP fallback - const endpoint = this.peers.get(nodeId) - if (!endpoint) { - throw new Error(`Unknown node: ${nodeId}`) - } - - return this.sendViaHTTP(endpoint, type, data) - } - } - - /** - * Send via HTTP - */ - private async sendViaHTTP(endpoint: NodeEndpoint, type: string, data: any): Promise { - return new Promise((resolve, reject) => { - const message: NetworkMessage = { - type, - from: this.nodeId, - to: endpoint.nodeId, - data, - timestamp: Date.now(), - id: this.generateMessageId() - } - - const postData = JSON.stringify(message) - - const req = http.request({ - hostname: endpoint.host, - port: endpoint.httpPort, - path: '/message', - method: 'POST', - headers: { - 'Content-Type': 'application/json', - 'Content-Length': Buffer.byteLength(postData) - } - }, (res) => { - let responseData = '' - - res.on('data', (chunk) => { - responseData += chunk - }) - - res.on('end', () => { - try { - const response = JSON.parse(responseData) - if (response.success) { - resolve(response.data) - } else { - reject(new Error(response.error)) - } - } catch (err) { - reject(err) - } - }) - }) - - req.on('error', reject) - req.setTimeout(5000, () => { - req.destroy() - reject(new Error('HTTP timeout')) - }) - - req.write(postData) - req.end() - }) - } - - /** - * Broadcast to all peers - */ - async broadcast(type: string, data: any): Promise { - const promises = [] - - for (const nodeId of this.peers.keys()) { - promises.push( - this.sendToNode(nodeId, type, data).catch(() => { - // Ignore broadcast failures - }) - ) - } - - await Promise.all(promises) - } - - /** - * Register message handler - */ - onMessage(type: string, handler: (msg: NetworkMessage) => Promise): void { - this.messageHandlers.set(type, handler) - } - - /** - * Get connected peers - */ - getPeers(): NodeEndpoint[] { - return Array.from(this.peers.values()) - } - - /** - * Check if connected - */ - isConnected(nodeId: string): boolean { - const ws = this.connections.get(nodeId) - return ws !== undefined && ws.readyState === WebSocket.OPEN - } - - /** - * Generate node ID - */ - private generateNodeId(): string { - return `node-${Date.now()}-${Math.random().toString(36).substring(2, 11)}` - } - - /** - * Generate message ID - */ - private generateMessageId(): string { - return `${this.nodeId}-${Date.now()}-${Math.random().toString(36).substring(2, 9)}` - } - - /** - * Get node ID - */ - getNodeId(): string { - return this.nodeId - } -} - -/** - * Create network transport - */ -export function createNetworkTransport(config: NetworkConfig): NetworkTransport { - return new NetworkTransport(config) -} \ No newline at end of file diff --git a/src/distributed/operationalModes.ts b/src/distributed/operationalModes.ts deleted file mode 100644 index c3aa6c7d..00000000 --- a/src/distributed/operationalModes.ts +++ /dev/null @@ -1,220 +0,0 @@ -/** - * Operational Modes for Distributed Brainy - * Defines different modes with optimized caching strategies - */ - -import { - OperationalMode, - CacheStrategy, - InstanceRole -} from '../types/distributedTypes.js' - -/** - * Base operational mode - */ -export abstract class BaseOperationalMode implements OperationalMode { - abstract canRead: boolean - abstract canWrite: boolean - abstract canDelete: boolean - abstract cacheStrategy: CacheStrategy - - /** - * Validate operation is allowed in this mode - */ - validateOperation(operation: 'read' | 'write' | 'delete'): void { - switch (operation) { - case 'read': - if (!this.canRead) { - throw new Error('Read operations are not allowed in write-only mode') - } - break - case 'write': - if (!this.canWrite) { - throw new Error('Write operations are not allowed in read-only mode') - } - break - case 'delete': - if (!this.canDelete) { - throw new Error('Delete operations are not allowed in this mode') - } - break - } - } -} - -/** - * Read-only mode optimized for query performance - */ -export class ReaderMode extends BaseOperationalMode { - canRead = true - canWrite = false - canDelete = false - - cacheStrategy: CacheStrategy = { - hotCacheRatio: 0.8, // 80% of memory for read cache - prefetchAggressive: true, // Aggressively prefetch related vectors - ttl: 3600000, // 1 hour cache TTL - compressionEnabled: true, // Trade CPU for more cache capacity - writeBufferSize: 0, // No write buffer needed - batchWrites: false, // No writes - adaptive: true // Adapt to query patterns - } - - /** - * Get optimized cache configuration for readers - */ - getCacheConfig() { - return { - hotCacheMaxSize: 1000000, // Large hot cache - hotCacheEvictionThreshold: 0.9, // Keep cache full - warmCacheTTL: 3600000, // 1 hour warm cache - batchSize: 100, // Large batch reads - autoTune: true, // Auto-tune for read patterns - autoTuneInterval: 60000, // Tune every minute - readOnly: true // Enable read-only optimizations - } - } -} - -/** - * Write-only mode optimized for ingestion - */ -export class WriterMode extends BaseOperationalMode { - canRead = false - canWrite = true - canDelete = true - - cacheStrategy: CacheStrategy = { - hotCacheRatio: 0.2, // Only 20% for cache, rest for write buffer - prefetchAggressive: false, // No prefetching needed - ttl: 60000, // Short TTL (1 minute) - compressionEnabled: false, // Speed over memory efficiency - writeBufferSize: 10000, // Large write buffer for batching - batchWrites: true, // Enable write batching - adaptive: false // Fixed strategy for consistent writes - } - - /** - * Get optimized cache configuration for writers - */ - getCacheConfig() { - return { - hotCacheMaxSize: 100000, // Small hot cache - hotCacheEvictionThreshold: 0.5, // Aggressive eviction - warmCacheTTL: 60000, // 1 minute warm cache - batchSize: 1000, // Large batch writes - autoTune: false, // Fixed configuration - writeOnly: true // Enable write-only optimizations - } - } -} - -/** - * Hybrid mode that can both read and write - */ -export class HybridMode extends BaseOperationalMode { - canRead = true - canWrite = true - canDelete = true - - cacheStrategy: CacheStrategy = { - hotCacheRatio: 0.5, // Balanced cache/buffer allocation - prefetchAggressive: false, // Moderate prefetching - ttl: 600000, // 10 minute TTL - compressionEnabled: true, // Compress when beneficial - writeBufferSize: 5000, // Moderate write buffer - batchWrites: true, // Batch writes when possible - adaptive: true // Adapt to workload mix - } - - private readWriteRatio: number = 0.5 // Track read/write ratio - - /** - * Get balanced cache configuration - */ - getCacheConfig() { - return { - hotCacheMaxSize: 500000, // Medium cache size - hotCacheEvictionThreshold: 0.7, // Balanced eviction - warmCacheTTL: 600000, // 10 minute warm cache - batchSize: 500, // Medium batch size - autoTune: true, // Auto-tune based on workload - autoTuneInterval: 300000 // Tune every 5 minutes - } - } - - /** - * Update cache strategy based on workload - * @param readCount - Number of recent reads - * @param writeCount - Number of recent writes - */ - updateWorkloadBalance(readCount: number, writeCount: number): void { - const total = readCount + writeCount - if (total === 0) return - - this.readWriteRatio = readCount / total - - // Adjust cache strategy based on workload - if (this.readWriteRatio > 0.8) { - // Read-heavy workload - this.cacheStrategy.hotCacheRatio = 0.7 - this.cacheStrategy.prefetchAggressive = true - this.cacheStrategy.writeBufferSize = 2000 - } else if (this.readWriteRatio < 0.2) { - // Write-heavy workload - this.cacheStrategy.hotCacheRatio = 0.3 - this.cacheStrategy.prefetchAggressive = false - this.cacheStrategy.writeBufferSize = 8000 - } else { - // Balanced workload - this.cacheStrategy.hotCacheRatio = 0.5 - this.cacheStrategy.prefetchAggressive = false - this.cacheStrategy.writeBufferSize = 5000 - } - } -} - -/** - * Factory for creating operational modes - */ -export class OperationalModeFactory { - /** - * Create operational mode based on role - * @param role - The instance role - * @returns The appropriate operational mode - */ - static createMode(role: InstanceRole): BaseOperationalMode { - switch (role) { - case 'reader': - return new ReaderMode() - case 'writer': - return new WriterMode() - case 'hybrid': - return new HybridMode() - default: - // Default to reader for safety - return new ReaderMode() - } - } - - /** - * Create mode with custom cache strategy - * @param role - The instance role - * @param customStrategy - Custom cache strategy overrides - * @returns The operational mode with custom strategy - */ - static createModeWithStrategy( - role: InstanceRole, - customStrategy: Partial - ): BaseOperationalMode { - const mode = this.createMode(role) - - // Apply custom strategy overrides - mode.cacheStrategy = { - ...mode.cacheStrategy, - ...customStrategy - } - - return mode - } -} \ No newline at end of file diff --git a/src/distributed/queryPlanner.ts b/src/distributed/queryPlanner.ts deleted file mode 100644 index 6e9d7411..00000000 --- a/src/distributed/queryPlanner.ts +++ /dev/null @@ -1,425 +0,0 @@ -/** - * Distributed Query Planner for Brainy 3.0 - * - * Intelligently plans and executes distributed queries across shards - * Optimizes for data locality, parallelism, and network efficiency - */ - -import type { StorageAdapter } from '../coreTypes.js' -import type { DistributedCoordinator } from './coordinator.js' -import type { ShardManager } from './shardManager.js' -import type { HTTPTransport } from './httpTransport.js' -import type { TripleIntelligenceSystem } from '../triple/TripleIntelligenceSystem.js' - -export interface QueryPlan { - /** - * Shards that need to be queried - */ - shards: string[] - - /** - * Nodes responsible for each shard - */ - nodeAssignments: Map - - /** - * Whether query can be parallelized - */ - parallel: boolean - - /** - * Estimated cost (0-1000) - */ - cost: number - - /** - * Query strategy - */ - strategy: 'broadcast' | 'targeted' | 'scatter-gather' | 'local-only' -} - -export interface DistributedQueryResult { - results: any[] - totalCount: number - executionTime: number - nodeStats: Map -} - -export class DistributedQueryPlanner { - private nodeId: string - private coordinator: DistributedCoordinator - private shardManager: ShardManager - private transport: HTTPTransport - private tripleEngine?: TripleIntelligenceSystem - private storage: StorageAdapter - - constructor( - nodeId: string, - coordinator: DistributedCoordinator, - shardManager: ShardManager, - transport: HTTPTransport, - storage: StorageAdapter, - tripleEngine?: TripleIntelligenceSystem - ) { - this.nodeId = nodeId - this.coordinator = coordinator - this.shardManager = shardManager - this.transport = transport - this.storage = storage - this.tripleEngine = tripleEngine - } - - /** - * Plan a distributed query - */ - async planQuery(query: any): Promise { - // Determine query type and scope - const queryType = this.getQueryType(query) - const affectedShards = await this.determineAffectedShards(query) - - // Get current shard assignments - const assignments = new Map() - for (const shardId of affectedShards) { - const nodes = await this.shardManager.getNodesForShard(shardId) - assignments.set(shardId, nodes) - } - - // Determine strategy based on query characteristics - let strategy: QueryPlan['strategy'] = 'scatter-gather' - let cost = 0 - - if (affectedShards.length === 0) { - // Local only query - strategy = 'local-only' - cost = 1 - } else if (affectedShards.length === this.shardManager.getTotalShards()) { - // Full table scan - strategy = 'broadcast' - cost = 1000 - } else if (affectedShards.length <= 3) { - // Targeted query - strategy = 'targeted' - cost = affectedShards.length * 10 - } else { - // Scatter-gather for medium queries - strategy = 'scatter-gather' - cost = affectedShards.length * 50 - } - - // Add network cost - const remoteShards = affectedShards.filter(shardId => { - const nodes = assignments.get(shardId) || [] - return !nodes.includes(this.nodeId) - }) - cost += remoteShards.length * 20 - - return { - shards: affectedShards, - nodeAssignments: assignments, - parallel: affectedShards.length > 1, - cost, - strategy - } - } - - /** - * Execute a distributed query based on plan - */ - async executeQuery(query: any, plan: QueryPlan): Promise { - const startTime = Date.now() - const nodeStats = new Map() - - // Group shards by node for efficient batching - const nodeToShards = new Map() - for (const [shardId, nodes] of plan.nodeAssignments) { - // Pick the first available node (could be optimized) - const targetNode = nodes[0] - if (!nodeToShards.has(targetNode)) { - nodeToShards.set(targetNode, []) - } - nodeToShards.get(targetNode)!.push(shardId) - } - - // Execute queries in parallel - const promises: Promise[] = [] - - for (const [targetNode, shards] of nodeToShards) { - if (targetNode === this.nodeId) { - // Local execution - promises.push(this.executeLocalQuery(query, shards)) - } else { - // Remote execution - promises.push(this.executeRemoteQuery(targetNode, query, shards)) - } - } - - // Wait for all results - const results = await Promise.allSettled(promises) - - // Aggregate results - const allResults: any[] = [] - let totalCount = 0 - - let nodeIndex = 0 - for (const [targetNode, shards] of nodeToShards) { - const result = results[nodeIndex] - const nodeTime = Date.now() - startTime - - if (result.status === 'fulfilled') { - const nodeResult = result.value - allResults.push(...(nodeResult.results || [])) - totalCount += nodeResult.count || 0 - - nodeStats.set(targetNode, { - resultsReturned: nodeResult.count || 0, - executionTime: nodeTime, - shards - }) - } else { - nodeStats.set(targetNode, { - resultsReturned: 0, - executionTime: nodeTime, - errors: [result.reason?.message || 'Unknown error'], - shards - }) - } - - nodeIndex++ - } - - // Merge and rank results using Triple Intelligence if available - const mergedResults = await this.mergeResults(allResults, query) - - return { - results: mergedResults, - totalCount, - executionTime: Date.now() - startTime, - nodeStats - } - } - - /** - * Execute query on local shards - */ - private async executeLocalQuery(query: any, shards: string[]): Promise { - const results: any[] = [] - - for (const shardId of shards) { - // Get data from storage for this shard - const shardData = await this.getShardData(shardId, query) - results.push(...shardData) - } - - return { - results, - count: results.length - } - } - - /** - * Execute query on remote node - */ - private async executeRemoteQuery( - targetNode: string, - query: any, - shards: string[] - ): Promise { - try { - const response = await this.transport.call(targetNode, 'query', { - query, - shards - }) - - return response || { results: [], count: 0 } - } catch (error) { - console.error(`Failed to query node ${targetNode}:`, error) - throw error - } - } - - /** - * Get data from a specific shard - */ - private async getShardData(shardId: string, query: any): Promise { - // This would interact with the actual storage adapter - // For now, return empty array since storage adapters don't have direct shard access - // In a real implementation, this would use storage-specific methods - - try { - // Would need to implement shard-aware storage methods - // For now, return empty to allow compilation - return [] - } catch (error) { - console.error(`Failed to get shard ${shardId} data:`, error) - return [] - } - } - - /** - * Filter data based on query criteria - */ - private filterData(data: any, query: any): any[] { - if (!Array.isArray(data)) return [] - - // Apply query filters - let filtered = data - - if (query.filter) { - filtered = filtered.filter((item: any) => { - // Apply filter logic - return this.matchesFilter(item, query.filter) - }) - } - - if (query.limit) { - filtered = filtered.slice(0, query.limit) - } - - return filtered - } - - /** - * Check if item matches filter - */ - private matchesFilter(item: any, filter: any): boolean { - // Simple filter matching - for (const [key, value] of Object.entries(filter)) { - if (item[key] !== value) { - return false - } - } - return true - } - - /** - * Merge results from multiple nodes using Triple Intelligence - */ - private async mergeResults(results: any[], query: any): Promise { - if (!this.tripleEngine) { - // Simple merge without Triple Intelligence - return this.deduplicateResults(results) - } - - // Use Triple Intelligence for intelligent merging - // Merge results by combining scores and maintaining order - const mergedMap = new Map() - - for (const result of results) { - const id = result.id || result.entity?.id - if (!id) continue - - if (mergedMap.has(id)) { - // Merge duplicate results by averaging scores - const existing = mergedMap.get(id) - existing.score = (existing.score + (result.score || 0)) / 2 - // Preserve highest confidence metadata - if (result.confidence > existing.confidence) { - existing.metadata = { ...existing.metadata, ...result.metadata } - } - } else { - mergedMap.set(id, { ...result }) - } - } - - // Sort by score and return - return Array.from(mergedMap.values()) - .sort((a, b) => (b.score || 0) - (a.score || 0)) - } - - /** - * Simple deduplication of results - */ - private deduplicateResults(results: any[]): any[] { - const seen = new Set() - const deduplicated: any[] = [] - - for (const result of results) { - const key = this.getResultKey(result) - if (!seen.has(key)) { - seen.add(key) - deduplicated.push(result) - } - } - - return deduplicated - } - - /** - * Get unique key for result - */ - private getResultKey(result: any): string { - if (result.id) return result.id - if (result.uuid) return result.uuid - return JSON.stringify(result) - } - - /** - * Determine query type - */ - private getQueryType(query: any): string { - if (query.vector) return 'vector' - if (query.triple) return 'triple' - if (query.filter) return 'filter' - return 'scan' - } - - /** - * Determine which shards are affected by query - */ - private async determineAffectedShards(query: any): Promise { - const totalShards = this.shardManager.getTotalShards() - const affectedShards: string[] = [] - - // If query has specific entity/key, determine shard - if (query.entity || query.key) { - const key = query.entity || query.key - const assignment = this.shardManager.getShardForKey(key) - if (assignment) { - return [assignment.shardId] - } - } - - // If query has partition hint - if (query.partition) { - return [query.partition] - } - - // Otherwise, query all shards (broadcast) - for (let i = 0; i < totalShards; i++) { - affectedShards.push(`shard-${i}`) - } - - return affectedShards - } - - /** - * Optimize query plan based on statistics - */ - async optimizePlan(plan: QueryPlan): Promise { - // Get node health and latency stats - const nodeHealth = await this.coordinator.getHealth() - - // Re-assign shards to healthier nodes if needed - const optimizedAssignments = new Map() - - for (const [shardId, nodes] of plan.nodeAssignments) { - // Sort nodes by health score (simple heuristic) - const sortedNodes = nodes.sort((a, b) => { - // Higher health score = better node - // For now, use a simple approach - return 0 - }) - - optimizedAssignments.set(shardId, sortedNodes) - } - - return { - ...plan, - nodeAssignments: optimizedAssignments - } - } -} \ No newline at end of file diff --git a/src/distributed/readWriteSeparation.ts b/src/distributed/readWriteSeparation.ts deleted file mode 100644 index 222c66ab..00000000 --- a/src/distributed/readWriteSeparation.ts +++ /dev/null @@ -1,457 +0,0 @@ -/** - * Read/Write Separation for Distributed Scaling - * Implements primary-replica architecture for scalable reads - */ - -import { EventEmitter } from 'node:events' -import { DistributedCoordinator } from './coordinator.js' -import { ShardManager } from './shardManager.js' -import { CacheSync } from './cacheSync.js' - -export interface ReplicationConfig { - nodeId: string - role?: 'primary' | 'replica' | 'auto' - primaryUrl?: string - replicaUrls?: string[] - syncInterval?: number - readPreference?: 'primary' | 'replica' | 'nearest' - consistencyLevel?: 'eventual' | 'strong' | 'bounded' - maxStaleness?: number -} - -export interface WriteOperation { - id: string - type: 'add' | 'update' | 'delete' - data: any - timestamp: number - version: number -} - -export interface ReplicationLog { - operations: WriteOperation[] - lastSequence: number - primaryVersion: number -} - -/** - * Read/Write Separation Manager - */ -export class ReadWriteSeparation extends EventEmitter { - private nodeId: string - private role: 'primary' | 'replica' - private coordinator: DistributedCoordinator - private cacheSync: CacheSync - private replicationLog: ReplicationLog - private replicas: Map = new Map() - private primaryConnection?: PrimaryConnection - private config: ReplicationConfig - private syncTimer?: NodeJS.Timeout - private isRunning: boolean = false - - constructor( - config: ReplicationConfig, - coordinator: DistributedCoordinator, - _shardManager: ShardManager, - cacheSync: CacheSync - ) { - super() - - this.config = config - this.nodeId = config.nodeId - this.role = config.role === 'auto' ? this.determineRole() : (config.role || 'replica') - this.coordinator = coordinator - this.cacheSync = cacheSync - - this.replicationLog = { - operations: [], - lastSequence: 0, - primaryVersion: 0 - } - - // Setup connections based on role - this.setupConnections() - } - - /** - * Start read/write separation - */ - async start(): Promise { - if (this.isRunning) return - - this.isRunning = true - - // Start components - await this.coordinator.start() - - // Setup role-specific behavior - if (this.role === 'primary') { - this.startAsPrimary() - } else { - this.startAsReplica() - } - - this.emit('started', { nodeId: this.nodeId, role: this.role }) - } - - /** - * Stop read/write separation - */ - async stop(): Promise { - if (!this.isRunning) return - - this.isRunning = false - - if (this.syncTimer) { - clearInterval(this.syncTimer) - this.syncTimer = undefined - } - - await this.coordinator.stop() - - this.emit('stopped', { nodeId: this.nodeId }) - } - - /** - * Execute a write operation (primary only) - */ - async write(operation: Omit): Promise { - if (this.role !== 'primary') { - // Forward to primary - if (this.primaryConnection) { - return this.primaryConnection.forwardWrite(operation) - } - throw new Error('Cannot write: not connected to primary') - } - - // Generate operation metadata - const writeOp: WriteOperation = { - id: this.generateOperationId(), - ...operation, - timestamp: Date.now(), - version: ++this.replicationLog.primaryVersion - } - - // Add to replication log - this.replicationLog.operations.push(writeOp) - this.replicationLog.lastSequence++ - - // Propagate to replicas - this.propagateToReplicas(writeOp) - - // Update cache - if (operation.type !== 'delete') { - this.cacheSync.set(writeOp.id, operation.data) - } else { - this.cacheSync.delete(writeOp.id) - } - - this.emit('write', writeOp) - - return writeOp.id - } - - /** - * Execute a read operation - */ - async read(key: string, options?: { consistency?: 'eventual' | 'strong' }): Promise { - const consistency = options?.consistency || this.config.consistencyLevel || 'eventual' - - if (consistency === 'strong' && this.role === 'replica') { - // For strong consistency, read from primary - if (this.primaryConnection) { - return this.primaryConnection.read(key) - } - throw new Error('Cannot guarantee strong consistency: not connected to primary') - } - - // Check cache first - const cached = this.cacheSync.get(key) - if (cached !== undefined) { - return cached - } - - // Read from appropriate source based on preference - if (this.config.readPreference === 'primary' && this.primaryConnection) { - return this.primaryConnection.read(key) - } - - // Read locally (replica or primary) - return this.readLocal(key) - } - - /** - * Get replication lag (replica only) - */ - getReplicationLag(): number { - if (this.role === 'primary') return 0 - - if (this.primaryConnection) { - return Date.now() - this.primaryConnection.lastSync - } - - return -1 // Unknown - } - - /** - * Setup connections based on role - */ - private setupConnections(): void { - if (this.role === 'primary') { - // Setup replica connections - if (this.config.replicaUrls) { - for (const url of this.config.replicaUrls) { - const replica = new ReplicaConnection(url) - this.replicas.set(url, replica) - } - } - } else { - // Setup primary connection - if (this.config.primaryUrl) { - this.primaryConnection = new PrimaryConnection(this.config.primaryUrl) - } - } - } - - /** - * Start as primary node - */ - private startAsPrimary(): void { - // Start accepting writes - this.emit('roleEstablished', { role: 'primary' }) - - // Start replication timer - this.syncTimer = setInterval(() => { - this.syncReplicas() - }, this.config.syncInterval || 1000) - } - - /** - * Start as replica node - */ - private startAsReplica(): void { - // Start syncing from primary - this.emit('roleEstablished', { role: 'replica' }) - - // Start sync timer - this.syncTimer = setInterval(() => { - this.syncFromPrimary() - }, this.config.syncInterval || 1000) - } - - /** - * Sync replicas (primary only) - */ - private async syncReplicas(): Promise { - const batch = this.replicationLog.operations.slice(-100) // Last 100 ops - - for (const [url, replica] of this.replicas) { - try { - await replica.sync(batch, this.replicationLog.primaryVersion) - } catch (error) { - this.emit('replicaSyncError', { url, error }) - } - } - } - - /** - * Sync from primary (replica only) - */ - private async syncFromPrimary(): Promise { - if (!this.primaryConnection) return - - try { - const updates = await this.primaryConnection.getUpdates( - this.replicationLog.lastSequence - ) - - // Apply updates - for (const op of updates) { - await this.applyOperation(op) - } - - this.emit('synced', { operations: updates.length }) - } catch (error) { - this.emit('primarySyncError', { error }) - } - } - - /** - * Apply a replicated operation - */ - private async applyOperation(op: WriteOperation): Promise { - // Update local state - this.replicationLog.operations.push(op) - this.replicationLog.lastSequence = Math.max( - this.replicationLog.lastSequence, - op.version - ) - - // Update cache - switch (op.type) { - case 'add': - case 'update': - this.cacheSync.set(op.id, op.data) - break - case 'delete': - this.cacheSync.delete(op.id) - break - } - - this.emit('operationApplied', op) - } - - /** - * Propagate operation to replicas - */ - private propagateToReplicas(op: WriteOperation): void { - for (const replica of this.replicas.values()) { - replica.sendOperation(op).catch(error => { - this.emit('replicationError', { replica, error }) - }) - } - } - - /** - * Determine role automatically - */ - private determineRole(): 'primary' | 'replica' { - // Use coordinator's leader election - return this.coordinator.isLeader() ? 'primary' : 'replica' - } - - /** - * Read from local storage - */ - private async readLocal(key: string): Promise { - // This would connect to actual storage - // For now, return from cache or undefined - return this.cacheSync.get(key) - } - - /** - * Generate unique operation ID - */ - private generateOperationId(): string { - return `${this.nodeId}-${Date.now()}-${Math.random().toString(36).substring(2, 11)}` - } - - /** - * Get replication statistics - */ - getStats(): { - role: string - replicas: number - replicationLag: number - operationsInLog: number - primaryVersion: number - } { - return { - role: this.role, - replicas: this.replicas.size, - replicationLag: this.getReplicationLag(), - operationsInLog: this.replicationLog.operations.length, - primaryVersion: this.replicationLog.primaryVersion - } - } - - /** - * Check if node can accept writes - */ - canWrite(): boolean { - return this.role === 'primary' - } - - /** - * Check if node can serve reads - */ - canRead(): boolean { - if (this.config.consistencyLevel === 'strong') { - return this.role === 'primary' || this.primaryConnection !== undefined - } - return true - } - - /** - * Set whether this node is primary (for leader election integration) - */ - setPrimary(isPrimary: boolean): void { - const newRole = isPrimary ? 'primary' : 'replica' - if (this.role !== newRole) { - this.role = newRole - this.emit('roleChange', { oldRole: this.role, newRole }) - - if (isPrimary) { - // Became primary - stop syncing from old primary - this.primaryConnection = undefined - } else { - // Became replica - connect to new primary if URL is known - if (this.config.primaryUrl) { - this.primaryConnection = new PrimaryConnection(this.config.primaryUrl) - } - } - } - } -} - -/** - * Connection to a replica (used by primary) - */ -class ReplicaConnection { - constructor(private url: string) { - // Store URL for connection - void this.url - } - - async sync(_operations: WriteOperation[], _version: number): Promise { - // In real implementation, this would use HTTP/gRPC to this.url - // For now, simulate network call - await new Promise(resolve => setTimeout(resolve, 10)) - } - - async sendOperation(_op: WriteOperation): Promise { - // Send single operation to replica at this.url - await new Promise(resolve => setTimeout(resolve, 5)) - } -} - -/** - * Connection to primary (used by replicas) - */ -class PrimaryConnection { - lastSync: number = Date.now() - - constructor(private url: string) { - // Store URL for connection - void this.url - } - - async getUpdates(_fromSequence: number): Promise { - // In real implementation, fetch from primary at this.url - this.lastSync = Date.now() - return [] - } - - async forwardWrite(_operation: any): Promise { - // Forward write to primary at this.url - await new Promise(resolve => setTimeout(resolve, 20)) - return `forwarded-${Date.now()}` - } - - async read(_key: string): Promise { - // Read from primary at this.url for strong consistency - await new Promise(resolve => setTimeout(resolve, 10)) - return undefined - } -} - -/** - * Create read/write separation manager - */ -export function createReadWriteSeparation( - config: ReplicationConfig, - coordinator: DistributedCoordinator, - shardManager: ShardManager, - cacheSync: CacheSync -): ReadWriteSeparation { - return new ReadWriteSeparation(config, coordinator, shardManager, cacheSync) -} \ No newline at end of file diff --git a/src/distributed/shardManager.ts b/src/distributed/shardManager.ts deleted file mode 100644 index 0d916918..00000000 --- a/src/distributed/shardManager.ts +++ /dev/null @@ -1,453 +0,0 @@ -/** - * Shard Manager for Horizontal Scaling - * Implements consistent hashing for data distribution across shards - */ - -import { createHash } from 'node:crypto' -import { EventEmitter } from 'node:events' - -export interface ShardConfig { - shardCount?: number - replicationFactor?: number - virtualNodes?: number - autoRebalance?: boolean -} - -export interface Shard { - id: string - nodeId: string - virtualNodes: string[] - itemCount: number - sizeBytes: number - status: 'active' | 'rebalancing' | 'offline' -} - -export interface ShardAssignment { - shardId: string - nodeId: string - replicas: string[] -} - -/** - * Consistent Hash Ring for shard distribution - */ -class ConsistentHashRing { - private ring: Map = new Map() - private virtualNodes: number - private sortedKeys: number[] = [] - - constructor(virtualNodes: number = 150) { - this.virtualNodes = virtualNodes - } - - /** - * Add a node to the hash ring - */ - addNode(nodeId: string): void { - for (let i = 0; i < this.virtualNodes; i++) { - const virtualNodeId = `${nodeId}:${i}` - const hash = this.hash(virtualNodeId) - this.ring.set(hash, nodeId) - } - this.updateSortedKeys() - } - - /** - * Remove a node from the hash ring - */ - removeNode(nodeId: string): void { - const keysToRemove: number[] = [] - - for (const [hash, node] of this.ring) { - if (node === nodeId) { - keysToRemove.push(hash) - } - } - - for (const key of keysToRemove) { - this.ring.delete(key) - } - - this.updateSortedKeys() - } - - /** - * Get the node responsible for a given key - */ - getNode(key: string): string | null { - if (this.ring.size === 0) return null - - const hash = this.hash(key) - - // Find the first node with hash >= key hash - for (const nodeHash of this.sortedKeys) { - if (nodeHash >= hash) { - return this.ring.get(nodeHash) || null - } - } - - // Wrap around to the first node - return this.ring.get(this.sortedKeys[0]) || null - } - - /** - * Get N nodes for replication - */ - getNodes(key: string, count: number): string[] { - if (this.ring.size === 0) return [] - - const nodes = new Set() - const hash = this.hash(key) - - // Start from the primary node position - let startIdx = 0 - for (let i = 0; i < this.sortedKeys.length; i++) { - if (this.sortedKeys[i] >= hash) { - startIdx = i - break - } - } - - // Collect unique nodes - let idx = startIdx - while (nodes.size < count && nodes.size < this.getUniqueNodeCount()) { - const nodeHash = this.sortedKeys[idx % this.sortedKeys.length] - const node = this.ring.get(nodeHash) - if (node) { - nodes.add(node) - } - idx++ - } - - return Array.from(nodes) - } - - /** - * Get unique node count - */ - private getUniqueNodeCount(): number { - return new Set(this.ring.values()).size - } - - /** - * Update sorted keys for efficient lookup - */ - private updateSortedKeys(): void { - this.sortedKeys = Array.from(this.ring.keys()).sort((a, b) => a - b) - } - - /** - * Hash function for consistent hashing - */ - private hash(key: string): number { - const hash = createHash('md5').update(key).digest() - return hash.readUInt32BE(0) - } - - /** - * Get all nodes in the ring - */ - getAllNodes(): string[] { - return Array.from(new Set(this.ring.values())) - } -} - -/** - * Shard Manager for distributing data across multiple nodes - */ -export class ShardManager extends EventEmitter { - private hashRing: ConsistentHashRing - private shards: Map = new Map() - private nodeToShards: Map> = new Map() - private shardCount: number - private replicationFactor: number - private autoRebalance: boolean - - constructor(config: ShardConfig = {}) { - super() - - this.shardCount = config.shardCount || 64 - this.replicationFactor = config.replicationFactor || 3 - this.autoRebalance = config.autoRebalance ?? true - - this.hashRing = new ConsistentHashRing(config.virtualNodes || 150) - - // Initialize shards - this.initializeShards() - } - - /** - * Initialize shard configuration - */ - private initializeShards(): void { - for (let i = 0; i < this.shardCount; i++) { - const shardId = `shard-${i.toString().padStart(3, '0')}` - this.shards.set(shardId, { - id: shardId, - nodeId: '', - virtualNodes: [], - itemCount: 0, - sizeBytes: 0, - status: 'offline' - }) - } - } - - /** - * Add a node to the cluster - */ - addNode(nodeId: string): void { - this.hashRing.addNode(nodeId) - this.nodeToShards.set(nodeId, new Set()) - - // Assign shards to the new node - this.rebalanceShards() - - this.emit('nodeAdded', { nodeId }) - } - - /** - * Remove a node from the cluster - */ - removeNode(nodeId: string): void { - const affectedShards = this.nodeToShards.get(nodeId) || new Set() - - this.hashRing.removeNode(nodeId) - this.nodeToShards.delete(nodeId) - - // Reassign affected shards - for (const shardId of affectedShards) { - const shard = this.shards.get(shardId) - if (shard) { - shard.status = 'rebalancing' - } - } - - this.rebalanceShards() - - this.emit('nodeRemoved', { nodeId, affectedShards: Array.from(affectedShards) }) - } - - /** - * Get shard assignment for a key - */ - getShardForKey(key: string): ShardAssignment | null { - const shardId = this.getShardId(key) - const nodes = this.hashRing.getNodes(shardId, this.replicationFactor) - - if (nodes.length === 0) return null - - return { - shardId, - nodeId: nodes[0], - replicas: nodes.slice(1) - } - } - - /** - * Get nodes responsible for a shard - */ - getNodesForShard(shardId: string): string[] { - const shard = this.shards.get(shardId) - if (!shard) return [] - - // Return primary node and replicas - const nodes = this.hashRing.getNodes(shardId, this.replicationFactor) - return nodes - } - - /** - * Get total number of shards - */ - getTotalShards(): number { - return this.shardCount - } - - /** - * Update shard assignment to a new node - */ - updateShardAssignment(shardId: string, newNodeId: string): void { - const shard = this.shards.get(shardId) - if (!shard) { - throw new Error(`Shard ${shardId} not found`) - } - - // Remove from old node - if (shard.nodeId) { - const oldNodeShards = this.nodeToShards.get(shard.nodeId) - if (oldNodeShards) { - oldNodeShards.delete(shardId) - } - } - - // Add to new node - shard.nodeId = newNodeId - const newNodeShards = this.nodeToShards.get(newNodeId) - if (newNodeShards) { - newNodeShards.add(shardId) - } else { - this.nodeToShards.set(newNodeId, new Set([shardId])) - } - - this.emit('shardReassigned', { shardId, newNodeId }) - } - - /** - * Get shard ID for a key - */ - private getShardId(key: string): string { - const hash = createHash('md5').update(key).digest() - const shardIndex = hash.readUInt16BE(0) % this.shardCount - return `shard-${shardIndex.toString().padStart(3, '0')}` - } - - /** - * Rebalance shards across nodes - */ - private rebalanceShards(): void { - if (!this.autoRebalance) return - - const nodes = this.hashRing.getAllNodes() - if (nodes.length === 0) return - - // Clear current assignments - for (const nodeSet of this.nodeToShards.values()) { - nodeSet.clear() - } - - // Reassign each shard - for (const [shardId, shard] of this.shards) { - const assignedNodes = this.hashRing.getNodes(shardId, 1) - if (assignedNodes.length > 0) { - shard.nodeId = assignedNodes[0] - shard.status = 'active' - - const nodeShards = this.nodeToShards.get(assignedNodes[0]) - if (nodeShards) { - nodeShards.add(shardId) - } - } else { - shard.status = 'offline' - } - } - - this.emit('rebalanced', { nodes, shardCount: this.shardCount }) - } - - /** - * Get shard assignment for all shards - */ - getShardAssignments(): ShardAssignment[] { - const assignments: ShardAssignment[] = [] - - for (const [shardId, shard] of this.shards) { - if (shard.nodeId) { - assignments.push({ - shardId, - nodeId: shard.nodeId, - replicas: this.hashRing.getNodes(shardId, this.replicationFactor).slice(1) - }) - } - } - - return assignments - } - - /** - * Get shard statistics - */ - getShardStats(): { - totalShards: number - activeShards: number - rebalancingShards: number - offlineShards: number - averageItemsPerShard: number - } { - let activeShards = 0 - let rebalancingShards = 0 - let offlineShards = 0 - let totalItems = 0 - - for (const shard of this.shards.values()) { - switch (shard.status) { - case 'active': - activeShards++ - break - case 'rebalancing': - rebalancingShards++ - break - case 'offline': - offlineShards++ - break - } - totalItems += shard.itemCount - } - - return { - totalShards: this.shardCount, - activeShards, - rebalancingShards, - offlineShards, - averageItemsPerShard: this.shardCount > 0 ? Math.floor(totalItems / this.shardCount) : 0 - } - } - - /** - * Update shard metrics - */ - updateShardMetrics(shardId: string, itemCount: number, sizeBytes: number): void { - const shard = this.shards.get(shardId) - if (shard) { - shard.itemCount = itemCount - shard.sizeBytes = sizeBytes - } - } - - /** - * Get replication nodes for a shard - */ - getReplicationNodes(shardId: string): string[] { - return this.hashRing.getNodes(shardId, this.replicationFactor) - } - - /** - * Check if rebalancing is needed - */ - needsRebalancing(): boolean { - const stats = this.getShardStats() - return stats.offlineShards > 0 || stats.rebalancingShards > 0 - } - - /** - * Get cluster health - */ - getHealth(): { - healthy: boolean - nodes: number - shards: { - total: number - active: number - inactive: number - } - } { - const nodes = this.hashRing.getAllNodes() - const stats = this.getShardStats() - - return { - healthy: stats.activeShards >= this.shardCount * 0.9, // 90% shards active - nodes: nodes.length, - shards: { - total: this.shardCount, - active: stats.activeShards, - inactive: stats.offlineShards + stats.rebalancingShards - } - } - } -} - -/** - * Create a shard manager instance - */ -export function createShardManager(config?: ShardConfig): ShardManager { - return new ShardManager(config) -} \ No newline at end of file diff --git a/src/distributed/shardMigration.ts b/src/distributed/shardMigration.ts deleted file mode 100644 index 023d70e4..00000000 --- a/src/distributed/shardMigration.ts +++ /dev/null @@ -1,398 +0,0 @@ -/** - * Shard Migration System for Brainy 3.0 - * - * Handles zero-downtime migration of data between nodes - * Uses streaming for efficient transfer of large datasets - */ - -import { EventEmitter } from 'node:events' -import type { StorageAdapter } from '../coreTypes.js' -import type { ShardManager } from './shardManager.js' -import type { HTTPTransport } from './httpTransport.js' -import type { DistributedCoordinator } from './coordinator.js' - -export interface MigrationTask { - id: string - shardId: string - sourceNode: string - targetNode: string - status: 'pending' | 'transferring' | 'validating' | 'switching' | 'completed' | 'failed' - progress: number // 0-100 - itemsTransferred: number - totalItems: number - startTime: number - endTime?: number - error?: string -} - -export interface MigrationOptions { - batchSize?: number - validateData?: boolean - maxRetries?: number - timeout?: number -} - -export class ShardMigrationManager extends EventEmitter { - private storage: StorageAdapter - private shardManager: ShardManager - private transport: HTTPTransport - private coordinator: DistributedCoordinator - private nodeId: string - - private activeMigrations = new Map() - private migrationQueue: MigrationTask[] = [] - private maxConcurrentMigrations = 2 - - constructor( - nodeId: string, - storage: StorageAdapter, - shardManager: ShardManager, - transport: HTTPTransport, - coordinator: DistributedCoordinator - ) { - super() - this.nodeId = nodeId - this.storage = storage - this.shardManager = shardManager - this.transport = transport - this.coordinator = coordinator - } - - /** - * Initiate migration of a shard to a new node - */ - async migrateShard( - shardId: string, - targetNode: string, - options: MigrationOptions = {} - ): Promise { - const task: MigrationTask = { - id: `migration-${Date.now()}-${Math.random().toString(36).substring(7)}`, - shardId, - sourceNode: this.nodeId, - targetNode, - status: 'pending', - progress: 0, - itemsTransferred: 0, - totalItems: 0, - startTime: Date.now() - } - - // Add to queue - this.migrationQueue.push(task) - this.processMigrationQueue() - - return task - } - - /** - * Process migration queue - */ - private async processMigrationQueue(): Promise { - while (this.migrationQueue.length > 0 && - this.activeMigrations.size < this.maxConcurrentMigrations) { - const task = this.migrationQueue.shift()! - this.activeMigrations.set(task.id, task) - - // Execute migration in background - this.executeMigration(task).catch(error => { - console.error(`Migration ${task.id} failed:`, error) - task.status = 'failed' - task.error = error.message - this.emit('migrationFailed', task) - }) - } - } - - /** - * Execute a single migration task - */ - private async executeMigration(task: MigrationTask): Promise { - try { - this.emit('migrationStarted', task) - - // Phase 1: Start transferring data - task.status = 'transferring' - await this.transferData(task) - - // Phase 2: Validate transferred data - task.status = 'validating' - await this.validateData(task) - - // Phase 3: Switch ownership atomically - task.status = 'switching' - await this.switchOwnership(task) - - // Phase 4: Cleanup source - task.status = 'completed' - task.endTime = Date.now() - task.progress = 100 - - this.activeMigrations.delete(task.id) - this.emit('migrationCompleted', task) - - // Process next in queue - this.processMigrationQueue() - - } catch (error) { - task.status = 'failed' - task.error = (error as Error).message - this.activeMigrations.delete(task.id) - throw error - } - } - - /** - * Transfer data from source to target node - */ - private async transferData(task: MigrationTask): Promise { - const batchSize = 1000 - let offset = 0 - - // Get total count - const totalItems = await this.getShardItemCount(task.shardId) - task.totalItems = totalItems - - while (offset < totalItems) { - // Get batch of items - const items = await this.getShardItems(task.shardId, offset, batchSize) - - if (items.length === 0) break - - // Send batch to target node - await this.transport.call(task.targetNode, 'receiveMigrationBatch', { - migrationId: task.id, - shardId: task.shardId, - items, - offset, - total: totalItems - }) - - offset += items.length - task.itemsTransferred = offset - task.progress = Math.floor((offset / totalItems) * 80) // 80% for transfer - - this.emit('migrationProgress', task) - } - } - - /** - * Get items from a shard - */ - private async getShardItems( - shardId: string, - offset: number, - limit: number - ): Promise { - // Get all noun IDs for this shard - const nounKey = `shard:${shardId}:nouns` - const verbKey = `shard:${shardId}:verbs` - - const items: any[] = [] - - try { - // Get nouns - const nouns = await this.storage.getNounsByNounType('*') - const shardNouns = nouns.filter(n => { - const assignment = this.shardManager.getShardForKey(n.id) - return assignment?.shardId === shardId - }).slice(offset, offset + limit) - - items.push(...shardNouns.map(n => ({ type: 'noun', data: n }))) - - // Get verbs if we have room - if (items.length < limit) { - const verbs = await this.storage.getVerbsByType('*') - const shardVerbs = verbs.filter(v => { - const assignment = this.shardManager.getShardForKey(v.id) - return assignment?.shardId === shardId - }).slice(0, limit - items.length) - - items.push(...shardVerbs.map(v => ({ type: 'verb', data: v }))) - } - - } catch (error) { - console.error(`Failed to get shard items for ${shardId}:`, error) - } - - return items - } - - /** - * Get count of items in a shard - */ - private async getShardItemCount(shardId: string): Promise { - // For now, estimate based on total items / shard count - // In production, maintain accurate per-shard counts - const status = await this.storage.getStorageStatus() - const totalShards = this.shardManager.getTotalShards() - return Math.ceil(status.used / totalShards) - } - - /** - * Validate transferred data - */ - private async validateData(task: MigrationTask): Promise { - // Request validation from target node - const response = await this.transport.call(task.targetNode, 'validateMigration', { - migrationId: task.id, - shardId: task.shardId, - expectedCount: task.totalItems - }) - - if (!response.valid) { - throw new Error(`Validation failed: ${response.error}`) - } - - task.progress = 90 // 90% after validation - this.emit('migrationProgress', task) - } - - /** - * Switch shard ownership atomically - */ - private async switchOwnership(task: MigrationTask): Promise { - // Coordinate with all nodes to update shard assignment - await this.coordinator.proposeMigration({ - shardId: task.shardId, - fromNode: task.sourceNode, - toNode: task.targetNode, - migrationId: task.id - }) - - // Wait for consensus - await this.waitForConsensus(task.id) - - // Update local shard manager - this.shardManager.updateShardAssignment(task.shardId, task.targetNode) - - task.progress = 95 // 95% after ownership switch - this.emit('migrationProgress', task) - - // Cleanup local data - await this.cleanupShardData(task.shardId) - } - - /** - * Wait for consensus on migration - */ - private async waitForConsensus(migrationId: string): Promise { - const maxWait = 30000 // 30 seconds - const startTime = Date.now() - - while (Date.now() - startTime < maxWait) { - const status = await this.coordinator.getMigrationStatus(migrationId) - - if (status === 'committed') { - return - } else if (status === 'rejected') { - throw new Error('Migration rejected by cluster') - } - - // Wait a bit before checking again - await new Promise(resolve => setTimeout(resolve, 1000)) - } - - throw new Error('Consensus timeout') - } - - /** - * Cleanup local shard data after migration - */ - private async cleanupShardData(shardId: string): Promise { - // Mark shard data for deletion - // Don't delete immediately in case of rollback - const cleanupKey = `cleanup:${shardId}:${Date.now()}` - await this.storage.saveMetadata(cleanupKey, { - noun: 'Document', - shardId, - scheduledFor: Date.now() + 3600000 // Delete after 1 hour - }) - } - - /** - * Handle incoming migration batch (when we're the target) - */ - async receiveMigrationBatch(data: { - migrationId: string - shardId: string - items: any[] - offset: number - total: number - }): Promise { - // Store items - for (const item of data.items) { - if (item.type === 'noun') { - await this.storage.saveNoun(item.data) - } else if (item.type === 'verb') { - await this.storage.saveVerb(item.data) - } - } - - // Track progress - const progress = { - noun: 'Document', - migrationId: data.migrationId, - shardId: data.shardId, - received: data.offset + data.items.length, - total: data.total - } - - await this.storage.saveMetadata(`migration:${data.migrationId}:progress`, progress) - } - - /** - * Validate received migration data - */ - async validateMigration(data: { - migrationId: string - shardId: string - expectedCount: number - }): Promise<{ valid: boolean; error?: string }> { - // Check if we received all expected items - const progressKey = `migration:${data.migrationId}:progress` - const progress = await this.storage.getMetadata(progressKey) - - if (!progress) { - return { valid: false, error: 'No migration progress found' } - } - - if (progress.received !== data.expectedCount) { - return { - valid: false, - error: `Expected ${data.expectedCount} items, received ${progress.received}` - } - } - - // Verify data integrity (could add checksums) - return { valid: true } - } - - /** - * Get status of all active migrations - */ - getActiveMigrations(): MigrationTask[] { - return Array.from(this.activeMigrations.values()) - } - - /** - * Cancel a migration - */ - async cancelMigration(migrationId: string): Promise { - const task = this.activeMigrations.get(migrationId) - if (!task) { - throw new Error(`Migration ${migrationId} not found`) - } - - task.status = 'failed' - task.error = 'Cancelled by user' - this.activeMigrations.delete(migrationId) - - // Notify target node - await this.transport.call(task.targetNode, 'cancelMigration', { - migrationId - }) - - this.emit('migrationCancelled', task) - } -} \ No newline at end of file diff --git a/src/distributed/storageDiscovery.ts b/src/distributed/storageDiscovery.ts deleted file mode 100644 index 07329432..00000000 --- a/src/distributed/storageDiscovery.ts +++ /dev/null @@ -1,687 +0,0 @@ -/** - * Storage-based Discovery for Zero-Config Distributed Brainy - * Uses shared storage (S3/GCS/R2) as coordination point - * REAL PRODUCTION CODE - No mocks, no stubs! - */ - -import { EventEmitter } from 'node:events' -import * as os from 'node:os' -import { NounMetadata, StorageAdapter } from '../coreTypes.js' - -export interface NodeInfo { - id: string - endpoint: string - hostname: string - started: number - lastSeen: number - role: 'primary' | 'replica' | 'candidate' - shards: string[] - capacity: { - cpu: number - memory: number - storage: number - } - stats: { - nouns: number - verbs: number - queries: number - latency: number - } -} - -export interface ClusterConfig { - version: number - created: number - updated: number - leader: string | null - nodes: Record - shards: { - count: number - assignments: Record // shardId -> [primaryNode, ...replicas] - } - settings: { - replicationFactor: number - shardCount: number - autoRebalance: boolean - minNodes: number - maxNodesPerShard: number - } -} - -export class StorageDiscovery extends EventEmitter { - private nodeId: string - private storage: StorageAdapter - private nodeInfo: NodeInfo - private clusterConfig: ClusterConfig | null = null - private heartbeatInterval: NodeJS.Timeout | null = null - private discoveryInterval: NodeJS.Timeout | null = null - private endpoint: string = '' - private isRunning: boolean = false - private readonly HEARTBEAT_INTERVAL = 5000 // 5 seconds - private readonly DISCOVERY_INTERVAL = 2000 // 2 seconds - private readonly NODE_TIMEOUT = 30000 // 30 seconds until node considered dead - private readonly CLUSTER_PATH = '_cluster' - - constructor(storage: StorageAdapter, nodeId?: string) { - super() - this.storage = storage - this.nodeId = nodeId || this.generateNodeId() - - // Initialize node info with REAL system data - this.nodeInfo = { - id: this.nodeId, - endpoint: '', // Will be set when HTTP server starts - hostname: os.hostname(), - started: Date.now(), - lastSeen: Date.now(), - role: 'candidate', - shards: [], - capacity: { - cpu: os.cpus().length, - memory: Math.floor(os.totalmem() / 1024 / 1024), // MB - storage: 0 // Will be updated based on actual usage - }, - stats: { - nouns: 0, - verbs: 0, - queries: 0, - latency: 0 - } - } - } - - /** - * Start discovery and registration - */ - async start(httpPort: number): Promise { - if (this.isRunning) return this.clusterConfig! - - this.isRunning = true - - // Set our endpoint - this.endpoint = await this.detectEndpoint(httpPort) - this.nodeInfo.endpoint = this.endpoint - - // Try to load existing cluster config - this.clusterConfig = await this.loadClusterConfig() - - if (!this.clusterConfig) { - // We're the first node - initialize cluster - await this.initializeCluster() - } else { - // Join existing cluster - await this.joinCluster() - } - - // Start heartbeat to keep our node alive - this.startHeartbeat() - - // Start discovery to find other nodes - this.startDiscovery() - - this.emit('started', this.nodeInfo) - - return this.clusterConfig! - } - - /** - * Stop discovery and unregister - */ - async stop(): Promise { - if (!this.isRunning) return - - this.isRunning = false - - // Stop intervals - if (this.heartbeatInterval) { - clearInterval(this.heartbeatInterval) - this.heartbeatInterval = null - } - - if (this.discoveryInterval) { - clearInterval(this.discoveryInterval) - this.discoveryInterval = null - } - - // Remove ourselves from cluster - await this.leaveCluster() - - this.emit('stopped') - } - - /** - * Initialize a new cluster (we're the first node) - */ - private async initializeCluster(): Promise { - console.log(`[${this.nodeId}] Initializing new cluster as first node`) - - this.nodeInfo.role = 'primary' - - this.clusterConfig = { - version: 1, - created: Date.now(), - updated: Date.now(), - leader: this.nodeId, - nodes: { - [this.nodeId]: this.nodeInfo - }, - shards: { - count: 64, // Default shard count - assignments: {} - }, - settings: { - replicationFactor: 3, - shardCount: 64, - autoRebalance: true, - minNodes: 1, - maxNodesPerShard: 5 - } - } - - // Assign all shards to ourselves initially - for (let i = 0; i < this.clusterConfig.shards.count; i++) { - const shardId = `shard-${i.toString().padStart(3, '0')}` - this.clusterConfig.shards.assignments[shardId] = [this.nodeId] - this.nodeInfo.shards.push(shardId) - } - - // Save cluster config - await this.saveClusterConfig() - - // Register ourselves - await this.registerNode() - - this.emit('clusterInitialized', this.clusterConfig) - } - - /** - * Join an existing cluster - */ - private async joinCluster(): Promise { - console.log(`[${this.nodeId}] Joining existing cluster`) - - if (!this.clusterConfig) throw new Error('No cluster config') - - // Add ourselves to the cluster - this.clusterConfig.nodes[this.nodeId] = this.nodeInfo - - // Determine our role based on cluster state - const nodeCount = Object.keys(this.clusterConfig.nodes).length - - if (!this.clusterConfig.leader || !this.clusterConfig.nodes[this.clusterConfig.leader]) { - // No leader or leader is gone - trigger election - await this.triggerLeaderElection() - } else { - // Become replica - this.nodeInfo.role = 'replica' - } - - // Register ourselves - await this.registerNode() - - // Request shard assignment if auto-rebalance is enabled - if (this.clusterConfig.settings.autoRebalance) { - await this.requestShardAssignment() - } - - this.emit('clusterJoined', this.clusterConfig) - } - - /** - * Leave cluster cleanly - */ - private async leaveCluster(): Promise { - if (!this.clusterConfig) return - - console.log(`[${this.nodeId}] Leaving cluster`) - - // Remove ourselves from node registry - try { - // Mark as deleted rather than actually deleting - const deadNode = { noun: 'Document', ...this.nodeInfo, lastSeen: 0, status: 'inactive' as const } - await this.storage.saveMetadata(`${this.CLUSTER_PATH}/nodes/${this.nodeId}.json`, deadNode) - } catch (err) { - // Ignore errors during shutdown - } - - // If we're the leader, trigger new election - if (this.clusterConfig.leader === this.nodeId) { - this.clusterConfig.leader = null - await this.saveClusterConfig() - } - - this.emit('clusterLeft') - } - - /** - * Register node in storage - */ - private async registerNode(): Promise { - const path = `${this.CLUSTER_PATH}/nodes/${this.nodeId}.json` - await this.storage.saveMetadata(path, { noun: 'Document', ...this.nodeInfo }) - - // Also update registry - await this.updateNodeRegistry(this.nodeId) - } - - /** - * Heartbeat to keep node alive - */ - private startHeartbeat(): void { - this.heartbeatInterval = setInterval(async () => { - try { - this.nodeInfo.lastSeen = Date.now() - await this.registerNode() - - // Also update cluster config if we're the leader - if (this.clusterConfig && this.clusterConfig.leader === this.nodeId) { - await this.saveClusterConfig() - } - } catch (err) { - console.error(`[${this.nodeId}] Heartbeat failed:`, err) - } - }, this.HEARTBEAT_INTERVAL) - } - - /** - * Discover other nodes and monitor health - */ - private startDiscovery(): void { - this.discoveryInterval = setInterval(async () => { - try { - await this.discoverNodes() - await this.checkNodeHealth() - - // Check if we need to rebalance - if (this.shouldRebalance()) { - await this.triggerRebalance() - } - } catch (err) { - console.error(`[${this.nodeId}] Discovery failed:`, err) - } - }, this.DISCOVERY_INTERVAL) - } - - /** - * Discover nodes from storage - */ - private async discoverNodes(): Promise { - try { - // Since we can't list arbitrary paths, we'll use a registry approach - // Each node registers in a central registry file - const registry = await this.loadNodeRegistry() - - const now = Date.now() - let updated = false - - for (const nodeId of registry) { - if (nodeId === this.nodeId) continue - - try { - const nodeInfoData = await this.storage.getMetadata( - `${this.CLUSTER_PATH}/nodes/${nodeId}.json` - ) - const nodeInfo = nodeInfoData as unknown as NodeInfo - - // Check if node is alive - if (now - nodeInfo.lastSeen < this.NODE_TIMEOUT) { - if (!this.clusterConfig!.nodes[nodeId]) { - // New node discovered! - console.log(`[${this.nodeId}] Discovered new node: ${nodeId}`) - this.clusterConfig!.nodes[nodeId] = nodeInfo - updated = true - this.emit('nodeDiscovered', nodeInfo) - } else { - // Update existing node info - this.clusterConfig!.nodes[nodeId] = nodeInfo - } - } - } catch (err) { - // Node file might be corrupted or deleted - console.warn(`[${this.nodeId}] Failed to read node ${nodeId}:`, err) - } - } - - if (updated) { - this.clusterConfig!.version++ - this.clusterConfig!.updated = Date.now() - } - } catch (err) { - // Storage might be unavailable - console.error(`[${this.nodeId}] Failed to discover nodes:`, err) - } - } - - /** - * Load node registry from storage - */ - private async loadNodeRegistry(): Promise { - try { - // The registry document is written by updateNodeRegistry() below with a - // `nodes` string-ID array on top of the base NounMetadata shape. - const registry = await this.storage.getMetadata( - `${this.CLUSTER_PATH}/registry.json` - ) as (NounMetadata & { nodes?: string[] }) | null - return registry?.nodes || [] - } catch (err) { - return [] - } - } - - /** - * Update node registry in storage - */ - private async updateNodeRegistry(add?: string, remove?: string): Promise { - try { - let registry = await this.loadNodeRegistry() - - if (add && !registry.includes(add)) { - registry.push(add) - } - - if (remove) { - registry = registry.filter(id => id !== remove) - } - - await this.storage.saveMetadata(`${this.CLUSTER_PATH}/registry.json`, { - noun: 'Document', - nodes: registry, - updated: Date.now() - }) - } catch (err) { - console.error(`[${this.nodeId}] Failed to update registry:`, err) - } - } - - /** - * Check health of known nodes - */ - private async checkNodeHealth(): Promise { - if (!this.clusterConfig) return - - const now = Date.now() - const deadNodes: string[] = [] - - for (const [nodeId, nodeInfo] of Object.entries(this.clusterConfig.nodes)) { - if (nodeId === this.nodeId) continue - - if (now - nodeInfo.lastSeen > this.NODE_TIMEOUT) { - console.log(`[${this.nodeId}] Node ${nodeId} is dead (last seen ${now - nodeInfo.lastSeen}ms ago)`) - deadNodes.push(nodeId) - } - } - - // Remove dead nodes - for (const nodeId of deadNodes) { - delete this.clusterConfig.nodes[nodeId] - this.emit('nodeLost', nodeId) - - // If dead node was leader, trigger election - if (this.clusterConfig.leader === nodeId) { - await this.triggerLeaderElection() - } - } - - if (deadNodes.length > 0) { - // Trigger rebalance to reassign shards from dead nodes - await this.triggerRebalance() - } - } - - /** - * Load cluster configuration from storage - */ - private async loadClusterConfig(): Promise { - try { - const config = await this.storage.getMetadata(`${this.CLUSTER_PATH}/config.json`) - return config as unknown as ClusterConfig - } catch (err) { - // No cluster config exists yet - return null - } - } - - /** - * Save cluster configuration to storage - */ - private async saveClusterConfig(): Promise { - if (!this.clusterConfig) return - - await this.storage.saveMetadata( - `${this.CLUSTER_PATH}/config.json`, - { noun: 'Document', ...this.clusterConfig } - ) - } - - /** - * Trigger leader election (simplified - not full Raft) - */ - private async triggerLeaderElection(): Promise { - console.log(`[${this.nodeId}] Triggering leader election`) - - // Simple election: node with lowest ID wins - // In production, use proper Raft consensus - const activeNodes = Object.entries(this.clusterConfig!.nodes) - .filter(([_, info]) => Date.now() - info.lastSeen < this.NODE_TIMEOUT) - .sort(([a], [b]) => a.localeCompare(b)) - - if (activeNodes.length > 0) { - const [leaderId, leaderInfo] = activeNodes[0] - this.clusterConfig!.leader = leaderId - - if (leaderId === this.nodeId) { - console.log(`[${this.nodeId}] Became leader`) - this.nodeInfo.role = 'primary' - this.emit('becameLeader') - } else { - console.log(`[${this.nodeId}] Node ${leaderId} is the new leader`) - this.nodeInfo.role = 'replica' - this.emit('leaderElected', leaderId) - } - - await this.saveClusterConfig() - } - } - - /** - * Request shard assignment for this node - */ - private async requestShardAssignment(): Promise { - if (!this.clusterConfig) return - - // Calculate how many shards each node should have - const nodeCount = Object.keys(this.clusterConfig.nodes).length - const shardsPerNode = Math.ceil(this.clusterConfig.shards.count / nodeCount) - - // Find shards that need assignment - const unassignedShards: string[] = [] - - for (let i = 0; i < this.clusterConfig.shards.count; i++) { - const shardId = `shard-${i.toString().padStart(3, '0')}` - - if (!this.clusterConfig.shards.assignments[shardId] || - this.clusterConfig.shards.assignments[shardId].length === 0) { - unassignedShards.push(shardId) - } - } - - // Assign some shards to ourselves - const ourShare = unassignedShards.slice(0, shardsPerNode) - for (const shardId of ourShare) { - this.clusterConfig.shards.assignments[shardId] = [this.nodeId] - this.nodeInfo.shards.push(shardId) - } - - if (ourShare.length > 0) { - console.log(`[${this.nodeId}] Assigned ${ourShare.length} shards`) - await this.saveClusterConfig() - } - } - - /** - * Check if rebalancing is needed - */ - private shouldRebalance(): boolean { - if (!this.clusterConfig || !this.clusterConfig.settings.autoRebalance) { - return false - } - - // Check if shards are evenly distributed - const nodeCount = Object.keys(this.clusterConfig.nodes).length - if (nodeCount <= 1) return false - - const targetShardsPerNode = Math.ceil(this.clusterConfig.shards.count / nodeCount) - const variance = 2 // Allow some variance - - for (const nodeInfo of Object.values(this.clusterConfig.nodes)) { - const shardCount = nodeInfo.shards.length - if (Math.abs(shardCount - targetShardsPerNode) > variance) { - return true - } - } - - return false - } - - /** - * Trigger shard rebalancing - */ - private async triggerRebalance(): Promise { - // Only leader can trigger rebalance - if (this.clusterConfig?.leader !== this.nodeId) return - - console.log(`[${this.nodeId}] Triggering shard rebalance`) - - // This will be implemented with actual data migration - // For now, just redistribute shard assignments - await this.redistributeShards() - - this.emit('rebalanceTriggered') - } - - /** - * Redistribute shards among active nodes - */ - private async redistributeShards(): Promise { - if (!this.clusterConfig) return - - const activeNodes = Object.keys(this.clusterConfig.nodes) - .filter(id => Date.now() - this.clusterConfig!.nodes[id].lastSeen < this.NODE_TIMEOUT) - - if (activeNodes.length === 0) return - - const shardsPerNode = Math.ceil(this.clusterConfig.shards.count / activeNodes.length) - const newAssignments: Record = {} - - // Clear current shard assignments from nodes - for (const nodeInfo of Object.values(this.clusterConfig.nodes)) { - nodeInfo.shards = [] - } - - // Redistribute shards - let nodeIndex = 0 - for (let i = 0; i < this.clusterConfig.shards.count; i++) { - const shardId = `shard-${i.toString().padStart(3, '0')}` - const primaryNode = activeNodes[nodeIndex % activeNodes.length] - - // Assign primary - newAssignments[shardId] = [primaryNode] - this.clusterConfig.nodes[primaryNode].shards.push(shardId) - - // Assign replicas - const replicas: string[] = [] - for (let r = 1; r < Math.min(this.clusterConfig.settings.replicationFactor, activeNodes.length); r++) { - const replicaNode = activeNodes[(nodeIndex + r) % activeNodes.length] - if (replicaNode !== primaryNode) { - replicas.push(replicaNode) - } - } - - if (replicas.length > 0) { - newAssignments[shardId].push(...replicas) - } - - nodeIndex++ - } - - this.clusterConfig.shards.assignments = newAssignments - this.clusterConfig.version++ - this.clusterConfig.updated = Date.now() - - await this.saveClusterConfig() - - console.log(`[${this.nodeId}] Rebalanced ${this.clusterConfig.shards.count} shards across ${activeNodes.length} nodes`) - } - - /** - * Detect our public endpoint - */ - private async detectEndpoint(port: number): Promise { - // Try to detect public IP - const interfaces = os.networkInterfaces() - let ip = '127.0.0.1' - - // Find first non-internal IPv4 address - for (const iface of Object.values(interfaces)) { - if (!iface) continue - for (const addr of iface) { - if (addr.family === 'IPv4' && !addr.internal) { - ip = addr.address - break - } - } - } - - // In cloud environments, might need to detect public IP differently - if (process.env.PUBLIC_IP) { - ip = process.env.PUBLIC_IP - } else if (process.env.KUBERNETES_SERVICE_HOST) { - // In Kubernetes, use pod IP - ip = process.env.POD_IP || ip - } - - return `http://${ip}:${port}` - } - - /** - * Generate unique node ID - */ - private generateNodeId(): string { - const hostname = os.hostname() - const pid = process.pid - const random = Math.random().toString(36).substring(2, 8) - return `${hostname}-${pid}-${random}` - } - - /** - * Get current cluster configuration - */ - getClusterConfig(): ClusterConfig | null { - return this.clusterConfig - } - - /** - * Get active nodes - */ - getActiveNodes(): NodeInfo[] { - if (!this.clusterConfig) return [] - - const now = Date.now() - return Object.values(this.clusterConfig.nodes) - .filter(node => now - node.lastSeen < this.NODE_TIMEOUT) - } - - /** - * Get shards assigned to this node - */ - getMyShards(): string[] { - return this.nodeInfo.shards - } - - /** - * Update node statistics - */ - updateStats(stats: Partial): void { - Object.assign(this.nodeInfo.stats, stats) - } -} \ No newline at end of file diff --git a/src/index.ts b/src/index.ts index 24676498..5474dab9 100644 --- a/src/index.ts +++ b/src/index.ts @@ -77,32 +77,9 @@ export type { // Export Aggregation Engine export { AggregationIndex, AggregateMaterializer, bucketTimestamp, parseBucketRange } from './aggregation/index.js' -// Export zero-configuration types and enums +// Export zero-configuration input type export { - // Preset names - PresetName, - // Model configuration - ModelPrecision, - // Storage configuration - StorageOption, - // Feature configuration - FeatureSet, - // Distributed roles - DistributedRole, - // Categories - PresetCategory, - // Config type - BrainyZeroConfig, - // Extensibility - StorageProvider, - registerStorageAugmentation, - registerPresetAugmentation, - // Preset utilities - getPreset, - isValidPreset, - getPresetsByCategory, - getAllPresetNames, - getPresetDescription + BrainyZeroConfig } from './config/index.js' // Export Neural Import (AI data understanding) diff --git a/src/storage/adapters/baseStorageAdapter.ts b/src/storage/adapters/baseStorageAdapter.ts index 692baa99..8b65d977 100644 --- a/src/storage/adapters/baseStorageAdapter.ts +++ b/src/storage/adapters/baseStorageAdapter.ts @@ -1159,13 +1159,13 @@ export abstract class BaseStorageAdapter implements StorageAdapter { } /** - * Increment count for entity type - O(1) operation - * Protected by storage-specific mechanisms (mutex, distributed consensus, etc.) + * Increment count for entity type - O(1) operation. + * Concurrency is handled by the process-global mutex + * ({@link incrementEntityCountSafe}); this raw form is for callers that + * already hold it. * @param type The entity type */ protected incrementEntityCount(type: string): void { - // For distributed scenarios, this is aggregated across shards - // For single-node, this is protected by storage-specific locking this.entityCounts.set(type, (this.entityCounts.get(type) || 0) + 1) this.totalNounCount++ // Update cache @@ -1176,11 +1176,11 @@ export abstract class BaseStorageAdapter implements StorageAdapter { } /** - * Thread-safe increment for concurrent scenarios - * Uses mutex for single-node, distributed consensus for multi-node + * Thread-safe increment for concurrent scenarios. + * A process-global mutex serialises the read-modify-write so concurrent + * writers in the same process cannot lose updates. */ protected async incrementEntityCountSafe(type: string): Promise { - // Single-node mutex protection (distributed mode handled by coordinator) const mutex = getGlobalMutex() await mutex.runExclusive(`count-entity-${type}`, async () => { this.incrementEntityCount(type) @@ -1225,8 +1225,10 @@ export abstract class BaseStorageAdapter implements StorageAdapter { } /** - * Increment verb count - O(1) operation (now synchronous) - * Protected by storage-specific mechanisms (mutex, distributed consensus, etc.) + * Increment verb count - O(1) operation (now synchronous). + * Concurrency is handled by the process-global mutex + * ({@link incrementVerbCountSafe}); this raw form is for callers that already + * hold it. * @param type The verb type */ protected incrementVerbCount(type: string): void { @@ -1240,8 +1242,9 @@ export abstract class BaseStorageAdapter implements StorageAdapter { } /** - * Thread-safe increment for verb counts - * Uses mutex for single-node, distributed consensus for multi-node + * Thread-safe increment for verb counts. + * A process-global mutex serialises the read-modify-write so concurrent + * writers in the same process cannot lose updates. * @param type The verb type */ protected async incrementVerbCountSafe(type: string): Promise { diff --git a/src/storage/adapters/memoryStorage.ts b/src/storage/adapters/memoryStorage.ts index e2c86da2..9f6410cf 100644 --- a/src/storage/adapters/memoryStorage.ts +++ b/src/storage/adapters/memoryStorage.ts @@ -407,13 +407,9 @@ export class MemoryStorage extends BaseStorage { // Include services if present ...(statistics.services && { services: statistics.services.map(s => ({...s})) - }), - // Include distributedConfig if present - ...(statistics.distributedConfig && { - distributedConfig: JSON.parse(JSON.stringify(statistics.distributedConfig)) }) } - + // Since this is in-memory, there's no need for time-based partitioning // or legacy file handling } @@ -459,10 +455,6 @@ export class MemoryStorage extends BaseStorage { // Include services if present ...(this.statistics.services && { services: this.statistics.services.map(s => ({...s})) - }), - // Include distributedConfig if present - ...(this.statistics.distributedConfig && { - distributedConfig: JSON.parse(JSON.stringify(this.statistics.distributedConfig)) }) } diff --git a/src/storage/baseStorage.ts b/src/storage/baseStorage.ts index a2f2f373..e83b6841 100644 --- a/src/storage/baseStorage.ts +++ b/src/storage/baseStorage.ts @@ -144,7 +144,6 @@ const SINGLETON_SYSTEM_KEYS = new Set([ const SINGLETON_SYSTEM_PREFIXES = [ 'statistics_', - 'distributed_', ] function isSingletonSystemKey(key: string): boolean { @@ -2292,11 +2291,11 @@ export abstract class BaseStorage extends BaseStorageAdapter { await this.ensureInitialized() const keyInfo = this.analyzeKey(id, 'system') - // Try new distributed path first + // Try the new shard-prefixed path first const data = await this.readCanonicalObject(keyInfo.fullPath) if (data !== null) return data - // Backward compat: if key was distributed, fall back to legacy flat path + // Backward compat: if the key was sharded, fall back to the legacy flat path if (keyInfo.shardId !== null) { const legacyPath = `${SYSTEM_DIR}/${id}.json` return this.readCanonicalObject(legacyPath) @@ -2428,10 +2427,9 @@ export abstract class BaseStorage extends BaseStorageAdapter { * @returns Metadata or null if not found * * @performance - * - O(1) direct ID lookup - always 1 read (~500ms on cloud, ~10ms local) + * - O(1) direct ID lookup - always 1 read (~10ms on local disk) * - No caching complexity * - No type search fallbacks - * - Works in distributed systems without sync issues * * Type-first paths (removed) * Promoted to fast path for brain.get() optimization diff --git a/src/storage/operationalModes.ts b/src/storage/operationalModes.ts new file mode 100644 index 00000000..921fa64b --- /dev/null +++ b/src/storage/operationalModes.ts @@ -0,0 +1,73 @@ +/** + * @module storage/operationalModes + * @description Operational modes that gate which operations a Brainy instance may + * perform. Brainy's multi-process model runs one writer process plus any number of + * reader processes against a single shared on-disk store; the mode object is the + * in-process guard that enforces that contract. + * + * - {@link HybridMode} (the default, used by writer instances) permits reads, + * writes, and deletes — a writer must be able to read its own data. + * - {@link ReaderMode} (used by `mode: 'reader'` / `Brainy.openReadOnly()` and by + * `asOf()` historical snapshots) permits reads only; every mutation throws. + * + * `validateOperation()` is called at the top of each mutation path, and the + * `canWrite` flag drives the public read-only checks on the instance. + */ + +/** + * @description Base class for operational modes. Concrete modes declare which of + * read/write/delete they permit; {@link BaseOperationalMode.validateOperation} + * turns a disallowed operation into an explicit error. + */ +export abstract class BaseOperationalMode { + abstract canRead: boolean + abstract canWrite: boolean + abstract canDelete: boolean + + /** + * @description Throw if the requested operation is not allowed in this mode. + * Called at the top of every mutation method so read-only instances fail loudly + * rather than silently dropping writes. + * @param operation - The operation being attempted. + * @throws {Error} When the mode does not permit the operation. + */ + validateOperation(operation: 'read' | 'write' | 'delete'): void { + switch (operation) { + case 'read': + if (!this.canRead) { + throw new Error('Read operations are not allowed in write-only mode') + } + break + case 'write': + if (!this.canWrite) { + throw new Error('Write operations are not allowed in read-only mode') + } + break + case 'delete': + if (!this.canDelete) { + throw new Error('Delete operations are not allowed in this mode') + } + break + } + } +} + +/** + * @description Read-only mode. Reads are permitted; writes and deletes throw. + * Used by reader processes, `Brainy.openReadOnly()`, and historical snapshots. + */ +export class ReaderMode extends BaseOperationalMode { + canRead = true + canWrite = false + canDelete = false +} + +/** + * @description Read-write mode and the default for writer instances. Permits + * reads, writes, and deletes — a writer needs to read its own data. + */ +export class HybridMode extends BaseOperationalMode { + canRead = true + canWrite = true + canDelete = true +} diff --git a/src/storage/readOnlyOptimizations.ts b/src/storage/readOnlyOptimizations.ts deleted file mode 100644 index f1196cfc..00000000 --- a/src/storage/readOnlyOptimizations.ts +++ /dev/null @@ -1,530 +0,0 @@ -/** - * Read-Only Storage Optimizations for Production Deployments - * Implements compression, memory-mapping, and pre-built index segments - */ - -import { HNSWNoun, Vector } from '../coreTypes.js' - -// Compression types supported -enum CompressionType { - NONE = 'none', - GZIP = 'gzip', - QUANTIZATION = 'quantization', - HYBRID = 'hybrid' - // BROTLI removed - was not actually implemented -} - -// Vector quantization methods -enum QuantizationType { - SCALAR = 'scalar', // 8-bit scalar quantization - PRODUCT = 'product', // Product quantization - BINARY = 'binary' // Binary quantization -} - -interface CompressionConfig { - vectorCompression: CompressionType - metadataCompression: CompressionType - quantizationType?: QuantizationType - quantizationBits?: number - compressionLevel?: number -} - -interface ReadOnlyConfig { - prebuiltIndexPath?: string - memoryMapped?: boolean - compression: CompressionConfig - segmentSize?: number // For index segmentation - prefetchSegments?: number - cacheIndexInMemory?: boolean -} - -interface IndexSegment { - id: string - nodeCount: number - vectorDimension: number - compression: CompressionType - s3Key?: string - localPath?: string - loadedInMemory: boolean - lastAccessed: number -} - -/** - * Read-only storage optimizations for high-performance production deployments - */ -export class ReadOnlyOptimizations { - private config: Required - private segments: Map = new Map() - private compressionStats = { - originalSize: 0, - compressedSize: 0, - compressionRatio: 0, - decompressionTime: 0 - } - - // Quantization codebooks for vector compression - private quantizationCodebooks: Map = new Map() - - // Memory-mapped buffers for large datasets - private memoryMappedBuffers: Map = new Map() - - constructor(config: Partial = {}) { - this.config = { - prebuiltIndexPath: '', - memoryMapped: true, - compression: { - vectorCompression: CompressionType.QUANTIZATION, - metadataCompression: CompressionType.GZIP, - quantizationType: QuantizationType.SCALAR, - quantizationBits: 8, - compressionLevel: 6 - }, - segmentSize: 10000, // 10k nodes per segment - prefetchSegments: 3, - cacheIndexInMemory: false, - ...config - } - - if (config.compression) { - this.config.compression = { ...this.config.compression, ...config.compression } - } - } - - /** - * Compress vector data using specified compression method - */ - public async compressVector(vector: Vector, segmentId: string): Promise { - const startTime = Date.now() - let compressedData: ArrayBuffer - - switch (this.config.compression.vectorCompression) { - case CompressionType.QUANTIZATION: - compressedData = await this.quantizeVector(vector, segmentId) - break - - case CompressionType.GZIP: - const gzipBuffer = new Float32Array(vector).buffer - compressedData = await this.gzipCompress(gzipBuffer.slice(0)) - break - - // Brotli removed - was not implemented - - case CompressionType.HYBRID: - // First quantize, then compress - const quantized = await this.quantizeVector(vector, segmentId) - compressedData = await this.gzipCompress(quantized) - break - - default: - const defaultBuffer = new Float32Array(vector).buffer - compressedData = defaultBuffer.slice(0) - break - } - - // Update compression statistics - const originalSize = vector.length * 4 // 4 bytes per float32 - this.compressionStats.originalSize += originalSize - this.compressionStats.compressedSize += compressedData.byteLength - this.compressionStats.decompressionTime += Date.now() - startTime - - this.updateCompressionRatio() - - return compressedData - } - - /** - * Decompress vector data - */ - public async decompressVector( - compressedData: ArrayBuffer, - segmentId: string, - originalDimension: number - ): Promise { - switch (this.config.compression.vectorCompression) { - case CompressionType.QUANTIZATION: - return this.dequantizeVector(compressedData, segmentId, originalDimension) - - case CompressionType.GZIP: - const gzipDecompressed = await this.gzipDecompress(compressedData) - return Array.from(new Float32Array(gzipDecompressed)) - - // Brotli removed - was not implemented - - case CompressionType.HYBRID: - const gzipStage = await this.gzipDecompress(compressedData) - return this.dequantizeVector(gzipStage, segmentId, originalDimension) - - default: - return Array.from(new Float32Array(compressedData)) - } - } - - /** - * Scalar quantization of vectors to 8-bit integers - */ - private async quantizeVector(vector: Vector, segmentId: string): Promise { - let codebook = this.quantizationCodebooks.get(segmentId) - - if (!codebook) { - // Create codebook (min/max values for scaling) - const min = Math.min(...vector) - const max = Math.max(...vector) - codebook = new Float32Array([min, max]) - this.quantizationCodebooks.set(segmentId, codebook) - } - - const [min, max] = codebook - const scale = (max - min) / 255 // 8-bit quantization - - const quantized = new Uint8Array(vector.length) - for (let i = 0; i < vector.length; i++) { - quantized[i] = Math.round((vector[i] - min) / scale) - } - - // Store codebook with quantized data - const result = new ArrayBuffer(quantized.byteLength + codebook.byteLength) - const resultView = new Uint8Array(result) - - // First 8 bytes: codebook (min, max as float32) - resultView.set(new Uint8Array(codebook.buffer), 0) - // Remaining bytes: quantized vector - resultView.set(quantized, codebook.byteLength) - - return result - } - - /** - * Dequantize 8-bit vectors back to float32 - */ - private dequantizeVector( - quantizedData: ArrayBuffer, - segmentId: string, - dimension: number - ): Vector { - const dataView = new Uint8Array(quantizedData) - - // Extract codebook (first 8 bytes) - const codebookBytes = dataView.slice(0, 8) - const codebook = new Float32Array(codebookBytes.buffer) - const [min, max] = codebook - - // Extract quantized vector - const quantized = dataView.slice(8) - const scale = (max - min) / 255 - - const result: Vector = [] - for (let i = 0; i < dimension; i++) { - result[i] = min + quantized[i] * scale - } - - return result - } - - /** - * GZIP compression using browser/Node.js APIs - */ - private async gzipCompress(data: ArrayBuffer): Promise { - if (typeof CompressionStream !== 'undefined') { - // Browser environment - const stream = new CompressionStream('gzip') - const writer = stream.writable.getWriter() - const reader = stream.readable.getReader() - - writer.write(new Uint8Array(data)) - writer.close() - - const chunks: Uint8Array[] = [] - let result = await reader.read() - - while (!result.done) { - chunks.push(result.value) - result = await reader.read() - } - - // Combine chunks - const totalLength = chunks.reduce((sum, chunk) => sum + chunk.length, 0) - const combined = new Uint8Array(totalLength) - let offset = 0 - - for (const chunk of chunks) { - combined.set(chunk, offset) - offset += chunk.length - } - - return combined.buffer - } else { - // Node.js environment - would use zlib - console.warn('GZIP compression not available, returning original data') - return data - } - } - - /** - * GZIP decompression - */ - private async gzipDecompress(compressedData: ArrayBuffer): Promise { - if (typeof DecompressionStream !== 'undefined') { - // Browser environment - const stream = new DecompressionStream('gzip') - const writer = stream.writable.getWriter() - const reader = stream.readable.getReader() - - writer.write(new Uint8Array(compressedData)) - writer.close() - - const chunks: Uint8Array[] = [] - let result = await reader.read() - - while (!result.done) { - chunks.push(result.value) - result = await reader.read() - } - - // Combine chunks - const totalLength = chunks.reduce((sum, chunk) => sum + chunk.length, 0) - const combined = new Uint8Array(totalLength) - let offset = 0 - - for (const chunk of chunks) { - combined.set(chunk, offset) - offset += chunk.length - } - - return combined.buffer - } else { - console.warn('GZIP decompression not available, returning original data') - return compressedData - } - } - - // Brotli methods removed - were not implemented - - /** - * Create prebuilt index segments for faster loading - */ - public async createPrebuiltSegments( - nodes: HNSWNoun[], - outputPath: string - ): Promise { - const segments: IndexSegment[] = [] - const segmentSize = this.config.segmentSize - - console.log(`Creating ${Math.ceil(nodes.length / segmentSize)} prebuilt segments`) - - for (let i = 0; i < nodes.length; i += segmentSize) { - const segmentNodes = nodes.slice(i, i + segmentSize) - const segmentId = `segment_${Math.floor(i / segmentSize)}` - - const segment: IndexSegment = { - id: segmentId, - nodeCount: segmentNodes.length, - vectorDimension: segmentNodes[0]?.vector.length || 0, - compression: this.config.compression.vectorCompression, - localPath: `${outputPath}/${segmentId}.dat`, - loadedInMemory: false, - lastAccessed: 0 - } - - // Compress and serialize segment data - const compressedData = await this.compressSegment(segmentNodes) - - // In a real implementation, you would write this to disk/S3 - console.log(`Created segment ${segmentId} with ${compressedData.byteLength} bytes`) - - segments.push(segment) - this.segments.set(segmentId, segment) - } - - return segments - } - - /** - * Compress an entire segment of nodes - */ - private async compressSegment(nodes: HNSWNoun[]): Promise { - const serialized = JSON.stringify(nodes.map(node => ({ - id: node.id, - vector: node.vector, - connections: this.serializeConnections(node.connections) - }))) - - const encoder = new TextEncoder() - const data = encoder.encode(serialized) - - // Apply metadata compression - switch (this.config.compression.metadataCompression) { - case CompressionType.GZIP: - return this.gzipCompress(data.buffer.slice(0) as ArrayBuffer) - // Brotli removed - was not implemented - default: - return data.buffer.slice(0) as ArrayBuffer - } - } - - /** - * Load a segment from storage with caching - */ - public async loadSegment(segmentId: string): Promise { - const segment = this.segments.get(segmentId) - if (!segment) { - throw new Error(`Segment ${segmentId} not found`) - } - - segment.lastAccessed = Date.now() - - // Check if segment is already loaded in memory - if (segment.loadedInMemory && this.memoryMappedBuffers.has(segmentId)) { - return this.deserializeSegment(this.memoryMappedBuffers.get(segmentId)!) - } - - // Load from storage (S3, disk, etc.) - const compressedData = await this.loadSegmentFromStorage(segment) - - // Cache in memory if configured - if (this.config.cacheIndexInMemory) { - this.memoryMappedBuffers.set(segmentId, compressedData) - segment.loadedInMemory = true - } - - return this.deserializeSegment(compressedData) - } - - /** - * Load segment data from storage - */ - private async loadSegmentFromStorage(segment: IndexSegment): Promise { - // Load segment from memory-mapped buffer if available - const cached = this.memoryMappedBuffers.get(segment.id) - if (cached) { - return cached - } - - // This feature requires actual storage backend integration (S3, file system, etc) - // Return empty buffer as this is an optional optimization feature - console.warn(`Segment loading optimization not available for segment ${segment.id}. Using standard storage.`) - return new ArrayBuffer(0) - } - - /** - * Deserialize and decompress segment data - */ - private async deserializeSegment(compressedData: ArrayBuffer): Promise { - // Decompress metadata - let decompressed: ArrayBuffer - - switch (this.config.compression.metadataCompression) { - case CompressionType.GZIP: - decompressed = await this.gzipDecompress(compressedData) - break - // Brotli removed - was not implemented - default: - decompressed = compressedData - break - } - - // Parse JSON - const decoder = new TextDecoder() - const jsonStr = decoder.decode(decompressed) - const parsed = JSON.parse(jsonStr) - - // Reconstruct HNSWNoun objects - return parsed.map((item: any) => ({ - id: item.id, - vector: item.vector, - connections: this.deserializeConnections(item.connections) - })) - } - - /** - * Serialize connections Map for storage - */ - private serializeConnections(connections: Map>): Record { - const result: Record = {} - for (const [level, nodeIds] of connections.entries()) { - result[level.toString()] = Array.from(nodeIds) - } - return result - } - - /** - * Deserialize connections from storage format - */ - private deserializeConnections(serialized: Record): Map> { - const result = new Map>() - for (const [levelStr, nodeIds] of Object.entries(serialized)) { - result.set(parseInt(levelStr), new Set(nodeIds)) - } - return result - } - - /** - * Prefetch segments based on access patterns - */ - public async prefetchSegments(currentSegmentId: string): Promise { - const segment = this.segments.get(currentSegmentId) - if (!segment) return - - // Simple prefetching strategy - load adjacent segments - const segmentNumber = parseInt(currentSegmentId.split('_')[1]) - const toPrefetch: string[] = [] - - for (let i = 1; i <= this.config.prefetchSegments; i++) { - const nextId = `segment_${segmentNumber + i}` - const prevId = `segment_${segmentNumber - i}` - - if (this.segments.has(nextId) && !this.memoryMappedBuffers.has(nextId)) { - toPrefetch.push(nextId) - } - if (this.segments.has(prevId) && !this.memoryMappedBuffers.has(prevId)) { - toPrefetch.push(prevId) - } - } - - // Prefetch in background - for (const segmentId of toPrefetch) { - this.loadSegment(segmentId).catch(error => { - console.warn(`Failed to prefetch segment ${segmentId}:`, error) - }) - } - } - - /** - * Update compression statistics - */ - private updateCompressionRatio(): void { - if (this.compressionStats.originalSize > 0) { - this.compressionStats.compressionRatio = - this.compressionStats.compressedSize / this.compressionStats.originalSize - } - } - - /** - * Get compression statistics - */ - public getCompressionStats(): typeof this.compressionStats & { - segmentCount: number - memoryUsage: number - } { - const memoryUsage = Array.from(this.memoryMappedBuffers.values()) - .reduce((sum, buffer) => sum + buffer.byteLength, 0) - - return { - ...this.compressionStats, - segmentCount: this.segments.size, - memoryUsage - } - } - - /** - * Cleanup memory-mapped buffers - */ - public cleanup(): void { - this.memoryMappedBuffers.clear() - this.quantizationCodebooks.clear() - - // Mark all segments as not loaded - for (const segment of this.segments.values()) { - segment.loadedInMemory = false - } - } -} \ No newline at end of file diff --git a/src/types/brainy.types.ts b/src/types/brainy.types.ts index cafa7b08..15d4072f 100644 --- a/src/types/brainy.types.ts +++ b/src/types/brainy.types.ts @@ -1206,25 +1206,6 @@ export interface BrainyConfig { } | StorageAdapter - /** - * Distributed-cluster configuration. **Explicit opt-in only** — Brainy - * never auto-enables cluster mode from environment heuristics (a single - * production process is the 90th-percentile deployment). Set - * `enabled: true` (or the `BRAINY_DISTRIBUTED=true` env var) to activate; - * the remaining fields then default sensibly (64 shards, 3 replicas, - * raft consensus, http transport, hostname-derived nodeId). - */ - distributed?: { - enabled: boolean - nodeId?: string - nodes?: string[] // Other nodes in cluster - coordinatorUrl?: string // Coordinator endpoint - shardCount?: number // Number of shards (default: 64) - replicationFactor?: number // Number of replicas (default: 3) - consensus?: 'raft' | 'none' // Consensus mechanism - transport?: 'tcp' | 'http' | 'udp' - } - /** * Disable the automatic index rebuild check during `init()`. By default * Brainy auto-decides from dataset size: small datasets rebuild missing diff --git a/src/types/distributedTypes.ts b/src/types/distributedTypes.ts deleted file mode 100644 index 94459b5e..00000000 --- a/src/types/distributedTypes.ts +++ /dev/null @@ -1,236 +0,0 @@ -/** - * Distributed types for Brainy - * Defines types for distributed operations across multiple instances - */ - -export type InstanceRole = 'reader' | 'writer' | 'hybrid' - -export type PartitionStrategy = 'hash' | 'semantic' | 'manual' - -export interface DistributedConfig { - /** - * Enable distributed mode - * Can be boolean for auto-detection or specific configuration - */ - enabled?: boolean | 'auto' - - /** - * Role of this instance in the distributed system - * - reader: Read-only access, optimized for queries - * - writer: Write-focused, handles data ingestion - * - hybrid: Can both read and write (requires coordination) - */ - role?: InstanceRole - - /** - * Unique identifier for this instance - * Auto-generated if not provided - */ - instanceId?: string - - /** - * Path to shared configuration file in S3 - * Default: '_brainy/config.json' - */ - configPath?: string - - /** - * Heartbeat interval in milliseconds - * Default: 30000 (30 seconds) - */ - heartbeatInterval?: number - - /** - * Config check interval in milliseconds - * Default: 10000 (10 seconds) - */ - configCheckInterval?: number - - /** - * Instance timeout in milliseconds - * Instances not seen for this duration are considered dead - * Default: 60000 (60 seconds) - */ - instanceTimeout?: number -} - -export interface SharedConfig { - /** - * Configuration version for compatibility checking - */ - version: number - - /** - * Last update timestamp - */ - updated: string - - /** - * Global settings that must be consistent across all instances - */ - settings: { - /** - * Partitioning strategy - * - hash: Deterministic hash-based partitioning (recommended for multi-writer) - * - semantic: Group similar vectors (single writer only) - * - manual: Explicit partition assignment - */ - partitionStrategy: PartitionStrategy - - /** - * Number of partitions (for hash strategy) - */ - partitionCount: number - - /** - * Embedding model name (must be consistent) - */ - embeddingModel: string - - /** - * Vector dimensions - */ - dimensions: number - - /** - * Distance metric - */ - distanceMetric: 'cosine' | 'euclidean' | 'manhattan' - - /** - * HNSW parameters (must be consistent for index compatibility) - */ - hnswParams?: { - M: number - efConstruction: number - maxElements?: number - } - } - - /** - * Active instances in the distributed system - */ - instances: { - [instanceId: string]: InstanceInfo - } - - /** - * Partition assignments (for manual strategy) - */ - partitionAssignments?: { - [instanceId: string]: string[] - } -} - -export interface InstanceInfo { - /** - * Instance role - */ - role: InstanceRole - - /** - * Instance status - */ - status: 'active' | 'inactive' | 'unhealthy' - - /** - * Last heartbeat timestamp - */ - lastHeartbeat: string - - /** - * Optional endpoint for health checks - */ - endpoint?: string - - /** - * Instance metrics - */ - metrics?: { - vectorCount?: number - cacheHitRate?: number - memoryUsage?: number - cpuUsage?: number - } - - /** - * Assigned partitions (for manual assignment) - */ - assignedPartitions?: string[] - - /** - * Preferred partitions (for affinity) - */ - preferredPartitions?: number[] -} - -export interface DomainMetadata { - /** - * Domain identifier for logical data separation - */ - domain?: string - - /** - * Additional domain-specific metadata - */ - domainMetadata?: Record -} - -export interface CacheStrategy { - /** - * Percentage of memory allocated to hot cache (0-1) - */ - hotCacheRatio: number - - /** - * Enable aggressive prefetching - */ - prefetchAggressive?: boolean - - /** - * Cache time-to-live in milliseconds - */ - ttl?: number - - /** - * Enable compression to trade CPU for memory - */ - compressionEnabled?: boolean - - /** - * Write buffer size for batching - */ - writeBufferSize?: number - - /** - * Enable write batching - */ - batchWrites?: boolean - - /** - * Adaptive caching based on workload - */ - adaptive?: boolean -} - -export interface OperationalMode { - /** - * Whether this mode can read - */ - canRead: boolean - - /** - * Whether this mode can write - */ - canWrite: boolean - - /** - * Whether this mode can delete - */ - canDelete: boolean - - /** - * Cache strategy for this mode - */ - cacheStrategy: CacheStrategy -} \ No newline at end of file diff --git a/src/utils/autoConfiguration.ts b/src/utils/autoConfiguration.ts index bf7e9886..8222dbae 100644 --- a/src/utils/autoConfiguration.ts +++ b/src/utils/autoConfiguration.ts @@ -25,7 +25,6 @@ export interface AutoConfigResult { targetSearchLatency: number enablePartitioning: boolean enableCompression: boolean - enableDistributedSearch: boolean enablePredictiveCaching: boolean partitionStrategy: 'semantic' | 'hash' maxNodesPerPartition: number @@ -121,7 +120,6 @@ export class AutoConfiguration { // Learn from search performance if (metrics.averageSearchTime > 200) { // Too slow - optimize for speed - adjustments.enableDistributedSearch = true adjustments.maxNodesPerPartition = Math.max(10000, (this.cachedConfig?.recommendedConfig.maxNodesPerPartition || 50000) * 0.8) } else if (metrics.averageSearchTime < 50) { // Very fast - can optimize for quality @@ -282,7 +280,6 @@ export class AutoConfiguration { targetSearchLatency: 150, enablePartitioning: datasetSize > 25000, enableCompression: memoryBudget < 2 * 1024 * 1024 * 1024, - enableDistributedSearch: resources.cpuCores > 2 && datasetSize > 50000, enablePredictiveCaching: true, partitionStrategy: 'semantic' as const, maxNodesPerPartition: 50000, @@ -316,7 +313,6 @@ export class AutoConfiguration { config.maxNodesPerPartition = 100000 } else if (datasetSize < 10000) { config.enablePartitioning = false - config.enableDistributedSearch = false config.partitionStrategy = 'semantic' // Keep semantic but disable partitioning } diff --git a/src/utils/metadataNamespace.ts b/src/utils/metadataNamespace.ts index 32c2ed6b..e14af1df 100644 --- a/src/utils/metadataNamespace.ts +++ b/src/utils/metadataNamespace.ts @@ -25,7 +25,6 @@ export interface BrainyInternalMetadata { updated: number // Unix timestamp // Optional internal fields - partition?: number // For distributed mode domain?: string // Domain classification priority?: number // Query priority hint ttl?: number // Time to live diff --git a/src/utils/statisticsCollector.ts b/src/utils/statisticsCollector.ts index b2574cc5..44936626 100644 --- a/src/utils/statisticsCollector.ts +++ b/src/utils/statisticsCollector.ts @@ -391,7 +391,8 @@ export class StatisticsCollector { } /** - * Merge statistics from storage (for distributed systems) + * Merge persisted statistics from storage into the in-memory collector + * (used when restoring counts on startup). */ mergeFromStorage(stored: Partial): void { // Merge content types diff --git a/tests/distributed-demo.test.ts b/tests/distributed-demo.test.ts deleted file mode 100644 index 370be956..00000000 --- a/tests/distributed-demo.test.ts +++ /dev/null @@ -1,202 +0,0 @@ -/** - * Distributed Brainy Demo Test - * - * Demonstrates the complete distributed features: - * - Zero-config discovery via S3 - * - Automatic sharding - * - Load balancing - * - Shard migration - * - Distributed queries - */ - -import { describe, it, expect, beforeAll, afterAll } from 'vitest' -import { Brainy } from '../src/brainy.js' - -describe('Distributed Brainy Demo', () => { - let node1: Brainy - let node2: Brainy - let node3: Brainy - - beforeAll(async () => { - // Node 1: First node starts cluster - node1 = new Brainy({ requireSubtype: false, - storage: { - s3Storage: { - bucket: process.env.S3_BUCKET || 'brainy-test', - region: process.env.AWS_REGION || 'us-east-1' - } - }, - distributed: true, // That's it! Everything else is automatic - logging: { verbose: true } - }) - - await node1.init() - console.log('Node 1 started - became cluster leader') - - // Node 2: Automatically discovers and joins - node2 = new Brainy({ requireSubtype: false, - storage: { - s3Storage: { - bucket: process.env.S3_BUCKET || 'brainy-test', - region: process.env.AWS_REGION || 'us-east-1' - } - }, - distributed: true, - logging: { verbose: true } - }) - - await node2.init() - console.log('Node 2 started - automatically joined cluster') - - // Node 3: Also joins automatically - node3 = new Brainy({ requireSubtype: false, - storage: { - s3Storage: { - bucket: process.env.S3_BUCKET || 'brainy-test', - region: process.env.AWS_REGION || 'us-east-1' - } - }, - distributed: true, - logging: { verbose: true } - }) - - await node3.init() - console.log('Node 3 started - automatically joined cluster') - - // Give nodes time to discover each other - await new Promise(resolve => setTimeout(resolve, 2000)) - }) - - afterAll(async () => { - await Promise.all([ - node1?.close(), - node2?.close(), - node3?.close() - ]) - }) - - it('should automatically distribute data across nodes', async () => { - // Add data from different nodes - automatically sharded - const doc1 = await node1.add('First document about AI', 'document') - const doc2 = await node2.add('Second document about machine learning', 'document') - const doc3 = await node3.add('Third document about neural networks', 'document') - - console.log('Added documents:', { doc1, doc2, doc3 }) - - // Each node can find all documents (distributed query) - const results1 = await node1.find('AI machine learning') - const results2 = await node2.find('AI machine learning') - const results3 = await node3.find('AI machine learning') - - // All nodes should see the same results - expect(results1.length).toBeGreaterThan(0) - expect(results2.length).toBe(results1.length) - expect(results3.length).toBe(results1.length) - - console.log(`All nodes found ${results1.length} documents`) - }) - - it('should handle node failures gracefully', async () => { - // Add a document - const docId = await node1.add('Important data that must not be lost', 'critical') - - // Simulate node 2 going down - await node2.close() - console.log('Node 2 went offline') - - // Data should still be accessible from other nodes - const result = await node1.get(docId) - expect(result).toBeDefined() - expect(result.data).toContain('Important data') - - console.log('Data still accessible after node failure') - }) - - it('should automatically rebalance when nodes join', async () => { - // Start a new node - const node4 = new Brainy({ requireSubtype: false, - storage: { - s3Storage: { - bucket: process.env.S3_BUCKET || 'brainy-test', - region: process.env.AWS_REGION || 'us-east-1' - } - }, - distributed: true - }) - - await node4.init() - console.log('Node 4 joined - automatic rebalancing started') - - // Give time for rebalancing - await new Promise(resolve => setTimeout(resolve, 3000)) - - // Node 4 should now handle queries - const results = await node4.find('data') - expect(results.length).toBeGreaterThan(0) - - console.log('Node 4 successfully serving queries after rebalancing') - - await node4.close() - }) - - it('should support complex distributed operations', async () => { - // Parallel writes from all nodes - const promises = [] - for (let i = 0; i < 10; i++) { - promises.push(node1.add(`Document ${i} from node 1`, 'batch')) - promises.push(node3.add(`Document ${i} from node 3`, 'batch')) - } - - await Promise.all(promises) - console.log('Added 20 documents in parallel from 2 nodes') - - // Complex query executed across all shards - const results = await node1.find('document', 5) - expect(results.length).toBe(5) - - // Triple Intelligence works across distributed data - const triple = await node1.tripleSearch( - 'node', // Subject - 'creates', // Verb - 'document' // Object - ) - - expect(triple.results).toBeDefined() - console.log('Triple Intelligence query executed across cluster') - }) -}) - -// Usage Example for Documentation -export function distributedExample() { - return ` -// Zero-Config Distributed Setup -const brain = new Brainy({ requireSubtype: false, - storage: { - s3Storage: { - bucket: 'my-data', - region: 'us-east-1' - } - }, - distributed: true // That's it! -}) - -await brain.init() - -// Everything else is automatic: -// ✅ Nodes discover each other via S3 -// ✅ Data automatically sharded across nodes -// ✅ Queries automatically distributed -// ✅ Automatic failover and recovery -// ✅ Zero-downtime scaling - -// Add data - automatically distributed -await brain.add('My document', 'doc') - -// Query - automatically searches all nodes -const results = await brain.find('search term') - -// Nodes can join/leave anytime -// Data automatically rebalances -// No configuration needed! -` -} \ No newline at end of file diff --git a/tests/helpers/distributed-cluster.ts b/tests/helpers/distributed-cluster.ts deleted file mode 100644 index b538330e..00000000 --- a/tests/helpers/distributed-cluster.ts +++ /dev/null @@ -1,300 +0,0 @@ -import { GenericContainer, StartedTestContainer, Wait } from 'testcontainers'; -import { Brainy } from '../../src/brainy'; - -export interface DistributedTestNode { - id: string; - brain: Brainy; - port: number; -} - -export interface DistributedClusterConfig { - nodeCount: number; - redisHost?: string; - redisPort?: number; - useTestContainers?: boolean; -} - -export class DistributedTestCluster { - private nodes: DistributedTestNode[] = []; - private redisContainer?: StartedTestContainer; - private config: DistributedClusterConfig; - private redisEndpoint?: { host: string; port: number }; - - constructor(config: DistributedClusterConfig) { - this.config = { - useTestContainers: config.useTestContainers !== false, - ...config, - nodeCount: config.nodeCount || 3 - }; - } - - async start(): Promise { - console.log(`Starting distributed test cluster with ${this.config.nodeCount} nodes...`); - - // Start Redis if using test containers - if (this.config.useTestContainers && !this.config.redisHost) { - await this.startRedis(); - } else { - this.redisEndpoint = { - host: this.config.redisHost || 'localhost', - port: this.config.redisPort || 6379 - }; - } - - // Start cluster nodes - await this.startNodes(); - - // Wait for nodes to discover each other - await this.waitForClusterFormation(); - - console.log(`Distributed cluster ready with ${this.nodes.length} nodes`); - } - - private async startRedis(): Promise { - console.log('Starting Redis test container...'); - - this.redisContainer = await new GenericContainer('redis:7-alpine') - .withExposedPorts(6379) - .withCommand(['redis-server', '--appendonly', 'yes']) - .withWaitStrategy(Wait.forLogMessage('Ready to accept connections')) - .start(); - - this.redisEndpoint = { - host: this.redisContainer.getHost(), - port: this.redisContainer.getMappedPort(6379) - }; - - console.log(`Redis started on ${this.redisEndpoint.host}:${this.redisEndpoint.port}`); - } - - private async startNodes(): Promise { - const basePort = 8000; - - for (let i = 0; i < this.config.nodeCount; i++) { - const nodeId = `node-${i + 1}`; - const port = basePort + i; - - console.log(`Starting ${nodeId} on port ${port}...`); - - const brain = new Brainy({ requireSubtype: false, - storage: { - type: 'memory' - } - // distributed config is not part of BrainyConfig - // distributed: { - // enabled: true, - // nodeId: nodeId, - // redis: { - // host: this.redisEndpoint!.host, - // port: this.redisEndpoint!.port - // }, - // server: { - // port: port, - // host: '0.0.0.0' - // }, - // discovery: { - // interval: 1000, - // timeout: 500 - // }, - // sharding: { - // enabled: true, - // replicas: 2 - // }, - // consensus: { - // enabled: true, - // quorum: Math.floor(this.config.nodeCount / 2) + 1 - // } - // } - }); - - await brain.init(); - - this.nodes.push({ - id: nodeId, - brain: brain, - port: port - }); - } - } - - private async waitForClusterFormation(): Promise { - console.log('Waiting for cluster formation...'); - - const maxAttempts = 30; - const delayMs = 1000; - - for (let attempt = 0; attempt < maxAttempts; attempt++) { - const allNodesDiscovered = await this.checkClusterHealth(); - - if (allNodesDiscovered) { - console.log('All nodes have discovered each other'); - return; - } - - await new Promise(resolve => setTimeout(resolve, delayMs)); - } - - throw new Error('Cluster formation timeout - nodes failed to discover each other'); - } - - private async checkClusterHealth(): Promise { - for (const node of this.nodes) { - const health = await node.brain.health(); - - if (!health.distributed || !health.distributed.connectedNodes) { - return false; - } - - // Each node should see all other nodes - const expectedNodeCount = this.config.nodeCount - 1; - if (health.distributed.connectedNodes.length < expectedNodeCount) { - return false; - } - } - - return true; - } - - async stop(): Promise { - console.log('Stopping distributed test cluster...'); - - // Stop all nodes - for (const node of this.nodes) { - try { - if (node.brain.close) { - await node.brain.close(); - } - console.log(`Stopped ${node.id}`); - } catch (error) { - console.warn(`Failed to stop ${node.id}:`, error); - } - } - - // Stop Redis container - if (this.redisContainer) { - await this.redisContainer.stop(); - console.log('Redis container stopped'); - } - - this.nodes = []; - console.log('Distributed cluster stopped'); - } - - getNodes(): DistributedTestNode[] { - return this.nodes; - } - - getNode(index: number): DistributedTestNode { - if (index < 0 || index >= this.nodes.length) { - throw new Error(`Invalid node index: ${index}`); - } - return this.nodes[index]; - } - - getPrimaryNode(): DistributedTestNode { - return this.nodes[0]; - } - - getSecondaryNodes(): DistributedTestNode[] { - return this.nodes.slice(1); - } - - async executeOnAllNodes(fn: (brain: Brainy) => Promise): Promise { - return Promise.all(this.nodes.map(node => fn(node.brain))); - } - - async executeOnNode(index: number, fn: (brain: Brainy) => Promise): Promise { - const node = this.getNode(index); - return fn(node.brain); - } - - async simulateNodeFailure(index: number): Promise { - const node = this.getNode(index); - console.log(`Simulating failure of ${node.id}...`); - - if (node.brain.close) { - await node.brain.close(); - } - - // Remove from active nodes - this.nodes.splice(index, 1); - } - - async addNode(): Promise { - const nodeId = `node-${this.nodes.length + 1}`; - const port = 8000 + this.nodes.length; - - console.log(`Adding new node ${nodeId} on port ${port}...`); - - const brain = new Brainy({ requireSubtype: false, - storage: { - type: 'memory' - } - // distributed config is not part of BrainyConfig - // distributed: { - // enabled: true, - // nodeId: nodeId, - // redis: { - // host: this.redisEndpoint!.host, - // port: this.redisEndpoint!.port - // }, - // server: { - // port: port, - // host: '0.0.0.0' - // }, - // discovery: { - // interval: 1000, - // timeout: 500 - // }, - // sharding: { - // enabled: true, - // replicas: 2 - // } - // } - }); - - await brain.init(); - - const newNode = { id: nodeId, brain, port }; - this.nodes.push(newNode); - - // Wait for new node to join cluster - await this.waitForNodeDiscovery(nodeId); - - return newNode; - } - - private async waitForNodeDiscovery(nodeId: string): Promise { - console.log(`Waiting for ${nodeId} to join cluster...`); - - const maxAttempts = 10; - const delayMs = 1000; - - for (let attempt = 0; attempt < maxAttempts; attempt++) { - const discovered = await this.isNodeDiscovered(nodeId); - - if (discovered) { - console.log(`${nodeId} has joined the cluster`); - return; - } - - await new Promise(resolve => setTimeout(resolve, delayMs)); - } - - throw new Error(`Timeout waiting for ${nodeId} to join cluster`); - } - - private async isNodeDiscovered(nodeId: string): Promise { - // Check if other nodes can see the new node - for (const node of this.nodes) { - if (node.id === nodeId) continue; - - const health = await node.brain.health(); - if (health.distributed?.connectedNodes?.includes(nodeId)) { - return true; - } - } - - return false; - } -} \ No newline at end of file diff --git a/tests/scripts/run-all-tests.sh b/tests/scripts/run-all-tests.sh index 304289d1..ff07abc3 100755 --- a/tests/scripts/run-all-tests.sh +++ b/tests/scripts/run-all-tests.sh @@ -67,7 +67,7 @@ echo "" echo "📦 Batch 10: Other Tests" echo "------------------------" -npm test -- --run tests/distributed*.test.ts tests/cli.test.ts tests/brainy-chat.test.ts tests/pagination.test.ts tests/vector-operations.test.ts tests/error-handling.test.ts tests/specialized-scenarios.test.ts tests/multi-environment.test.ts tests/statistics.test.ts tests/type-utils.test.ts 2>&1 | tee batch10.log | grep -E "✓|×|↓" | tail -20 +npm test -- --run tests/cli.test.ts tests/brainy-chat.test.ts tests/pagination.test.ts tests/vector-operations.test.ts tests/error-handling.test.ts tests/specialized-scenarios.test.ts tests/multi-environment.test.ts tests/statistics.test.ts tests/type-utils.test.ts 2>&1 | tee batch10.log | grep -E "✓|×|↓" | tail -20 echo "" # Aggregate results diff --git a/tests/transaction/integration/distributed-transactions.test.ts b/tests/transaction/integration/distributed-transactions.test.ts deleted file mode 100644 index 2a0cda2a..00000000 --- a/tests/transaction/integration/distributed-transactions.test.ts +++ /dev/null @@ -1,393 +0,0 @@ -/** - * Distributed + Transactions Integration Tests - * - * Verifies that transactions work correctly with distributed storage: - * - Remote storage adapters (S3, Azure, GCS) - * - Distributed coordination - * - Cache coherence - * - Read/write separation - */ - -import { describe, it, expect, beforeEach, afterEach } from 'vitest' -import { Brainy } from '../../../src/brainy.js' -import { NounType, VerbType } from '../../../src/types/graphTypes.js' -import { tmpdir } from 'os' -import { join } from 'path' -import { mkdirSync, rmSync } from 'fs' - -describe('Transactions + Distributed Storage Integration', () => { - let brain: Brainy - let testDir: string - - beforeEach(async () => { - testDir = join(tmpdir(), `brainy-distributed-test-${Date.now()}`) - mkdirSync(testDir, { recursive: true }) - - // Use filesystem as proxy for distributed storage - // (In production, this would be S3, Azure, GCS, etc.) - brain = new Brainy({ requireSubtype: false, - storage: { - type: 'filesystem', - path: testDir - } - }) - - await brain.init() - }) - - afterEach(async () => { - if (brain) { - await brain.shutdown() - } - if (testDir) { - rmSync(testDir, { recursive: true, force: true }) - } - }) - - describe('Remote Storage Adapters', () => { - it('should handle transactions with filesystem storage (proxy for remote)', async () => { - // Add entity (atomic operation) - const id = await brain.add({ - data: { name: 'Remote Entity', location: 'cloud' }, - type: NounType.Thing - }) - - expect(id).toBeTruthy() - - // Verify entity persisted to storage - const entity = await brain.get(id) - expect(entity).toBeTruthy() - expect(entity?.data.name).toBe('Remote Entity') - }) - - it('should rollback failed operations with remote storage', async () => { - // Add first entity (succeeds) - const id1 = await brain.add({ - data: { name: 'Entity 1' }, - type: NounType.Thing - }) - - // Attempt to add with invalid data (fails) - let failed = false - try { - await brain.add({ - data: null as any, - type: NounType.Thing - }) - } catch (e) { - failed = true - } - - expect(failed).toBe(true) - - // First entity should still exist - const entity1 = await brain.get(id1) - expect(entity1).toBeTruthy() - }) - - it('should handle update operations with remote storage atomically', async () => { - // Add entity - const id = await brain.add({ - data: { name: 'Original', version: 1 }, - type: NounType.Thing - }) - - // Update atomically - await brain.update({ - id, - data: { name: 'Updated', version: 2 } - }) - - // Verify update persisted - const entity = await brain.get(id) - expect(entity?.data.version).toBe(2) - }) - }) - - describe('Write Coordinator Atomicity', () => { - it('should ensure atomicity at write coordinator level', async () => { - // Simulate write coordinator scenario - // (Single Brainy instance coordinating writes) - - const entities: string[] = [] - - // Multiple atomic writes - for (let i = 0; i < 5; i++) { - const id = await brain.add({ - data: { name: `Entity ${i}`, index: i }, - type: NounType.Thing - }) - entities.push(id) - } - - // Create relationships (atomic) - for (let i = 0; i < entities.length - 1; i++) { - await brain.relate({ - from: entities[i], - to: entities[i + 1], - type: VerbType.RelatesTo - }) - } - - // Verify all operations succeeded - for (let i = 0; i < entities.length; i++) { - const entity = await brain.get(entities[i]) - expect(entity).toBeTruthy() - expect(entity?.data.index).toBe(i) - } - - // Verify relationships - for (let i = 0; i < entities.length - 1; i++) { - const relations = await brain.related({ from: entities[i] }) - expect(relations).toHaveLength(1) - } - }) - - it('should handle batch operations atomically on write coordinator', async () => { - const batchSize = 20 - const ids: string[] = [] - - // Batch add operations - for (let i = 0; i < batchSize; i++) { - const id = await brain.add({ - data: { name: `Batch Entity ${i}`, batch: true }, - type: NounType.Thing - }) - ids.push(id) - } - - // Verify all entities persisted - let count = 0 - for (const id of ids) { - const entity = await brain.get(id) - if (entity) count++ - } - - expect(count).toBe(batchSize) - }) - }) - - describe('Read-After-Write Consistency', () => { - it('should ensure read-after-write consistency', async () => { - // Write entity - const id = await brain.add({ - data: { name: 'RAW Test', timestamp: Date.now() }, - type: NounType.Thing - }) - - // Immediate read (should see the write) - const entity = await brain.get(id) - expect(entity).toBeTruthy() - expect(entity?.data.name).toBe('RAW Test') - }) - - it('should maintain consistency after update', async () => { - const id = await brain.add({ - data: { name: 'Original', counter: 0 }, - type: NounType.Thing - }) - - // Multiple updates - for (let i = 1; i <= 5; i++) { - await brain.update({ - id, - data: { name: `Updated ${i}`, counter: i } - }) - - // Read immediately after each update - const entity = await brain.get(id) - expect(entity?.data.counter).toBe(i) - } - }) - }) - - describe('Concurrent Write Handling', () => { - it('should handle sequential writes correctly', async () => { - const ids: string[] = [] - - // Sequential writes (simulating distributed writes to coordinator) - for (let i = 0; i < 10; i++) { - const id = await brain.add({ - data: { name: `Sequential ${i}`, order: i }, - type: NounType.Thing - }) - ids.push(id) - } - - // Verify all writes succeeded - for (let i = 0; i < ids.length; i++) { - const entity = await brain.get(ids[i]) - expect(entity?.data.order).toBe(i) - } - }) - - it('should handle interleaved operations atomically', async () => { - // Create entities - const id1 = await brain.add({ - data: { name: 'Entity A', value: 100 }, - type: NounType.Thing - }) - - const id2 = await brain.add({ - data: { name: 'Entity B', value: 200 }, - type: NounType.Thing - }) - - // Interleaved updates - await brain.update({ id: id1, data: { value: 150 } }) - await brain.update({ id: id2, data: { value: 250 } }) - await brain.update({ id: id1, data: { value: 175 } }) - - // Verify final state - const entity1 = await brain.get(id1) - const entity2 = await brain.get(id2) - - expect(entity1?.data.value).toBe(175) - expect(entity2?.data.value).toBe(250) - }) - }) - - describe('Delete Operations with Distributed Storage', () => { - it('should handle delete operations atomically', async () => { - // Create entity - const id = await brain.add({ - data: { name: 'To Delete', status: 'active' }, - type: NounType.Thing - }) - - // Verify exists - let entity = await brain.get(id) - expect(entity).toBeTruthy() - - // Delete atomically - await brain.remove(id) - - // Verify deleted - entity = await brain.get(id) - expect(entity).toBeNull() - }) - - it('should handle delete with relationships atomically', async () => { - // Create entities and relationships - const id1 = await brain.add({ - data: { name: 'Entity 1' }, - type: NounType.Thing - }) - - const id2 = await brain.add({ - data: { name: 'Entity 2' }, - type: NounType.Thing - }) - - await brain.relate({ - from: id1, - to: id2, - type: VerbType.RelatesTo - }) - - // Delete first entity (should delete relationships) - await brain.remove(id1) - - // Verify entity deleted - const entity1 = await brain.get(id1) - expect(entity1).toBeNull() - - // Verify relationships deleted - const relations = await brain.related({ from: id1 }) - expect(relations).toHaveLength(0) - - // Entity 2 should still exist - const entity2 = await brain.get(id2) - expect(entity2).toBeTruthy() - }) - }) - - describe('Storage Adapter Transparency', () => { - it('should work transparently with any storage adapter', async () => { - // This test verifies that transactions don't make assumptions - // about the underlying storage implementation - - // Add entity - const id = await brain.add({ - data: { name: 'Adapter Test', adapter: 'filesystem' }, - type: NounType.Thing - }) - - // Update entity - await brain.update({ - id, - data: { name: 'Updated via Adapter', adapter: 'filesystem' } - }) - - // Query entity - const entity = await brain.get(id) - expect(entity).toBeTruthy() - expect(entity?.data.name).toBe('Updated via Adapter') - - // Delete entity - await brain.remove(id) - const deletedEntity = await brain.get(id) - expect(deletedEntity).toBeNull() - }) - - it('should maintain atomicity regardless of storage latency', async () => { - // Simulate scenario with storage latency - // (In distributed setup, network latency is a factor) - - const startTime = Date.now() - - // Operations that might have latency - const id1 = await brain.add({ - data: { name: 'Latency Test 1' }, - type: NounType.Thing - }) - - const id2 = await brain.add({ - data: { name: 'Latency Test 2' }, - type: NounType.Thing - }) - - await brain.relate({ - from: id1, - to: id2, - type: VerbType.RelatesTo - }) - - const endTime = Date.now() - const duration = endTime - startTime - - // Verify all operations succeeded (regardless of latency) - const entity1 = await brain.get(id1) - const entity2 = await brain.get(id2) - const relations = await brain.related({ from: id1 }) - - expect(entity1).toBeTruthy() - expect(entity2).toBeTruthy() - expect(relations).toHaveLength(1) - - // Should complete in reasonable time (even with storage latency) - expect(duration).toBeLessThan(5000) - }) - }) - - describe('Transaction Statistics with Distributed Storage', () => { - it('should track transaction statistics accurately', async () => { - // Get transaction manager stats - const stats = (brain as any).transactionManager?.getStats() - - if (stats) { - const initialTotal = stats.totalTransactions - - // Perform operations - await brain.add({ - data: { name: 'Stats Test' }, - type: NounType.Thing - }) - - // Check stats updated - const updatedStats = (brain as any).transactionManager?.getStats() - expect(updatedStats.totalTransactions).toBeGreaterThan(initialTotal) - } - }) - }) -}) diff --git a/tests/transaction/integration/sharding-transactions.test.ts b/tests/transaction/integration/sharding-transactions.test.ts deleted file mode 100644 index 60281c05..00000000 --- a/tests/transaction/integration/sharding-transactions.test.ts +++ /dev/null @@ -1,295 +0,0 @@ -/** - * Sharding + Transactions Integration Tests - * - * Verifies that transactions work correctly with sharded storage: - * - Cross-shard atomicity - * - Shard-aware routing - * - Rollback across shards - * - UUID-based shard distribution - */ - -import { describe, it, expect, beforeEach, afterEach } from 'vitest' -import { Brainy } from '../../../src/brainy.js' -import { NounType, VerbType } from '../../../src/types/graphTypes.js' -import { tmpdir } from 'os' -import { join } from 'path' -import { mkdirSync, rmSync } from 'fs' - -describe('Transactions + Sharding Integration', () => { - let brain: Brainy - let testDir: string - - beforeEach(async () => { - testDir = join(tmpdir(), `brainy-shard-test-${Date.now()}`) - mkdirSync(testDir, { recursive: true }) - - brain = new Brainy({ requireSubtype: false, - storage: { - type: 'filesystem', - path: testDir - } - }) - - await brain.init() - }) - - afterEach(async () => { - if (brain) { - await brain.shutdown() - } - if (testDir) { - rmSync(testDir, { recursive: true, force: true }) - } - }) - - describe('Cross-Shard Operations', () => { - it('should handle atomic operations across multiple shards', async () => { - // Create entities with different UUID prefixes (different shards) - const id1 = 'aaa00000-1111-4111-8111-111111111111' // Shard: aaa - const id2 = 'bbb00000-2222-4222-8222-222222222222' // Shard: bbb - - // Add entities to different shards (atomic) - await brain.add({ - id: id1, - data: { name: 'Entity in Shard A', shard: 'aaa' }, - type: NounType.Thing - }) - - await brain.add({ - id: id2, - data: { name: 'Entity in Shard B', shard: 'bbb' }, - type: NounType.Thing - }) - - // Create relationship across shards (atomic) - const relationId = await brain.relate({ - from: id1, - to: id2, - type: VerbType.RelatesTo - }) - - // Verify all entities exist - const entity1 = await brain.get(id1) - const entity2 = await brain.get(id2) - const relations = await brain.related({ from: id1 }) - - expect(entity1).toBeTruthy() - expect(entity2).toBeTruthy() - expect(relations).toHaveLength(1) - expect(relations[0].targetId).toBe(id2) - }) - - it('should rollback operations across multiple shards', async () => { - const id1 = 'ccc00000-1111-4111-8111-111111111111' // Shard: ccc - const id2 = 'ddd00000-2222-4222-8222-222222222222' // Shard: ddd - - // Add first entity (succeeds) - await brain.add({ - id: id1, - data: { name: 'Entity in Shard C' }, - type: NounType.Thing - }) - - // Attempt to add second entity with invalid data (fails) - let failed = false - try { - await brain.add({ - id: id2, - data: null as any, // Invalid - type: NounType.Thing - }) - } catch (e) { - failed = true - } - - expect(failed).toBe(true) - - // First entity should still exist (in shard C) - const entity1 = await brain.get(id1) - expect(entity1).toBeTruthy() - - // Second entity should not exist (rollback in shard D) - const entity2 = await brain.get(id2) - expect(entity2).toBeNull() - }) - - it('should handle updates across shards atomically', async () => { - const id1 = 'eee00000-1111-4111-8111-111111111111' - const id2 = 'fff00000-2222-4222-8222-222222222222' - - // Add entities - await brain.add({ - id: id1, - data: { name: 'Original E', version: 1 }, - type: NounType.Thing - }) - - await brain.add({ - id: id2, - data: { name: 'Original F', version: 1 }, - type: NounType.Thing - }) - - // Update both entities (different shards) - await brain.update({ - id: id1, - data: { name: 'Updated E', version: 2 } - }) - - await brain.update({ - id: id2, - data: { name: 'Updated F', version: 2 } - }) - - // Verify updates in both shards - const entity1 = await brain.get(id1) - const entity2 = await brain.get(id2) - - expect(entity1?.data.version).toBe(2) - expect(entity2?.data.version).toBe(2) - }) - }) - - describe('Shard Distribution', () => { - it('should distribute entities across shards based on UUID', async () => { - // Create entities with various UUID prefixes - const ids = [ - 'aaa00000-1111-4111-8111-111111111111', - 'bbb00000-2222-4222-8222-222222222222', - 'ccc00000-3333-4333-8333-333333333333', - 'ddd00000-4444-4444-8444-444444444444' - ] - - // Add entities to different shards - for (let i = 0; i < ids.length; i++) { - await brain.add({ - id: ids[i], - data: { name: `Entity ${i}`, index: i }, - type: NounType.Thing - }) - } - - // Verify all entities can be retrieved (regardless of shard) - for (let i = 0; i < ids.length; i++) { - const entity = await brain.get(ids[i]) - expect(entity).toBeTruthy() - expect(entity?.data.index).toBe(i) - } - }) - - it('should handle relationships between different shard combinations', async () => { - const entities = [ - { id: 'aaa00000-1111-4111-8111-111111111111', name: 'A' }, - { id: 'bbb00000-2222-4222-8222-222222222222', name: 'B' }, - { id: 'ccc00000-3333-4333-8333-333333333333', name: 'C' } - ] - - // Add all entities - for (const e of entities) { - await brain.add({ - id: e.id, - data: { name: e.name }, - type: NounType.Thing - }) - } - - // Create relationships across all shards - await brain.relate({ - from: entities[0].id, - to: entities[1].id, - type: VerbType.RelatesTo - }) - - await brain.relate({ - from: entities[1].id, - to: entities[2].id, - type: VerbType.RelatesTo - }) - - await brain.relate({ - from: entities[2].id, - to: entities[0].id, - type: VerbType.RelatesTo - }) - - // Verify all relationships exist - const relations0 = await brain.related({ from: entities[0].id }) - const relations1 = await brain.related({ from: entities[1].id }) - const relations2 = await brain.related({ from: entities[2].id }) - - expect(relations0).toHaveLength(1) - expect(relations1).toHaveLength(1) - expect(relations2).toHaveLength(1) - }) - }) - - describe('Delete Operations Across Shards', () => { - it('should delete entities and relationships across shards atomically', async () => { - const id1 = 'ggg00000-1111-4111-8111-111111111111' - const id2 = 'hhh00000-2222-4222-8222-222222222222' - - // Add entities in different shards - await brain.add({ - id: id1, - data: { name: 'Entity G' }, - type: NounType.Thing - }) - - await brain.add({ - id: id2, - data: { name: 'Entity H' }, - type: NounType.Thing - }) - - // Create relationship - await brain.relate({ - from: id1, - to: id2, - type: VerbType.RelatesTo - }) - - // Delete first entity (should also delete relationship) - await brain.remove(id1) - - // Verify entity deleted from shard G - const entity1 = await brain.get(id1) - expect(entity1).toBeNull() - - // Entity H should still exist in shard H - const entity2 = await brain.get(id2) - expect(entity2).toBeTruthy() - - // Relationship should be deleted - const relations = await brain.related({ from: id1 }) - expect(relations).toHaveLength(0) - }) - }) - - describe('Batch Operations Across Shards', () => { - it('should handle batch adds across multiple shards atomically', async () => { - const entities = [ - { id: 'shard1-00-1111-4111-8111-111111111111', data: { name: 'S1-E1' } }, - { id: 'shard2-00-2222-4222-8222-222222222222', data: { name: 'S2-E1' } }, - { id: 'shard3-00-3333-4333-8333-333333333333', data: { name: 'S3-E1' } }, - { id: 'shard1-00-4444-4444-8444-444444444444', data: { name: 'S1-E2' } }, - { id: 'shard2-00-5555-4555-8555-555555555555', data: { name: 'S2-E2' } } - ] - - // Add all entities (distributed across shards) - for (const e of entities) { - await brain.add({ - id: e.id, - data: e.data, - type: NounType.Thing - }) - } - - // Verify all entities exist in their respective shards - for (const e of entities) { - const entity = await brain.get(e.id) - expect(entity).toBeTruthy() - expect(entity?.data.name).toBe(e.data.name) - } - }) - }) -})