refactor(8.0)!: remove distributed clustering subsystem — inert/orphaned, scale is single-process + native provider
The distributed-clustering subsystem never ran in production: it was inert, orphaned dead code (faked consensus, stub replication, no live wiring, and it did not interoperate with the 8.0 Db API). Brainy 8.0 is a single-process library. Scale is single-process + the optional native provider (@soulcraft/cortex, on-disk DiskANN to 10B+ vectors) + per-tenant pools + horizontal read scaling (many reader processes, one writer). Removed: - src/distributed/ entirely (coordinator, shardManager, cacheSync, readWriteSeparation, queryPlanner, healthMonitor, configManager, hashPartitioner, shardMigration, domainDetector, storageDiscovery, http/network transports). ReaderMode/HybridMode relocated to src/storage/operationalModes.ts (slimmed to the live surface). - src/types/distributedTypes.ts; config.distributed field + JSDoc; coreTypes distributedConfig; memoryStorage distributedConfig persistence. - DistributedRole enum + src/config/distributedPresets.ts and the orphaned src/config/extensibleConfig.ts (config/augmentation registry built on removed cloud adapters + distributed presets), plus their src/index.ts re-exports. - 13 BRAINY_* cluster env vars; the storage setDistributedComponents hook; enableDistributedSearch (dead config flag); the metadata partition field; the distributed_ reserved key prefix. - Orphaned src/storage/readOnlyOptimizations.ts (zero importers). - Tests targeting the subsystem: distributed-demo, distributed-cluster helper, distributed-transactions, sharding-transactions. - Docs: EXTENDING_STORAGE.md (deleted); scrubbed distributed/cluster/Raft/ shard-manager/multi-node prose from v3-features, enterprise-for-everyone, augmentations-actual, complete-feature-list, vfs/README, vfs/ROADMAP, vfs/VFS_CORE, capacity-planning, transactions, MIGRATION-V3-TO-V4, storage-architecture; reframed scale prose to the 8.0 model. Kept: src/storage/sharding.ts (local-disk 256-bucket directory sharding via getShardIdFromUuid — used live by baseStorage, unrelated to clustering); the multi-process mode: 'reader' | 'writer' roles; semantic/HNSW clustering. RELEASES.md: added a removed-surfaces row documenting the cut and the 8.0 scale model.
This commit is contained in:
parent
f8e0079d3f
commit
00d3203d68
51 changed files with 153 additions and 9889 deletions
|
|
@ -1206,25 +1206,6 @@ export interface BrainyConfig {
|
|||
}
|
||||
| StorageAdapter
|
||||
|
||||
/**
|
||||
* Distributed-cluster configuration. **Explicit opt-in only** — Brainy
|
||||
* never auto-enables cluster mode from environment heuristics (a single
|
||||
* production process is the 90th-percentile deployment). Set
|
||||
* `enabled: true` (or the `BRAINY_DISTRIBUTED=true` env var) to activate;
|
||||
* the remaining fields then default sensibly (64 shards, 3 replicas,
|
||||
* raft consensus, http transport, hostname-derived nodeId).
|
||||
*/
|
||||
distributed?: {
|
||||
enabled: boolean
|
||||
nodeId?: string
|
||||
nodes?: string[] // Other nodes in cluster
|
||||
coordinatorUrl?: string // Coordinator endpoint
|
||||
shardCount?: number // Number of shards (default: 64)
|
||||
replicationFactor?: number // Number of replicas (default: 3)
|
||||
consensus?: 'raft' | 'none' // Consensus mechanism
|
||||
transport?: 'tcp' | 'http' | 'udp'
|
||||
}
|
||||
|
||||
/**
|
||||
* Disable the automatic index rebuild check during `init()`. By default
|
||||
* Brainy auto-decides from dataset size: small datasets rebuild missing
|
||||
|
|
|
|||
|
|
@ -1,236 +0,0 @@
|
|||
/**
|
||||
* Distributed types for Brainy
|
||||
* Defines types for distributed operations across multiple instances
|
||||
*/
|
||||
|
||||
export type InstanceRole = 'reader' | 'writer' | 'hybrid'
|
||||
|
||||
export type PartitionStrategy = 'hash' | 'semantic' | 'manual'
|
||||
|
||||
export interface DistributedConfig {
|
||||
/**
|
||||
* Enable distributed mode
|
||||
* Can be boolean for auto-detection or specific configuration
|
||||
*/
|
||||
enabled?: boolean | 'auto'
|
||||
|
||||
/**
|
||||
* Role of this instance in the distributed system
|
||||
* - reader: Read-only access, optimized for queries
|
||||
* - writer: Write-focused, handles data ingestion
|
||||
* - hybrid: Can both read and write (requires coordination)
|
||||
*/
|
||||
role?: InstanceRole
|
||||
|
||||
/**
|
||||
* Unique identifier for this instance
|
||||
* Auto-generated if not provided
|
||||
*/
|
||||
instanceId?: string
|
||||
|
||||
/**
|
||||
* Path to shared configuration file in S3
|
||||
* Default: '_brainy/config.json'
|
||||
*/
|
||||
configPath?: string
|
||||
|
||||
/**
|
||||
* Heartbeat interval in milliseconds
|
||||
* Default: 30000 (30 seconds)
|
||||
*/
|
||||
heartbeatInterval?: number
|
||||
|
||||
/**
|
||||
* Config check interval in milliseconds
|
||||
* Default: 10000 (10 seconds)
|
||||
*/
|
||||
configCheckInterval?: number
|
||||
|
||||
/**
|
||||
* Instance timeout in milliseconds
|
||||
* Instances not seen for this duration are considered dead
|
||||
* Default: 60000 (60 seconds)
|
||||
*/
|
||||
instanceTimeout?: number
|
||||
}
|
||||
|
||||
export interface SharedConfig {
|
||||
/**
|
||||
* Configuration version for compatibility checking
|
||||
*/
|
||||
version: number
|
||||
|
||||
/**
|
||||
* Last update timestamp
|
||||
*/
|
||||
updated: string
|
||||
|
||||
/**
|
||||
* Global settings that must be consistent across all instances
|
||||
*/
|
||||
settings: {
|
||||
/**
|
||||
* Partitioning strategy
|
||||
* - hash: Deterministic hash-based partitioning (recommended for multi-writer)
|
||||
* - semantic: Group similar vectors (single writer only)
|
||||
* - manual: Explicit partition assignment
|
||||
*/
|
||||
partitionStrategy: PartitionStrategy
|
||||
|
||||
/**
|
||||
* Number of partitions (for hash strategy)
|
||||
*/
|
||||
partitionCount: number
|
||||
|
||||
/**
|
||||
* Embedding model name (must be consistent)
|
||||
*/
|
||||
embeddingModel: string
|
||||
|
||||
/**
|
||||
* Vector dimensions
|
||||
*/
|
||||
dimensions: number
|
||||
|
||||
/**
|
||||
* Distance metric
|
||||
*/
|
||||
distanceMetric: 'cosine' | 'euclidean' | 'manhattan'
|
||||
|
||||
/**
|
||||
* HNSW parameters (must be consistent for index compatibility)
|
||||
*/
|
||||
hnswParams?: {
|
||||
M: number
|
||||
efConstruction: number
|
||||
maxElements?: number
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Active instances in the distributed system
|
||||
*/
|
||||
instances: {
|
||||
[instanceId: string]: InstanceInfo
|
||||
}
|
||||
|
||||
/**
|
||||
* Partition assignments (for manual strategy)
|
||||
*/
|
||||
partitionAssignments?: {
|
||||
[instanceId: string]: string[]
|
||||
}
|
||||
}
|
||||
|
||||
export interface InstanceInfo {
|
||||
/**
|
||||
* Instance role
|
||||
*/
|
||||
role: InstanceRole
|
||||
|
||||
/**
|
||||
* Instance status
|
||||
*/
|
||||
status: 'active' | 'inactive' | 'unhealthy'
|
||||
|
||||
/**
|
||||
* Last heartbeat timestamp
|
||||
*/
|
||||
lastHeartbeat: string
|
||||
|
||||
/**
|
||||
* Optional endpoint for health checks
|
||||
*/
|
||||
endpoint?: string
|
||||
|
||||
/**
|
||||
* Instance metrics
|
||||
*/
|
||||
metrics?: {
|
||||
vectorCount?: number
|
||||
cacheHitRate?: number
|
||||
memoryUsage?: number
|
||||
cpuUsage?: number
|
||||
}
|
||||
|
||||
/**
|
||||
* Assigned partitions (for manual assignment)
|
||||
*/
|
||||
assignedPartitions?: string[]
|
||||
|
||||
/**
|
||||
* Preferred partitions (for affinity)
|
||||
*/
|
||||
preferredPartitions?: number[]
|
||||
}
|
||||
|
||||
export interface DomainMetadata {
|
||||
/**
|
||||
* Domain identifier for logical data separation
|
||||
*/
|
||||
domain?: string
|
||||
|
||||
/**
|
||||
* Additional domain-specific metadata
|
||||
*/
|
||||
domainMetadata?: Record<string, any>
|
||||
}
|
||||
|
||||
export interface CacheStrategy {
|
||||
/**
|
||||
* Percentage of memory allocated to hot cache (0-1)
|
||||
*/
|
||||
hotCacheRatio: number
|
||||
|
||||
/**
|
||||
* Enable aggressive prefetching
|
||||
*/
|
||||
prefetchAggressive?: boolean
|
||||
|
||||
/**
|
||||
* Cache time-to-live in milliseconds
|
||||
*/
|
||||
ttl?: number
|
||||
|
||||
/**
|
||||
* Enable compression to trade CPU for memory
|
||||
*/
|
||||
compressionEnabled?: boolean
|
||||
|
||||
/**
|
||||
* Write buffer size for batching
|
||||
*/
|
||||
writeBufferSize?: number
|
||||
|
||||
/**
|
||||
* Enable write batching
|
||||
*/
|
||||
batchWrites?: boolean
|
||||
|
||||
/**
|
||||
* Adaptive caching based on workload
|
||||
*/
|
||||
adaptive?: boolean
|
||||
}
|
||||
|
||||
export interface OperationalMode {
|
||||
/**
|
||||
* Whether this mode can read
|
||||
*/
|
||||
canRead: boolean
|
||||
|
||||
/**
|
||||
* Whether this mode can write
|
||||
*/
|
||||
canWrite: boolean
|
||||
|
||||
/**
|
||||
* Whether this mode can delete
|
||||
*/
|
||||
canDelete: boolean
|
||||
|
||||
/**
|
||||
* Cache strategy for this mode
|
||||
*/
|
||||
cacheStrategy: CacheStrategy
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue