feat: add distributed architecture with sharding and coordination
- Wire up distributed components (Coordinator, ShardManager, CacheSync) - Implement automatic sharding for S3 storage (256 shards) - Add read/write separation for operational modes - Zero-config automatic detection for distributed mode - Add mutex implementation for thread safety - Fix metadata filtering in find operations - Fix neural API vector similarity calculations - Improve batch operations performance - Add Bluesky distributed setup example BREAKING CHANGE: None - backward compatible
This commit is contained in:
parent
8aafc769a3
commit
ed64c266ec
30 changed files with 2084 additions and 439 deletions
|
|
@ -337,13 +337,32 @@ export interface BrainyConfig {
|
|||
|
||||
// Augmentations
|
||||
augmentations?: Record<string, any>
|
||||
|
||||
|
||||
// Distributed configuration
|
||||
distributed?: {
|
||||
enabled: boolean
|
||||
nodeId?: string
|
||||
nodes?: string[] // Other nodes in cluster
|
||||
coordinatorUrl?: string // Coordinator endpoint
|
||||
shardCount?: number // Number of shards (default: 64)
|
||||
replicationFactor?: number // Number of replicas (default: 3)
|
||||
consensus?: 'raft' | 'none' // Consensus mechanism
|
||||
transport?: 'tcp' | 'http' | 'udp'
|
||||
}
|
||||
|
||||
// Advanced options
|
||||
warmup?: boolean // Warm up on init
|
||||
realtime?: boolean // Enable real-time updates
|
||||
multiTenancy?: boolean // Enable service isolation
|
||||
telemetry?: boolean // Send anonymous usage stats
|
||||
|
||||
// Performance tuning options for production
|
||||
disableAutoRebuild?: boolean // Disable automatic index rebuilding on init
|
||||
disableMetrics?: boolean // Completely disable metrics collection
|
||||
disableAutoOptimize?: boolean // Disable automatic index optimization
|
||||
batchWrites?: boolean // Enable write batching for better performance
|
||||
maxConcurrentOperations?: number // Limit concurrent file operations
|
||||
|
||||
// Logging configuration
|
||||
verbose?: boolean // Enable verbose logging
|
||||
silent?: boolean // Suppress all logging output
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue