soulcraft/brainy
1
0
Fork 0
Code Issues 1 Pull requests Projects Releases 335 Packages Wiki Activity Actions

refactor(8.0)!: remove distributed clustering subsystem — inert/orphaned, scale is single-process + native provider

Browse source 
The distributed-clustering subsystem never ran in production: it was inert,
orphaned dead code (faked consensus, stub replication, no live wiring, and it
did not interoperate with the 8.0 Db API). Brainy 8.0 is a single-process
library. Scale is single-process + the optional native provider
(@soulcraft/cortex, on-disk DiskANN to 10B+ vectors) + per-tenant pools +
horizontal read scaling (many reader processes, one writer).

Removed:
- src/distributed/ entirely (coordinator, shardManager, cacheSync,
  readWriteSeparation, queryPlanner, healthMonitor, configManager,
  hashPartitioner, shardMigration, domainDetector, storageDiscovery, http/network
  transports). ReaderMode/HybridMode relocated to src/storage/operationalModes.ts
  (slimmed to the live surface).
- src/types/distributedTypes.ts; config.distributed field + JSDoc;
  coreTypes distributedConfig; memoryStorage distributedConfig persistence.
- DistributedRole enum + src/config/distributedPresets.ts and the orphaned
  src/config/extensibleConfig.ts (config/augmentation registry built on removed
  cloud adapters + distributed presets), plus their src/index.ts re-exports.
- 13 BRAINY_* cluster env vars; the storage setDistributedComponents hook;
  enableDistributedSearch (dead config flag); the metadata partition field;
  the distributed_ reserved key prefix.
- Orphaned src/storage/readOnlyOptimizations.ts (zero importers).
- Tests targeting the subsystem: distributed-demo, distributed-cluster helper,
  distributed-transactions, sharding-transactions.
- Docs: EXTENDING_STORAGE.md (deleted); scrubbed distributed/cluster/Raft/
  shard-manager/multi-node prose from v3-features, enterprise-for-everyone,
  augmentations-actual, complete-feature-list, vfs/README, vfs/ROADMAP,
  vfs/VFS_CORE, capacity-planning, transactions, MIGRATION-V3-TO-V4,
  storage-architecture; reframed scale prose to the 8.0 model.

Kept: src/storage/sharding.ts (local-disk 256-bucket directory sharding via
getShardIdFromUuid — used live by baseStorage, unrelated to clustering);
the multi-process mode: 'reader' | 'writer' roles; semantic/HNSW clustering.

RELEASES.md: added a removed-surfaces row documenting the cut and the 8.0
scale model.
This commit is contained in:
David Snelling 2026-06-15 10:37:39 -07:00
parent f8e0079d3f
commit 00d3203d68
51 changed files with 153 additions and 9889 deletions
Download patch file Download diff file Expand all files Collapse all files

167
src/brainy.ts
View file

@ -72,14 +72,10 @@ import {
DeleteVerbMetadataOperation
} from './transaction/operations/index.js'
import {
DistributedCoordinator,
ShardManager,
CacheSync,
ReadWriteSeparation,
BaseOperationalMode,
ReaderMode,
HybridMode
} from './distributed/index.js'
} from './storage/operationalModes.js'
import {
Entity,
Relation,
@ -199,7 +195,6 @@ interface StorageBackgroundInitHooks {
type ResolvedBrainyConfig = Required<
Omit<
BrainyConfig,
| 'distributed'
| 'maxQueryLimit'
| 'reservedQueryMemory'
| 'vector'
@ -211,7 +206,6 @@ type ResolvedBrainyConfig = Required<
> &
Pick<
BrainyConfig,
| 'distributed'
| 'maxQueryLimit'
| 'reservedQueryMemory'
| 'vector'
@ -329,12 +323,6 @@ export class Brainy<T = any> implements BrainyInterface<T> {
/** Cap for {@link Brainy.verbIntWarmCache} (~100k entries ≈ a few MB). */
private static readonly VERB_INT_WARM_CACHE_MAX = 100_000
// Distributed components (optional)
private coordinator?: DistributedCoordinator
private shardManager?: ShardManager
private cacheSync?: CacheSync
private readWriteSeparation?: ReadWriteSeparation
// Silent mode state
private originalConsole?: {
log: typeof console.log
@ -414,9 +402,9 @@ export class Brainy<T = any> implements BrainyInterface<T> {
this.config = this.normalizeConfig(config)
// Multi-process mode — default 'writer' uses HybridMode (read + write),
// 'reader' uses ReaderMode (read-only, all mutations throw).
// `WriterMode` from operationalModes.ts blocks reads, which is too strict
// for a Brainy instance — a writer needs to read its own data.
// 'reader' uses ReaderMode (read-only, all mutations throw). A writer needs
// to read its own data, so the writer role is read-write rather than
// write-only.
this.operationalMode = this.config.mode === 'reader'
? new ReaderMode()
: new HybridMode()
@ -435,11 +423,6 @@ export class Brainy<T = any> implements BrainyInterface<T> {
this.embedder = this.setupEmbedder()
this.transactionManager = new TransactionManager()
// Setup distributed components if enabled
if (this.config.distributed?.enabled) {
this.setupDistributedComponents()
}
// Initialize ready Promise
// This allows consumers to await brain.ready before using the database
this._readyPromise = new Promise<void>((resolve, reject) => {
@ -846,9 +829,6 @@ export class Brainy<T = any> implements BrainyInterface<T> {
// Check for pending data migrations
await this.checkMigrations()
// Connect distributed components to storage
await this.connectDistributedStorage()
// Register shutdown hooks for graceful count flushing (once globally)
if (!Brainy.shutdownHooksRegisteredGlobally) {
this.registerShutdownHooks()
@ -10511,13 +10491,8 @@ export class Brainy<T = any> implements BrainyInterface<T> {
)
}
// Distributed mode is explicit opt-in only (config or BRAINY_DISTRIBUTED
// env var) — never inferred from deployment heuristics.
const distributedConfig = this.resolveDistributedConfig(config?.distributed)
return {
storage: config?.storage || { type: 'auto' },
distributed: distributedConfig,
verbose: config?.verbose ?? false,
silent: config?.silent ?? false,
// false = auto-decide based on dataset size (inline vs lazy rebuild)
@ -11144,140 +11119,6 @@ export class Brainy<T = any> implements BrainyInterface<T> {
// operation (ensureInitialized() throws once this is set).
this.closed = true
}
/**
* Intelligently auto-detect distributed configuration
* Zero-config: Automatically determines best distributed settings
*/
/**
* Resolve distributed-cluster configuration. **Explicit opt-in only**:
* either `config.distributed.enabled === true` or the
* `BRAINY_DISTRIBUTED=true` environment variable. Once opted in, the
* remaining fields default sensibly (64 shards, 3 replicas, raft, http,
* hostname-derived nodeId).
*
* Earlier versions auto-enabled cluster mode from deployment heuristics
* (`NODE_ENV=production`, a Kubernetes service host, `CLUSTER_SIZE`).
* That violated zero-config safety: a single-process production
* deployment — the overwhelmingly common case — would silently start a
* coordinator, bind a port, and run raft against itself. Deployment
* environment does not imply a multi-node Brainy cluster, so the
* heuristics are gone.
*/
private resolveDistributedConfig(config?: BrainyConfig['distributed']): BrainyConfig['distributed'] {
// Env-var opt-in for deployments that can't touch code (the ONLY
// environment signal honoured — it names brainy explicitly).
if (!config && process.env.BRAINY_DISTRIBUTED === 'true') {
return {
enabled: true,
nodeId: process.env.HOSTNAME || process.env.NODE_ID || `node-${Date.now()}`,
nodes: process.env.BRAINY_NODES?.split(',') || [],
coordinatorUrl: process.env.BRAINY_COORDINATOR || undefined,
shardCount: parseInt(process.env.BRAINY_SHARDS || '64'),
replicationFactor: parseInt(process.env.BRAINY_REPLICAS || '3'),
consensus: (process.env.BRAINY_CONSENSUS as 'raft' | 'none' | undefined) || 'raft',
transport: (process.env.BRAINY_TRANSPORT as 'tcp' | 'http' | 'udp' | undefined) || 'http'
}
}
// Explicit config: fill the per-field defaults.
return config ? {
...config,
nodeId: config.nodeId || process.env.HOSTNAME || `node-${Date.now()}`,
shardCount: config.shardCount || 64,
replicationFactor: config.replicationFactor || 3,
consensus: config.consensus || 'raft',
transport: config.transport || 'http'
} : undefined
}
/**
* Setup distributed components with zero-config intelligence
*/
private setupDistributedComponents(): void {
const distConfig = this.config.distributed
if (!distConfig?.enabled) return
console.log('🌍 Initializing distributed mode:', {
nodeId: distConfig.nodeId,
shards: distConfig.shardCount,
replicas: distConfig.replicationFactor
})
// Initialize coordinator for consensus
this.coordinator = new DistributedCoordinator({
nodeId: distConfig.nodeId,
address: distConfig.coordinatorUrl?.split(':')[0] || 'localhost',
port: parseInt(distConfig.coordinatorUrl?.split(':')[1] || '8080'),
nodes: distConfig.nodes
})
// Start the coordinator to establish leadership
this.coordinator.start().catch(err => {
console.warn('Coordinator start failed (will retry on init):', err.message)
})
// Initialize shard manager for data distribution
this.shardManager = new ShardManager({
shardCount: distConfig.shardCount,
replicationFactor: distConfig.replicationFactor,
virtualNodes: 150, // Optimal for consistent distribution
autoRebalance: true
})
// Initialize cache synchronization
this.cacheSync = new CacheSync({
nodeId: distConfig.nodeId!,
syncInterval: 1000
})
// Initialize read/write separation if we have replicas
// Note: Will be properly initialized after coordinator starts
if (distConfig.replicationFactor && distConfig.replicationFactor > 1) {
// Defer creation until coordinator is ready
setTimeout(() => {
this.readWriteSeparation = new ReadWriteSeparation(
{
nodeId: distConfig.nodeId!,
consistencyLevel: 'eventual',
role: 'replica', // Start as replica, will promote if leader
syncInterval: 5000
},
this.coordinator!,
this.shardManager!,
this.cacheSync!
)
}, 100)
}
}
/**
* Pass distributed components to storage adapter
*/
private async connectDistributedStorage(): Promise<void> {
if (!this.config.distributed?.enabled) return
// Check if storage supports distributed operations. No in-repo adapter
// implements this hook — it exists for external/plugin adapters, so the
// `in` guard above is the contract and this cast just names its shape.
if ('setDistributedComponents' in this.storage) {
(this.storage as BaseStorage & {
setDistributedComponents(components: {
coordinator?: DistributedCoordinator
shardManager?: ShardManager
cacheSync?: CacheSync
readWriteSeparation?: ReadWriteSeparation
}): void
}).setDistributedComponents({
coordinator: this.coordinator,
shardManager: this.shardManager,
cacheSync: this.cacheSync,
readWriteSeparation: this.readWriteSeparation
})
console.log('✅ Distributed storage connected')
}
}
}
/**

357
src/config/distributedPresets.ts
View file

@ -1,357 +0,0 @@
/**
* Extended Distributed Configuration Presets
* Common patterns for distributed and multi-service architectures
* All strongly typed with enums for compile-time safety
*/
/**
* Strongly typed enum for preset names
*/
export enum PresetName {
// Basic presets
PRODUCTION = 'production',
DEVELOPMENT = 'development',
MINIMAL = 'minimal',
ZERO = 'zero',
// Distributed presets
WRITER = 'writer',
READER = 'reader',
// Service-specific presets
INGESTION_SERVICE = 'ingestion-service',
SEARCH_API = 'search-api',
ANALYTICS_SERVICE = 'analytics-service',
EDGE_CACHE = 'edge-cache',
BATCH_PROCESSOR = 'batch-processor',
STREAMING_SERVICE = 'streaming-service',
ML_TRAINING = 'ml-training',
SIDECAR = 'sidecar'
}
/**
* Preset categories for organization
*/
export enum PresetCategory {
BASIC = 'basic',
DISTRIBUTED = 'distributed',
SERVICE = 'service'
}
/**
* Model precision options
*/
export enum ModelPrecision {
FP32 = 'fp32',
Q8 = 'q8',
AUTO = 'auto',
FAST = 'fast',
SMALL = 'small'
}
/**
* Storage options
*/
export enum StorageOption {
AUTO = 'auto',
MEMORY = 'memory',
DISK = 'disk',
CLOUD = 'cloud'
}
/**
* Feature set options
*/
export enum FeatureSet {
MINIMAL = 'minimal',
DEFAULT = 'default',
FULL = 'full',
CUSTOM = 'custom' // For custom feature arrays
}
/**
* Distributed role options
*/
export enum DistributedRole {
WRITER = 'writer',
READER = 'reader',
HYBRID = 'hybrid'
}
/**
* Preset configuration interface
*/
export interface PresetConfig {
storage: StorageOption
model: ModelPrecision
features: FeatureSet | string[]
distributed: boolean
role?: DistributedRole
readOnly?: boolean
writeOnly?: boolean
allowDirectReads?: boolean
lazyLoadInReadOnlyMode?: boolean
cache?: {
hotCacheMaxSize?: number
autoTune?: boolean
batchSize?: number
}
verbose?: boolean
description: string
category: PresetCategory
}
/**
* Strongly typed preset configurations
*/
export const PRESET_CONFIGS: Readonly<Record<PresetName, PresetConfig>> = {
// Basic presets
[PresetName.PRODUCTION]: {
storage: StorageOption.DISK,
model: ModelPrecision.AUTO,
features: FeatureSet.DEFAULT,
distributed: false,
verbose: false,
description: 'Optimized for production use',
category: PresetCategory.BASIC
},
[PresetName.DEVELOPMENT]: {
storage: StorageOption.MEMORY,
model: ModelPrecision.FP32,
features: FeatureSet.FULL,
distributed: false,
verbose: true,
description: 'Optimized for development with verbose logging',
category: PresetCategory.BASIC
},
[PresetName.MINIMAL]: {
storage: StorageOption.MEMORY,
model: ModelPrecision.Q8,
features: FeatureSet.MINIMAL,
distributed: false,
verbose: false,
description: 'Minimal footprint configuration',
category: PresetCategory.BASIC
},
[PresetName.ZERO]: {
storage: StorageOption.AUTO,
model: ModelPrecision.AUTO,
features: FeatureSet.DEFAULT,
distributed: false,
verbose: false,
description: 'True zero configuration with auto-detection',
category: PresetCategory.BASIC
},
// Distributed basic presets
[PresetName.WRITER]: {
storage: StorageOption.AUTO,
model: ModelPrecision.AUTO,
features: FeatureSet.MINIMAL,
distributed: true,
role: DistributedRole.WRITER,
writeOnly: true,
allowDirectReads: true,
verbose: false,
description: 'Write-only instance for distributed setups',
category: PresetCategory.DISTRIBUTED
},
[PresetName.READER]: {
storage: StorageOption.AUTO,
model: ModelPrecision.AUTO,
features: FeatureSet.DEFAULT,
distributed: true,
role: DistributedRole.READER,
readOnly: true,
lazyLoadInReadOnlyMode: true,
verbose: false,
description: 'Read-only instance for distributed setups',
category: PresetCategory.DISTRIBUTED
},
// Service-specific presets
[PresetName.INGESTION_SERVICE]: {
storage: StorageOption.CLOUD,
model: ModelPrecision.Q8,
features: ['core', 'batch-processing', 'entity-registry'],
distributed: true,
role: DistributedRole.WRITER,
writeOnly: true,
allowDirectReads: true,
verbose: false,
description: 'High-throughput data ingestion service',
category: PresetCategory.SERVICE
},
[PresetName.SEARCH_API]: {
storage: StorageOption.CLOUD,
model: ModelPrecision.FP32,
features: ['core', 'search', 'cache', 'triple-intelligence'],
distributed: true,
role: DistributedRole.READER,
readOnly: true,
lazyLoadInReadOnlyMode: true,
cache: {
hotCacheMaxSize: 10000,
autoTune: true
},
verbose: false,
description: 'Low-latency search API service',
category: PresetCategory.SERVICE
},
[PresetName.ANALYTICS_SERVICE]: {
storage: StorageOption.CLOUD,
model: ModelPrecision.AUTO,
features: ['core', 'search', 'metrics', 'monitoring'],
distributed: true,
role: DistributedRole.HYBRID,
verbose: false,
description: 'Analytics and data processing service',
category: PresetCategory.SERVICE
},
[PresetName.EDGE_CACHE]: {
storage: StorageOption.AUTO,
model: ModelPrecision.Q8,
features: ['core', 'search', 'cache'],
distributed: true,
role: DistributedRole.READER,
readOnly: true,
lazyLoadInReadOnlyMode: true,
cache: {
hotCacheMaxSize: 1000,
autoTune: false
},
verbose: false,
description: 'Edge location cache with minimal footprint',
category: PresetCategory.SERVICE
},
[PresetName.BATCH_PROCESSOR]: {
storage: StorageOption.CLOUD,
model: ModelPrecision.Q8,
features: ['core', 'batch-processing', 'neural-api'],
distributed: true,
role: DistributedRole.HYBRID,
cache: {
hotCacheMaxSize: 5000,
batchSize: 500
},
verbose: false,
description: 'Batch processing and bulk operations',
category: PresetCategory.SERVICE
},
[PresetName.STREAMING_SERVICE]: {
storage: StorageOption.CLOUD,
model: ModelPrecision.Q8,
features: ['core', 'batch-processing', 'wal'],
distributed: true,
role: DistributedRole.WRITER,
writeOnly: true,
allowDirectReads: false,
verbose: false,
description: 'Real-time data streaming service',
category: PresetCategory.SERVICE
},
[PresetName.ML_TRAINING]: {
storage: StorageOption.CLOUD,
model: ModelPrecision.FP32,
features: FeatureSet.FULL,
distributed: true,
role: DistributedRole.HYBRID,
cache: {
hotCacheMaxSize: 20000,
autoTune: true
},
verbose: true,
description: 'Machine learning training service',
category: PresetCategory.SERVICE
},
[PresetName.SIDECAR]: {
storage: StorageOption.AUTO,
model: ModelPrecision.Q8,
features: FeatureSet.MINIMAL,
distributed: false,
verbose: false,
description: 'Lightweight sidecar for microservices',
category: PresetCategory.SERVICE
}
} as const
/**
* Type-safe preset getter
*/
export function getPreset(name: PresetName): PresetConfig {
return PRESET_CONFIGS[name]
}
/**
* Check if a string is a valid preset name
*/
export function isValidPreset(name: string): name is PresetName {
return Object.values(PresetName).includes(name as PresetName)
}
/**
* Get presets by category
*/
export function getPresetsByCategory(category: PresetCategory): PresetName[] {
return Object.entries(PRESET_CONFIGS)
.filter(([_, config]) => config.category === category)
.map(([name]) => name as PresetName)
}
/**
* Get all preset names
*/
export function getAllPresetNames(): PresetName[] {
return Object.values(PresetName)
}
/**
* Get preset description
*/
export function getPresetDescription(name: PresetName): string {
return PRESET_CONFIGS[name].description
}
/**
* Convert preset config to Brainy config
*/
export function presetToBrainyConfig(preset: PresetConfig): any {
const config: any = {
storage: preset.storage,
model: preset.model,
verbose: preset.verbose
}
// Handle features
if (Array.isArray(preset.features)) {
config.features = preset.features
} else {
config.features = preset.features // Will be expanded by processZeroConfig
}
// Handle distributed settings
if (preset.distributed) {
config.distributed = {
enabled: true,
role: preset.role
}
if (preset.readOnly) config.readOnly = true
if (preset.writeOnly) config.writeOnly = true
if (preset.allowDirectReads) config.allowDirectReads = true
if (preset.lazyLoadInReadOnlyMode) config.lazyLoadInReadOnlyMode = true
}
return config
}

342
src/config/extensibleConfig.ts
View file

@ -1,342 +0,0 @@
/**
* Extensible Configuration System
* Allows augmentations to register new storage types, presets, and configurations
*/
import { StorageOption, PresetName, PresetConfig, ModelPrecision, DistributedRole, PresetCategory } from './distributedPresets.js'
import { StorageConfigResult } from './storageAutoConfig.js'
/**
* Storage provider registration interface
*/
export interface StorageProvider {
type: string // e.g., 'redis', 'mongodb', 'postgres'
name: string
description: string
// Detection function - returns true if this storage should be auto-selected
detect: () => Promise<boolean>
// Configuration builder
getConfig: () => Promise<any>
// Priority for auto-detection (higher = checked first)
priority?: number
// Required environment variables or packages
requirements?: {
env?: string[]
packages?: string[]
}
}
/**
* Preset extension interface
*/
export interface PresetExtension {
name: string
config: PresetConfig
override?: boolean // Override existing preset
}
/**
* Global registry for extensions
*/
class ConfigurationRegistry {
private static instance: ConfigurationRegistry
// Registered storage providers
private storageProviders: Map<string, StorageProvider> = new Map()
// Registered preset extensions
private presetExtensions: Map<string, PresetExtension> = new Map()
// Custom auto-detection hooks
private autoDetectHooks: Array<() => Promise<any>> = []
private constructor() {
// Initialize with built-in providers
this.registerBuiltInProviders()
}
static getInstance(): ConfigurationRegistry {
if (!ConfigurationRegistry.instance) {
ConfigurationRegistry.instance = new ConfigurationRegistry()
}
return ConfigurationRegistry.instance
}
/**
* Register a new storage provider
* This is how augmentations add new storage types
*/
registerStorageProvider(provider: StorageProvider): void {
console.log(`📦 Registering storage provider: ${provider.type} (${provider.name})`)
this.storageProviders.set(provider.type, provider)
}
/**
* Register a new preset
*/
registerPreset(name: string, extension: PresetExtension): void {
console.log(`🎨 Registering preset: ${name}`)
this.presetExtensions.set(name, extension)
}
/**
* Register an auto-detection hook
*/
registerAutoDetectHook(hook: () => Promise<any>): void {
this.autoDetectHooks.push(hook)
}
/**
* Get all registered storage providers
*/
getStorageProviders(): StorageProvider[] {
return Array.from(this.storageProviders.values())
.sort((a, b) => (b.priority || 0) - (a.priority || 0))
}
/**
* Get all registered presets (built-in + extensions)
*/
getAllPresets(): Map<string, PresetConfig> {
// Start with built-in presets
const allPresets = new Map<string, PresetConfig>()
// Note: Would import from distributedPresets-new.ts
// Add extended presets
for (const [name, extension] of this.presetExtensions) {
if (extension.override || !allPresets.has(name)) {
allPresets.set(name, extension.config)
}
}
return allPresets
}
/**
* Auto-detect storage including extensions
*/
async autoDetectStorage(): Promise<StorageConfigResult> {
// Check registered providers first (in priority order)
for (const provider of this.getStorageProviders()) {
try {
if (await provider.detect()) {
const config = await provider.getConfig()
return {
// Registered providers deliberately extend the built-in storage
// vocabulary ('redis', 'mongodb', ...) — the registry contract is
// open, so narrow the provider's free-form type string here.
type: provider.type as StorageConfigResult['type'],
config,
reason: `Auto-detected ${provider.name}`,
autoSelected: true
}
}
} catch (error) {
console.warn(`Failed to detect ${provider.type}:`, error)
}
}
// Fallback to built-in detection
const { autoDetectStorage } = await import('./storageAutoConfig.js')
return autoDetectStorage()
}
/**
* Register built-in providers
*/
private registerBuiltInProviders(): void {
// These would be the built-in ones, but could be overridden
}
}
/**
* Example: Redis storage provider registration
* This would be in the redis augmentation package
*/
export const redisStorageProvider: StorageProvider = {
type: 'redis',
name: 'Redis Storage',
description: 'High-performance in-memory data store',
priority: 10, // Check before filesystem
requirements: {
env: ['REDIS_URL', 'REDIS_HOST'],
packages: ['redis', 'ioredis']
},
async detect(): Promise<boolean> {
// Check for Redis connection info
if (process.env.REDIS_URL || process.env.REDIS_HOST) {
try {
// Try to connect to Redis (dynamic import for optional dependency)
const redis: any = await new Function('return import("ioredis")')().catch(() => null)
if (!redis) return false
const client = new redis.default(process.env.REDIS_URL)
await client.ping()
await client.quit()
return true
} catch {
// Redis not available
}
}
return false
},
async getConfig(): Promise<any> {
return {
type: 'redis',
redisStorage: {
url: process.env.REDIS_URL || `redis://${process.env.REDIS_HOST}:${process.env.REDIS_PORT || 6379}`,
prefix: process.env.REDIS_PREFIX || 'brainy:',
ttl: process.env.REDIS_TTL ? parseInt(process.env.REDIS_TTL) : undefined
}
}
}
}
/**
* Example: MongoDB storage provider
*/
export const mongoStorageProvider: StorageProvider = {
type: 'mongodb',
name: 'MongoDB Storage',
description: 'Document database for complex data',
priority: 8,
requirements: {
env: ['MONGODB_URI', 'MONGO_URL'],
packages: ['mongodb']
},
async detect(): Promise<boolean> {
if (process.env.MONGODB_URI || process.env.MONGO_URL) {
try {
const mongodb: any = await new Function('return import("mongodb")')().catch(() => null)
if (!mongodb) return false
const client = new mongodb.MongoClient(process.env.MONGODB_URI || process.env.MONGO_URL!)
await client.connect()
await client.close()
return true
} catch {
// MongoDB not available
}
}
return false
},
async getConfig(): Promise<any> {
return {
type: 'mongodb',
mongoStorage: {
uri: process.env.MONGODB_URI || process.env.MONGO_URL,
database: process.env.MONGO_DATABASE || 'brainy',
collection: process.env.MONGO_COLLECTION || 'vectors'
}
}
}
}
/**
* Example: PostgreSQL with pgvector extension
*/
export const postgresStorageProvider: StorageProvider = {
type: 'postgres',
name: 'PostgreSQL Storage',
description: 'PostgreSQL with pgvector for scalable vector search',
priority: 9,
requirements: {
env: ['DATABASE_URL', 'POSTGRES_URL'],
packages: ['pg', 'pgvector']
},
async detect(): Promise<boolean> {
const url = process.env.DATABASE_URL || process.env.POSTGRES_URL
if (url && url.includes('postgres')) {
try {
const pg: any = await new Function('return import("pg")')().catch(() => null)
if (!pg) return false
const client = new pg.Client({ connectionString: url })
await client.connect()
// Check for pgvector extension
const result = await client.query(
"SELECT * FROM pg_extension WHERE extname = 'vector'"
)
await client.end()
return result.rows.length > 0
} catch {
// PostgreSQL not available or pgvector not installed
}
}
return false
},
async getConfig(): Promise<any> {
return {
type: 'postgres',
postgresStorage: {
connectionString: process.env.DATABASE_URL || process.env.POSTGRES_URL,
table: process.env.POSTGRES_TABLE || 'brainy_vectors',
schema: process.env.POSTGRES_SCHEMA || 'public'
}
}
}
}
/**
* How an augmentation would register its storage provider
*/
export function registerStorageAugmentation(provider: StorageProvider): void {
const registry = ConfigurationRegistry.getInstance()
registry.registerStorageProvider(provider)
}
/**
* How to register a new preset
*/
export function registerPresetAugmentation(name: string, config: PresetConfig): void {
const registry = ConfigurationRegistry.getInstance()
registry.registerPreset(name, {
name,
config,
override: false
})
}
/**
* Example preset for Redis-based caching service
*/
export const redisCachePreset: PresetConfig = {
// 'redis' is an extended storage type outside the built-in StorageOption
// enum — extension presets resolve it at runtime through the provider
// registry, so this is a genuine typed boundary (hence the double
// assertion: the literal has no overlap with the enum).
storage: 'redis' as unknown as StorageOption,
model: ModelPrecision.Q8,
features: ['core', 'cache', 'search'],
distributed: true,
role: DistributedRole.READER,
readOnly: true,
cache: {
hotCacheMaxSize: 50000, // Large Redis cache
autoTune: true
},
description: 'Redis-backed caching layer',
category: PresetCategory.SERVICE
}
/**
* Get the configuration registry
*/
export function getConfigRegistry(): ConfigurationRegistry {
return ConfigurationRegistry.getInstance()
}

26
src/config/index.ts
View file

@ -35,32 +35,6 @@ export {
createEmbeddingFunctionWithPrecision
} from './zeroConfig.js'
// Strongly-typed presets and enums
export {
PresetName,
PresetCategory,
ModelPrecision,
StorageOption,
FeatureSet,
DistributedRole,
PresetConfig,
PRESET_CONFIGS,
getPreset,
isValidPreset,
getPresetsByCategory,
getAllPresetNames,
getPresetDescription,
presetToBrainyConfig
} from './distributedPresets.js'
// Extensible configuration
export {
StorageProvider,
registerStorageAugmentation,
registerPresetAugmentation,
getConfigRegistry
} from './extensibleConfig.js'
/**
* Main zero-config processor
* This is what Brainy will call

61
src/config/zeroConfig.ts
View file

@ -18,8 +18,10 @@ export interface BrainyZeroConfig {
* - 'development': Optimized for development (memory storage, fp32, verbose logging)
* - 'minimal': Minimal footprint (memory storage, q8, minimal features)
* - 'zero': True zero config (all auto-detected)
* - 'writer': Write-only instance for distributed setups (no index loading)
* - 'reader': Read-only instance for distributed setups (no write operations)
* - 'writer': Writer process (read + write) — the single mutating process in
* Brainy's multi-process model
* - 'reader': Reader process (read-only, all mutations throw) — any number of
* these may run against the same on-disk store as the writer
*/
mode?: 'production' | 'development' | 'minimal' | 'zero' | 'writer' | 'reader'
@ -84,21 +86,15 @@ const PRESETS = {
storage: 'auto' as const,
features: 'minimal' as const,
verbose: false,
// Writer-specific settings
distributed: true,
role: 'writer' as const,
writeOnly: true,
allowDirectReads: true // Allow deduplication checks
// The single mutating process: HybridMode (read + write).
mode: 'writer' as const
},
reader: {
storage: 'auto' as const,
features: 'default' as const,
verbose: false,
// Reader-specific settings
distributed: true,
role: 'reader' as const,
readOnly: true,
lazyLoadInReadOnlyMode: true // Optimize for search
// Read-only process: ReaderMode — every mutation throws.
mode: 'reader' as const
}
}
@ -263,44 +259,17 @@ export async function processZeroConfig(input?: string | BrainyZeroConfig): Prom
...config.advanced
}
// Apply distributed preset settings if applicable
// Multi-process role: writer (read + write) or reader (read-only). This is the
// only signal Brainy needs — `mode` drives the operational mode (HybridMode vs
// ReaderMode) that gates every mutation path.
if (config.mode === 'writer' || config.mode === 'reader') {
// The writer/reader presets each carry their own subset of the
// distributed flags beyond the base preset shape — widen to the union of
// those flags so both branches read uniformly.
const presetSettings: {
distributed: boolean
role: 'writer' | 'reader'
readOnly?: boolean
writeOnly?: boolean
allowDirectReads?: boolean
lazyLoadInReadOnlyMode?: boolean
} = PRESETS[config.mode]
// Apply distributed-specific settings
finalConfig.distributed = presetSettings.distributed
finalConfig.readOnly = presetSettings.readOnly || false
finalConfig.writeOnly = presetSettings.writeOnly || false
finalConfig.allowDirectReads = presetSettings.allowDirectReads || false
finalConfig.lazyLoadInReadOnlyMode = presetSettings.lazyLoadInReadOnlyMode || false
// Set distributed role in distributed config
if (finalConfig.distributed) {
finalConfig.distributed = {
enabled: true,
role: presetSettings.role
}
}
// Log distributed mode if verbose
finalConfig.mode = config.mode
if (verbose) {
console.log(`📡 Distributed mode: ${config.mode.toUpperCase()}`)
console.log(` Role: ${presetSettings.role}`)
console.log(` Read-only: ${finalConfig.readOnly}`)
console.log(` Write-only: ${finalConfig.writeOnly}`)
console.log(`📡 Process role: ${config.mode.toUpperCase()}`)
}
}
return finalConfig
}

6
src/coreTypes.ts
View file

@ -703,12 +703,6 @@ export interface StatisticsData {
* Last updated timestamp
*/
lastUpdated: string
/**
* Distributed configuration (stored in index folder for easy access)
* This is used for distributed Brainy instances coordination
*/
distributedConfig?: import('./types/distributedTypes.js').SharedConfig
}
/**

342
src/distributed/cacheSync.ts
View file

@ -1,342 +0,0 @@
/**
* Distributed Cache Synchronization
* Provides cache coherence across multiple Brainy instances
*/
import { EventEmitter } from 'node:events'
export interface CacheSyncConfig {
nodeId: string
syncInterval?: number
maxSyncBatchSize?: number
compressionEnabled?: boolean
}
export interface CacheEntry {
key: string
value: any
version: number
timestamp: number
ttl?: number
nodeId: string
}
export interface SyncMessage {
type: 'invalidate' | 'update' | 'delete' | 'batch'
entries: CacheEntry[]
source: string
timestamp: number
}
/**
* Distributed Cache Synchronizer
*/
export class CacheSync extends EventEmitter {
private nodeId: string
private localCache: Map<string, CacheEntry> = new Map()
private versionVector: Map<string, number> = new Map()
private syncQueue: SyncMessage[] = []
private syncInterval: number
private maxSyncBatchSize: number
private syncTimer?: NodeJS.Timeout
private isRunning: boolean = false
constructor(config: CacheSyncConfig) {
super()
this.nodeId = config.nodeId
this.syncInterval = config.syncInterval || 1000
this.maxSyncBatchSize = config.maxSyncBatchSize || 100
}
/**
* Start cache synchronization
*/
start(): void {
if (this.isRunning) return
this.isRunning = true
this.startSyncTimer()
this.emit('started', { nodeId: this.nodeId })
}
/**
* Stop cache synchronization
*/
stop(): void {
if (!this.isRunning) return
this.isRunning = false
if (this.syncTimer) {
clearInterval(this.syncTimer)
this.syncTimer = undefined
}
this.emit('stopped', { nodeId: this.nodeId })
}
/**
* Get a value from cache
*/
get(key: string): any | undefined {
const entry = this.localCache.get(key)
if (!entry) return undefined
// Check TTL
if (entry.ttl && Date.now() - entry.timestamp > entry.ttl) {
this.localCache.delete(key)
return undefined
}
return entry.value
}
/**
* Set a value in cache and propagate
*/
set(key: string, value: any, ttl?: number): void {
const version = this.incrementVersion(key)
const entry: CacheEntry = {
key,
value,
version,
timestamp: Date.now(),
ttl,
nodeId: this.nodeId
}
this.localCache.set(key, entry)
// Queue for sync
this.queueSync('update', [entry])
}
/**
* Delete a value from cache and propagate
*/
delete(key: string): boolean {
const existed = this.localCache.has(key)
if (existed) {
const version = this.incrementVersion(key)
this.localCache.delete(key)
// Queue deletion for sync
this.queueSync('delete', [{
key,
value: null,
version,
timestamp: Date.now(),
nodeId: this.nodeId
}])
}
return existed
}
/**
* Invalidate a cache entry across all nodes
*/
invalidate(key: string): void {
const version = this.incrementVersion(key)
this.localCache.delete(key)
// Queue invalidation
this.queueSync('invalidate', [{
key,
value: null,
version,
timestamp: Date.now(),
nodeId: this.nodeId
}])
}
/**
* Clear all cache entries
*/
clear(): void {
const entries: CacheEntry[] = []
for (const key of this.localCache.keys()) {
const version = this.incrementVersion(key)
entries.push({
key,
value: null,
version,
timestamp: Date.now(),
nodeId: this.nodeId
})
}
this.localCache.clear()
if (entries.length > 0) {
this.queueSync('delete', entries)
}
}
/**
* Handle incoming sync message from another node
*/
handleSyncMessage(message: SyncMessage): void {
if (message.source === this.nodeId) return // Ignore own messages
for (const entry of message.entries) {
this.handleRemoteEntry(message.type, entry)
}
this.emit('synced', {
type: message.type,
entries: message.entries.length,
source: message.source
})
}
/**
* Handle a remote cache entry
*/
private handleRemoteEntry(type: 'invalidate' | 'update' | 'delete' | 'batch', entry: CacheEntry): void {
const localEntry = this.localCache.get(entry.key)
const localVersion = this.versionVector.get(entry.key) || 0
// Version vector check - only accept if remote version is newer
if (entry.version <= localVersion) {
return // Our version is newer or same, ignore
}
// Update version vector
this.versionVector.set(entry.key, entry.version)
switch (type) {
case 'update':
case 'batch':
// Update local cache with remote value
this.localCache.set(entry.key, entry)
break
case 'delete':
case 'invalidate':
// Remove from local cache
this.localCache.delete(entry.key)
break
}
}
/**
* Queue a sync message
*/
private queueSync(type: 'invalidate' | 'update' | 'delete' | 'batch', entries: CacheEntry[]): void {
const message: SyncMessage = {
type,
entries,
source: this.nodeId,
timestamp: Date.now()
}
this.syncQueue.push(message)
// If queue is getting large, sync immediately
if (this.syncQueue.length >= this.maxSyncBatchSize) {
this.performSync()
}
}
/**
* Start sync timer
*/
private startSyncTimer(): void {
this.syncTimer = setInterval(() => {
this.performSync()
}, this.syncInterval)
}
/**
* Perform sync operation
*/
private performSync(): void {
if (this.syncQueue.length === 0) return
// Batch multiple messages if possible
const messages = this.syncQueue.splice(0, this.maxSyncBatchSize)
if (messages.length === 1) {
// Single message
this.emit('sync', messages[0])
} else {
// Batch multiple messages
const batchedEntries: CacheEntry[] = []
for (const msg of messages) {
batchedEntries.push(...msg.entries)
}
const batchMessage: SyncMessage = {
type: 'batch',
entries: batchedEntries,
source: this.nodeId,
timestamp: Date.now()
}
this.emit('sync', batchMessage)
}
}
/**
* Increment version for a key
*/
private incrementVersion(key: string): number {
const current = this.versionVector.get(key) || 0
const next = current + 1
this.versionVector.set(key, next)
return next
}
/**
* Get cache statistics
*/
getStats(): {
entries: number
pendingSync: number
versionedKeys: number
memoryUsage: number
} {
// Estimate memory usage (rough approximation)
let memoryUsage = 0
for (const entry of this.localCache.values()) {
memoryUsage += JSON.stringify(entry).length
}
return {
entries: this.localCache.size,
pendingSync: this.syncQueue.length,
versionedKeys: this.versionVector.size,
memoryUsage
}
}
/**
* Get cache entries for debugging
*/
getEntries(): CacheEntry[] {
return Array.from(this.localCache.values())
}
/**
* Merge cache state from another node (for recovery)
*/
mergeState(entries: CacheEntry[]): void {
for (const entry of entries) {
this.handleRemoteEntry('update', entry)
}
}
}
/**
* Create a cache sync instance
*/
export function createCacheSync(config: CacheSyncConfig): CacheSync {
return new CacheSync(config)
}

519
src/distributed/configManager.ts
View file

@ -1,519 +0,0 @@
/**
* Distributed Configuration Manager
* Manages shared configuration in S3 for distributed Brainy instances
*/
import { v4 as uuidv4 } from '../universal/uuid.js'
import {
DistributedConfig,
SharedConfig,
InstanceInfo,
InstanceRole
} from '../types/distributedTypes.js'
import { StorageAdapter } from '../coreTypes.js'
// Constants for config storage locations
const DISTRIBUTED_CONFIG_KEY = 'distributed_config'
const LEGACY_CONFIG_KEY = '_distributed_config'
export class DistributedConfigManager {
private config: SharedConfig | null = null
private instanceId: string
private role: InstanceRole | undefined
private configPath: string
private heartbeatInterval: number
private configCheckInterval: number
private instanceTimeout: number
private storage: StorageAdapter
private heartbeatTimer?: NodeJS.Timeout
private configWatchTimer?: NodeJS.Timeout
private lastConfigVersion: number = 0
private onConfigUpdate?: (config: SharedConfig) => void
private hasMigrated: boolean = false
constructor(
storage: StorageAdapter,
distributedConfig?: DistributedConfig,
brainyMode?: { readOnly?: boolean; writeOnly?: boolean }
) {
this.storage = storage
this.instanceId = distributedConfig?.instanceId || `instance-${uuidv4()}`
// Updated default path to use _system instead of _brainy
this.configPath = distributedConfig?.configPath || '_system/distributed_config.json'
this.heartbeatInterval = distributedConfig?.heartbeatInterval || 30000
this.configCheckInterval = distributedConfig?.configCheckInterval || 10000
this.instanceTimeout = distributedConfig?.instanceTimeout || 60000
// Set role from distributed config if provided
if (distributedConfig?.role) {
this.role = distributedConfig.role
}
// Infer role from Brainy's read/write mode if not explicitly set
else if (brainyMode) {
if (brainyMode.writeOnly) {
this.role = 'writer'
} else if (brainyMode.readOnly) {
this.role = 'reader'
}
// If neither readOnly nor writeOnly, role must be explicitly set
}
}
/**
* Initialize the distributed configuration
*/
async initialize(): Promise<SharedConfig> {
// Load or create configuration
this.config = await this.loadOrCreateConfig()
// Determine role if not explicitly set
if (!this.role) {
this.role = await this.determineRole()
}
// Register this instance
await this.registerInstance()
// Start heartbeat and config watching
this.startHeartbeat()
this.startConfigWatch()
return this.config
}
/**
* Load existing config or create new one
*/
private async loadOrCreateConfig(): Promise<SharedConfig> {
// First, try to load from the new location in index folder
try {
const configData = await this.storage.getStatistics()
if (configData && configData.distributedConfig) {
this.lastConfigVersion = configData.distributedConfig.version
return configData.distributedConfig as SharedConfig
}
} catch (error) {
// Config doesn't exist in new location yet
}
// Check if we need to migrate from old location
if (!this.hasMigrated) {
const migrated = await this.migrateConfigFromLegacyLocation()
if (migrated) {
return migrated
}
}
// Legacy fallback - try old location
try {
const configData = await this.storage.getMetadata(LEGACY_CONFIG_KEY)
if (configData) {
// Migrate to new location
const config = configData as unknown as SharedConfig
await this.migrateConfig(config)
this.lastConfigVersion = config.version
return config
}
} catch (error) {
// Config doesn't exist yet
}
// Create default config
const newConfig: SharedConfig = {
version: 1,
updated: new Date().toISOString(),
settings: {
partitionStrategy: 'hash',
partitionCount: 100,
embeddingModel: 'text-embedding-ada-002',
dimensions: 1536,
distanceMetric: 'cosine',
hnswParams: {
M: 16,
efConstruction: 200
}
},
instances: {}
}
await this.saveConfig(newConfig)
return newConfig
}
/**
* Determine role based on configuration
* IMPORTANT: Role must be explicitly set - no automatic assignment based on order
*/
private async determineRole(): Promise<InstanceRole> {
// Check environment variable first
if (process.env.BRAINY_ROLE) {
const role = process.env.BRAINY_ROLE.toLowerCase()
if (role === 'writer' || role === 'reader' || role === 'hybrid') {
return role as InstanceRole
}
throw new Error(`Invalid BRAINY_ROLE: ${process.env.BRAINY_ROLE}. Must be 'writer', 'reader', or 'hybrid'`)
}
// Check if explicitly passed in distributed config
if (this.role) {
return this.role
}
// DO NOT auto-assign roles based on deployment order or existing instances
// This is dangerous and can lead to data corruption or loss
throw new Error(
'Distributed mode requires explicit role configuration. ' +
'Set BRAINY_ROLE environment variable or pass role in distributed config. ' +
'Valid roles: "writer", "reader", "hybrid"'
)
}
/**
* Check if an instance is still alive
*/
private isInstanceAlive(instance: InstanceInfo): boolean {
const lastSeen = new Date(instance.lastHeartbeat).getTime()
const now = Date.now()
return (now - lastSeen) < this.instanceTimeout
}
/**
* Register this instance in the shared config
*/
private async registerInstance(): Promise<void> {
if (!this.config) return
// Role must be set by this point
if (!this.role) {
throw new Error('Cannot register instance without a role')
}
const instanceInfo: InstanceInfo = {
role: this.role,
status: 'active',
lastHeartbeat: new Date().toISOString(),
metrics: {
memoryUsage: process.memoryUsage().heapUsed
}
}
// Add endpoint if available
if (process.env.SERVICE_ENDPOINT) {
instanceInfo.endpoint = process.env.SERVICE_ENDPOINT
}
this.config.instances[this.instanceId] = instanceInfo
await this.saveConfig(this.config)
}
/**
* Migrate config from legacy location to new location
*/
private async migrateConfigFromLegacyLocation(): Promise<SharedConfig | null> {
try {
// Try to load from old location
const legacyConfig = await this.storage.getMetadata(LEGACY_CONFIG_KEY)
if (legacyConfig) {
console.log('Migrating distributed config from legacy location to index folder...')
const config = legacyConfig as unknown as SharedConfig
// Save to new location
await this.migrateConfig(config)
// Delete from old location (optional - we can keep it for rollback)
// await this.storage.deleteMetadata(LEGACY_CONFIG_KEY)
this.hasMigrated = true
this.lastConfigVersion = config.version
return config
}
} catch (error) {
console.error('Error during config migration:', error)
}
this.hasMigrated = true
return null
}
/**
* Migrate config to new location in index folder
*/
private async migrateConfig(config: SharedConfig): Promise<void> {
// Get existing statistics or create new
let stats = await this.storage.getStatistics()
if (!stats) {
stats = {
nounCount: {},
verbCount: {},
metadataCount: {},
hnswIndexSize: 0,
lastUpdated: new Date().toISOString()
}
}
// Add distributed config to statistics
stats.distributedConfig = config
// Save updated statistics
await this.storage.saveStatistics(stats)
}
/**
* Save configuration with version increment
*/
private async saveConfig(config: SharedConfig): Promise<void> {
config.version++
config.updated = new Date().toISOString()
this.lastConfigVersion = config.version
// Save to new location in index folder along with statistics
let stats = await this.storage.getStatistics()
if (!stats) {
stats = {
nounCount: {},
verbCount: {},
metadataCount: {},
hnswIndexSize: 0,
lastUpdated: new Date().toISOString()
}
}
// Update distributed config in statistics
stats.distributedConfig = config
// Save updated statistics
await this.storage.saveStatistics(stats)
this.config = config
}
/**
* Start heartbeat to keep instance alive in config
*/
private startHeartbeat(): void {
this.heartbeatTimer = setInterval(async () => {
await this.updateHeartbeat()
}, this.heartbeatInterval)
}
/**
* Update heartbeat and clean stale instances
*/
private async updateHeartbeat(): Promise<void> {
if (!this.config) return
// Reload config to get latest state
try {
const latestConfig = await this.loadConfig()
if (latestConfig) {
this.config = latestConfig
}
} catch (error) {
console.error('Failed to reload config:', error)
}
// Update our heartbeat
if (this.config.instances[this.instanceId]) {
this.config.instances[this.instanceId].lastHeartbeat = new Date().toISOString()
this.config.instances[this.instanceId].status = 'active'
// Update metrics if available
this.config.instances[this.instanceId].metrics = {
memoryUsage: process.memoryUsage().heapUsed
}
} else {
// Re-register if we were removed
await this.registerInstance()
return
}
// Clean up stale instances
const now = Date.now()
let hasChanges = false
for (const [id, instance] of Object.entries(this.config.instances)) {
if (id === this.instanceId) continue
const lastSeen = new Date(instance.lastHeartbeat).getTime()
if (now - lastSeen > this.instanceTimeout) {
delete this.config.instances[id]
hasChanges = true
}
}
// Save if there were changes
if (hasChanges) {
await this.saveConfig(this.config)
} else {
// Just update our heartbeat without version increment
// Get existing statistics
let stats = await this.storage.getStatistics()
if (!stats) {
stats = {
nounCount: {},
verbCount: {},
metadataCount: {},
hnswIndexSize: 0,
lastUpdated: new Date().toISOString()
}
}
// Update distributed config in statistics without version increment
stats.distributedConfig = this.config
// Save updated statistics
await this.storage.saveStatistics(stats)
}
}
/**
* Start watching for config changes
*/
private startConfigWatch(): void {
this.configWatchTimer = setInterval(async () => {
await this.checkForConfigUpdates()
}, this.configCheckInterval)
}
/**
* Check for configuration updates
*/
private async checkForConfigUpdates(): Promise<void> {
try {
const latestConfig = await this.loadConfig()
if (!latestConfig) return
if (latestConfig.version > this.lastConfigVersion) {
this.config = latestConfig
this.lastConfigVersion = latestConfig.version
// Notify listeners of config update
if (this.onConfigUpdate) {
this.onConfigUpdate(latestConfig)
}
}
} catch (error) {
console.error('Failed to check config updates:', error)
}
}
/**
* Load configuration from storage
*/
private async loadConfig(): Promise<SharedConfig | null> {
try {
// Try new location first
const stats = await this.storage.getStatistics()
if (stats && stats.distributedConfig) {
return stats.distributedConfig as SharedConfig
}
// Fallback to legacy location if not migrated yet
if (!this.hasMigrated) {
const configData = await this.storage.getMetadata(LEGACY_CONFIG_KEY)
if (configData) {
// Trigger migration on next save
return configData as unknown as SharedConfig
}
}
} catch (error) {
console.error('Failed to load config:', error)
}
return null
}
/**
* Get current configuration
*/
getConfig(): SharedConfig | null {
return this.config
}
/**
* Get instance role
*/
getRole(): InstanceRole {
if (!this.role) {
throw new Error('Role not initialized')
}
return this.role
}
/**
* Get instance ID
*/
getInstanceId(): string {
return this.instanceId
}
/**
* Set config update callback
*/
setOnConfigUpdate(callback: (config: SharedConfig) => void): void {
this.onConfigUpdate = callback
}
/**
* Get all active instances of a specific role
*/
getInstancesByRole(role: InstanceRole): InstanceInfo[] {
if (!this.config) return []
return Object.entries(this.config.instances)
.filter(([_, instance]) =>
instance.role === role &&
this.isInstanceAlive(instance)
)
.map(([_, instance]) => instance)
}
/**
* Update instance metrics
*/
async updateMetrics(metrics: Partial<InstanceInfo['metrics']>): Promise<void> {
if (!this.config || !this.config.instances[this.instanceId]) return
this.config.instances[this.instanceId].metrics = {
...this.config.instances[this.instanceId].metrics,
...metrics
}
// Don't increment version for metric updates
// Get existing statistics
let stats = await this.storage.getStatistics()
if (!stats) {
stats = {
nounCount: {},
verbCount: {},
metadataCount: {},
hnswIndexSize: 0,
lastUpdated: new Date().toISOString()
}
}
// Update distributed config in statistics without version increment
stats.distributedConfig = this.config
// Save updated statistics
await this.storage.saveStatistics(stats)
}
/**
* Cleanup resources
*/
async cleanup(): Promise<void> {
// Stop timers
if (this.heartbeatTimer) {
clearInterval(this.heartbeatTimer)
}
if (this.configWatchTimer) {
clearInterval(this.configWatchTimer)
}
// Mark instance as inactive
if (this.config && this.config.instances[this.instanceId]) {
this.config.instances[this.instanceId].status = 'inactive'
await this.saveConfig(this.config)
}
}
}

686
src/distributed/coordinator.ts
View file

@ -1,686 +0,0 @@
/**
* Distributed Coordinator for Brainy 3.0
* Provides leader election, consensus, and coordination for distributed instances
*/
import { EventEmitter } from 'node:events'
import { NetworkTransport, NetworkMessage } from './networkTransport.js'
import { createHash } from 'node:crypto'
export interface NodeInfo {
id: string
address: string
port: number
lastSeen: number
status: 'active' | 'inactive' | 'suspected'
state?: 'follower' | 'candidate' | 'leader'
lastHeartbeat?: number
}
export interface CoordinatorConfig {
nodeId?: string
address?: string
port?: number
heartbeatInterval?: number
electionTimeout?: number
nodes?: string[]
}
export interface ConsensusState {
term: number
votedFor: string | null
leader: string | null
state: 'follower' | 'candidate' | 'leader'
}
export interface RaftMessage {
type: 'requestVote' | 'voteResponse' | 'appendEntries' | 'appendResponse'
term: number
from: string
to?: string
data?: any
}
/**
* Distributed Coordinator implementing Raft-like consensus
*/
export class DistributedCoordinator extends EventEmitter {
private nodeId: string
private nodes: Map<string, NodeInfo> = new Map()
private consensusState: ConsensusState
private heartbeatInterval: number
private electionTimeout: number
private electionTimer?: NodeJS.Timeout
private heartbeatTimer?: NodeJS.Timeout
private isRunning: boolean = false
private networkTransport?: NetworkTransport
private votesReceived: Set<string> = new Set()
private currentTerm: number = 0
private lastLogIndex: number = -1
private lastLogTerm: number = 0
private logEntries: Array<{ term: number; index: number; data: any }> = []
private transport: any = null // For migration proposals
private pendingMigrations = new Map<string, any>()
private committedMigrations = new Set<string>()
constructor(config: CoordinatorConfig = {}) {
super()
// Generate node ID if not provided
this.nodeId = config.nodeId || this.generateNodeId()
// Configuration
this.heartbeatInterval = config.heartbeatInterval || 1000
this.electionTimeout = config.electionTimeout || 5000
// Initialize consensus state
this.consensusState = {
term: 0,
votedFor: null,
leader: null,
state: 'follower'
}
// Register this node
this.nodes.set(this.nodeId, {
id: this.nodeId,
address: config.address || 'localhost',
port: config.port || 3000,
lastSeen: Date.now(),
status: 'active',
state: 'follower',
lastHeartbeat: Date.now()
})
// Register other nodes if provided
if (config.nodes) {
this.registerNodes(config.nodes)
}
}
/**
* Start the coordinator
*/
async start(networkTransport?: NetworkTransport): Promise<void> {
if (this.isRunning) return
this.isRunning = true
this.networkTransport = networkTransport
// Setup network message handlers if transport is provided
if (this.networkTransport) {
this.setupNetworkHandlers()
}
this.emit('started', { nodeId: this.nodeId })
// Start as follower
this.becomeFollower()
}
/**
* Setup network message handlers
*/
private setupNetworkHandlers(): void {
if (!this.networkTransport) return
// Register handlers through the transport's public onMessage(), which
// writes to the same messageHandlers map this code used to reach into.
const transport = this.networkTransport
// Handle vote requests
transport.onMessage('requestVote', async (msg: NetworkMessage) => {
const response = await this.handleVoteRequest(msg)
// Send response back
if (this.networkTransport) {
await this.networkTransport.sendToNode(msg.from, 'voteResponse', response)
}
return response
})
// Handle vote responses
transport.onMessage('voteResponse', async (msg: NetworkMessage) => {
this.handleVoteResponse(msg)
})
// Handle heartbeats/append entries
transport.onMessage('appendEntries', async (msg: NetworkMessage) => {
const response = await this.handleAppendEntries(msg)
// Send response back
if (this.networkTransport) {
await this.networkTransport.sendToNode(msg.from, 'appendResponse', response)
}
return response
})
// Handle append responses
transport.onMessage('appendResponse', async (msg: NetworkMessage) => {
this.handleAppendResponse(msg)
})
}
/**
* Stop the coordinator
*/
async stop(): Promise<void> {
if (!this.isRunning) return
this.isRunning = false
if (this.electionTimer) {
clearTimeout(this.electionTimer)
this.electionTimer = undefined
}
if (this.heartbeatTimer) {
clearInterval(this.heartbeatTimer)
this.heartbeatTimer = undefined
}
this.emit('stopped')
}
/**
* Register additional nodes
*/
registerNodes(nodes: string[]): void {
for (const node of nodes) {
const [address, port] = node.split(':')
const nodeId = this.generateNodeId(node)
if (!this.nodes.has(nodeId)) {
this.nodes.set(nodeId, {
id: nodeId,
address,
port: parseInt(port || '3000'),
lastSeen: Date.now(),
status: 'active',
state: 'follower'
})
}
}
}
/**
* Become a follower
*/
private becomeFollower(): void {
this.consensusState.state = 'follower'
const node = this.nodes.get(this.nodeId)
if (node) {
node.state = 'follower'
}
this.resetElectionTimeout()
this.emit('stateChange', 'follower')
}
/**
* Become a candidate and start election
*/
private async becomeCandidate(): Promise<void> {
this.consensusState.state = 'candidate'
this.currentTerm++
this.consensusState.term = this.currentTerm
this.consensusState.votedFor = this.nodeId
this.votesReceived = new Set([this.nodeId])
const node = this.nodes.get(this.nodeId)
if (node) {
node.state = 'candidate'
}
this.emit('stateChange', 'candidate')
// Request votes from all other nodes
await this.requestVotes()
// Reset election timeout
this.resetElectionTimeout()
}
/**
* Become the leader
*/
private becomeLeader(): void {
this.consensusState.state = 'leader'
this.consensusState.leader = this.nodeId
const node = this.nodes.get(this.nodeId)
if (node) {
node.state = 'leader'
}
// Stop election timer as leader
if (this.electionTimer) {
clearTimeout(this.electionTimer)
this.electionTimer = undefined
}
this.emit('stateChange', 'leader')
this.emit('leaderElected', this.nodeId)
// Start sending heartbeats
this.startHeartbeat()
}
/**
* Request votes from all nodes
*/
private async requestVotes(): Promise<void> {
if (!this.networkTransport) {
// Simulate vote for testing
this.checkVoteMajority()
return
}
const voteRequest = {
type: 'requestVote' as const,
term: this.currentTerm,
candidateId: this.nodeId,
lastLogIndex: this.getLastLogIndex(),
lastLogTerm: this.getLastLogTerm()
}
// Send vote requests to all other nodes
for (const [nodeId] of this.nodes) {
if (nodeId !== this.nodeId) {
try {
await this.networkTransport.sendToNode(nodeId, 'requestVote', voteRequest)
} catch (err) {
console.error(`Failed to request vote from ${nodeId}:`, err)
}
}
}
}
/**
* Handle vote request from another node
*/
private async handleVoteRequest(msg: NetworkMessage): Promise<any> {
const { term, candidateId, lastLogIndex, lastLogTerm } = msg.data
// Update term if necessary
if (term > this.currentTerm) {
this.currentTerm = term
this.consensusState.term = term
this.consensusState.votedFor = null
this.becomeFollower()
}
// Grant vote if conditions are met
let voteGranted = false
if (term >= this.currentTerm &&
(!this.consensusState.votedFor || this.consensusState.votedFor === candidateId) &&
this.isLogUpToDate(lastLogIndex, lastLogTerm)) {
this.consensusState.votedFor = candidateId
voteGranted = true
this.resetElectionTimeout()
}
return {
type: 'voteResponse',
term: this.currentTerm,
voteGranted
}
}
/**
* Handle vote response
*/
private handleVoteResponse(msg: NetworkMessage): void {
const { term, voteGranted } = msg.data
// Ignore old responses
if (term < this.currentTerm) return
// Update term if necessary
if (term > this.currentTerm) {
this.currentTerm = term
this.consensusState.term = term
this.becomeFollower()
return
}
// Count vote if granted
if (voteGranted && this.consensusState.state === 'candidate') {
this.votesReceived.add(msg.from)
this.checkVoteMajority()
}
}
/**
* Check if we have majority votes
*/
private checkVoteMajority(): void {
const majority = Math.floor(this.nodes.size / 2) + 1
if (this.votesReceived.size >= majority) {
this.becomeLeader()
}
}
/**
* Handle append entries (heartbeat) from leader
*/
private async handleAppendEntries(msg: NetworkMessage): Promise<any> {
const { term, leaderId, prevLogIndex, prevLogTerm, entries, leaderCommit } = msg.data
// Update term if necessary
if (term > this.currentTerm) {
this.currentTerm = term
this.consensusState.term = term
this.consensusState.votedFor = null
this.becomeFollower()
}
// Reset election timeout when receiving valid heartbeat
if (term >= this.currentTerm) {
this.consensusState.leader = leaderId
this.resetElectionTimeout()
// Update leader node's last heartbeat
const leaderNode = this.nodes.get(leaderId)
if (leaderNode) {
leaderNode.lastHeartbeat = Date.now()
leaderNode.lastSeen = Date.now()
}
}
// Check log consistency
let success = false
if (term >= this.currentTerm) {
if (this.checkLogConsistency(prevLogIndex, prevLogTerm)) {
// Append new entries if any
if (entries && entries.length > 0) {
this.appendLogEntries(prevLogIndex, entries)
}
success = true
}
}
return {
type: 'appendResponse',
term: this.currentTerm,
success
}
}
/**
* Handle append response from follower
*/
private handleAppendResponse(msg: NetworkMessage): void {
const { term, success } = msg.data
// Update term if necessary
if (term > this.currentTerm) {
this.currentTerm = term
this.consensusState.term = term
this.becomeFollower()
}
// Process successful append
if (success && this.consensusState.state === 'leader') {
// Update follower's match index
// In a real implementation, this would track replication progress
}
}
/**
* Send heartbeat to followers
*/
private async sendHeartbeat(): Promise<void> {
if (!this.networkTransport) {
// Fallback for testing
await new Promise(resolve => setTimeout(resolve, 10))
return
}
const heartbeat = {
type: 'appendEntries' as const,
term: this.currentTerm,
leaderId: this.nodeId,
prevLogIndex: this.getLastLogIndex(),
prevLogTerm: this.getLastLogTerm(),
entries: [],
leaderCommit: 0
}
// Send heartbeat to all followers
for (const [nodeId] of this.nodes) {
if (nodeId !== this.nodeId) {
try {
await this.networkTransport.sendToNode(nodeId, 'appendEntries', heartbeat)
} catch (err) {
// Node might be down, mark as suspected
const node = this.nodes.get(nodeId)
if (node) {
node.status = 'suspected'
}
}
}
}
}
/**
* Start heartbeat timer as leader
*/
private startHeartbeat(): void {
if (this.heartbeatTimer) {
clearInterval(this.heartbeatTimer)
}
this.heartbeatTimer = setInterval(() => {
this.sendHeartbeat()
}, this.heartbeatInterval)
// Send immediate heartbeat
this.sendHeartbeat()
}
/**
* Reset election timeout
*/
private resetElectionTimeout(): void {
if (this.electionTimer) {
clearTimeout(this.electionTimer)
}
// Randomize timeout to prevent split votes
const timeout = this.electionTimeout + Math.random() * this.electionTimeout
this.electionTimer = setTimeout(() => {
if (this.consensusState.state === 'follower') {
this.becomeCandidate()
}
}, timeout)
}
/**
* Check if log is up to date
*/
private isLogUpToDate(lastLogIndex: number, lastLogTerm: number): boolean {
const myLastLogTerm = this.getLastLogTerm()
const myLastLogIndex = this.getLastLogIndex()
if (lastLogTerm > myLastLogTerm) return true
if (lastLogTerm < myLastLogTerm) return false
return lastLogIndex >= myLastLogIndex
}
/**
* Check log consistency
*/
private checkLogConsistency(prevLogIndex: number, prevLogTerm: number): boolean {
if (prevLogIndex === -1) return true
if (prevLogIndex >= this.logEntries.length) return false
const entry = this.logEntries[prevLogIndex]
return entry && entry.term === prevLogTerm
}
/**
* Append log entries
*/
private appendLogEntries(prevLogIndex: number, entries: any[]): void {
// Remove conflicting entries
this.logEntries = this.logEntries.slice(0, prevLogIndex + 1)
// Append new entries
for (const entry of entries) {
this.logEntries.push({
term: entry.term,
index: this.logEntries.length,
data: entry.data
})
}
this.lastLogIndex = this.logEntries.length - 1
if (this.lastLogIndex >= 0) {
this.lastLogTerm = this.logEntries[this.lastLogIndex].term
}
}
/**
* Get last log index
*/
private getLastLogIndex(): number {
return this.lastLogIndex
}
/**
* Get last log term
*/
private getLastLogTerm(): number {
return this.lastLogTerm
}
/**
* Generate a unique node ID
*/
private generateNodeId(seed?: string): string {
const source = seed || `${process.pid}-${Date.now()}-${Math.random()}`
return createHash('sha256').update(source).digest('hex').substring(0, 16)
}
/**
* Get current leader
*/
getLeader(): string | null {
return this.consensusState.leader
}
/**
* Propose a shard migration to the cluster
*/
async proposeMigration(migration: {
shardId: string
fromNode: string
toNode: string
migrationId: string
}): Promise<void> {
if (!this.isLeader()) {
throw new Error('Only leader can propose migrations')
}
// Broadcast migration proposal to all nodes
const message = {
type: 'migration-proposal',
migration,
timestamp: Date.now()
}
await this.transport.broadcast('migration', message)
// Store migration as pending
this.pendingMigrations.set(migration.migrationId, {
...migration,
status: 'pending'
})
}
/**
* Get migration status
*/
async getMigrationStatus(migrationId: string): Promise<'pending' | 'committed' | 'rejected'> {
const migration = this.pendingMigrations.get(migrationId)
if (!migration) {
// Check if it was committed
return this.committedMigrations.has(migrationId) ? 'committed' : 'rejected'
}
return migration.status || 'pending'
}
/**
* Check if this node is the leader
*/
isLeader(): boolean {
return this.consensusState.state === 'leader'
}
/**
* Get all nodes in the cluster
*/
getNodes(): NodeInfo[] {
return Array.from(this.nodes.values())
}
/**
* Get cluster health status
*/
getHealth(): { healthy: boolean; leader: string | null; nodes: number; activeNodes: number } {
const now = Date.now()
const activeNodes = Array.from(this.nodes.values()).filter(
node => now - node.lastSeen < this.electionTimeout
).length
return {
healthy: this.consensusState.leader !== null && activeNodes > this.nodes.size / 2,
leader: this.consensusState.leader,
nodes: this.nodes.size,
activeNodes
}
}
/**
* Propose a command to the cluster
*/
async proposeCommand(command: any): Promise<void> {
if (this.consensusState.state !== 'leader') {
throw new Error(`Not the leader. Current leader: ${this.consensusState.leader}`)
}
// Add to log
const entry = {
term: this.currentTerm,
index: this.logEntries.length,
data: command
}
this.logEntries.push(entry)
this.lastLogIndex = entry.index
this.lastLogTerm = entry.term
// Replicate to followers
await this.sendHeartbeat()
this.emit('commandProposed', command)
}
/**
* Get current state
*/
getState(): ConsensusState {
return { ...this.consensusState }
}
}
/**
* Create a coordinator instance
*/
export function createCoordinator(config?: CoordinatorConfig): DistributedCoordinator {
return new DistributedCoordinator(config)
}

323
src/distributed/domainDetector.ts
View file

@ -1,323 +0,0 @@
/**
* Domain Detector
* Automatically detects and manages data domains for logical separation
*/
import { DomainMetadata } from '../types/distributedTypes.js'
export interface DomainPattern {
domain: string
patterns: {
fields?: string[]
keywords?: string[]
regex?: RegExp
}
priority?: number
}
export class DomainDetector {
private domainPatterns: DomainPattern[] = [
{
domain: 'medical',
patterns: {
fields: ['symptoms', 'diagnosis', 'treatment', 'medication', 'patient'],
keywords: ['medical', 'health', 'disease', 'symptom', 'treatment', 'doctor', 'patient']
},
priority: 1
},
{
domain: 'legal',
patterns: {
fields: ['contract', 'clause', 'litigation', 'statute', 'jurisdiction'],
keywords: ['legal', 'law', 'contract', 'court', 'attorney', 'litigation', 'statute']
},
priority: 1
},
{
domain: 'product',
patterns: {
fields: ['price', 'sku', 'inventory', 'category', 'brand'],
keywords: ['product', 'price', 'sale', 'inventory', 'catalog', 'item', 'sku']
},
priority: 1
},
{
domain: 'customer',
patterns: {
fields: ['customerId', 'email', 'phone', 'address', 'orders'],
keywords: ['customer', 'client', 'user', 'account', 'profile', 'contact']
},
priority: 1
},
{
domain: 'financial',
patterns: {
fields: ['amount', 'currency', 'transaction', 'balance', 'account'],
keywords: ['financial', 'money', 'payment', 'transaction', 'bank', 'credit', 'debit']
},
priority: 1
},
{
domain: 'technical',
patterns: {
fields: ['code', 'function', 'error', 'stack', 'api'],
keywords: ['code', 'software', 'api', 'error', 'debug', 'function', 'class', 'method']
},
priority: 2
}
]
private customPatterns: DomainPattern[] = []
private domainStats: Map<string, number> = new Map()
/**
* Detect domain from data object
* @param data - The data object to analyze
* @returns The detected domain and metadata
*/
detectDomain(data: any): DomainMetadata {
if (!data || typeof data !== 'object') {
return { domain: 'general' }
}
// Check for explicit domain field
if (data.domain && typeof data.domain === 'string') {
this.updateStats(data.domain)
return {
domain: data.domain,
domainMetadata: this.extractDomainMetadata(data, data.domain)
}
}
// Score each domain pattern
const scores = new Map<string, number>()
// Check custom patterns first (higher priority)
for (const pattern of this.customPatterns) {
const score = this.scorePattern(data, pattern)
if (score > 0) {
scores.set(pattern.domain, score * (pattern.priority || 1))
}
}
// Check default patterns
for (const pattern of this.domainPatterns) {
const score = this.scorePattern(data, pattern)
if (score > 0) {
const currentScore = scores.get(pattern.domain) || 0
scores.set(pattern.domain, currentScore + score * (pattern.priority || 1))
}
}
// Find highest scoring domain
let bestDomain = 'general'
let bestScore = 0
for (const [domain, score] of scores.entries()) {
if (score > bestScore) {
bestDomain = domain
bestScore = score
}
}
this.updateStats(bestDomain)
return {
domain: bestDomain,
domainMetadata: this.extractDomainMetadata(data, bestDomain)
}
}
/**
* Score a data object against a domain pattern
*/
private scorePattern(data: any, pattern: DomainPattern): number {
let score = 0
// Check field matches
if (pattern.patterns.fields) {
const dataKeys = Object.keys(data)
for (const field of pattern.patterns.fields) {
if (dataKeys.some(key => key.toLowerCase().includes(field.toLowerCase()))) {
score += 2 // Field match is strong signal
}
}
}
// Check keyword matches in values
if (pattern.patterns.keywords) {
const dataStr = JSON.stringify(data).toLowerCase()
for (const keyword of pattern.patterns.keywords) {
if (dataStr.includes(keyword.toLowerCase())) {
score += 1
}
}
}
// Check regex patterns
if (pattern.patterns.regex) {
const dataStr = JSON.stringify(data)
if (pattern.patterns.regex.test(dataStr)) {
score += 3 // Regex match is very specific
}
}
return score
}
/**
* Extract domain-specific metadata
*/
private extractDomainMetadata(data: any, domain: string): Record<string, any> {
const metadata: Record<string, any> = {}
switch (domain) {
case 'medical':
if (data.patientId) metadata.patientId = data.patientId
if (data.condition) metadata.condition = data.condition
if (data.severity) metadata.severity = data.severity
break
case 'legal':
if (data.caseId) metadata.caseId = data.caseId
if (data.jurisdiction) metadata.jurisdiction = data.jurisdiction
if (data.documentType) metadata.documentType = data.documentType
break
case 'product':
if (data.sku) metadata.sku = data.sku
if (data.category) metadata.category = data.category
if (data.brand) metadata.brand = data.brand
if (data.price) metadata.priceRange = this.getPriceRange(data.price)
break
case 'customer':
if (data.customerId) metadata.customerId = data.customerId
if (data.segment) metadata.segment = data.segment
if (data.lifetime_value) metadata.valueCategory = this.getValueCategory(data.lifetime_value)
break
case 'financial':
if (data.accountId) metadata.accountId = data.accountId
if (data.transactionType) metadata.transactionType = data.transactionType
if (data.amount) metadata.amountRange = this.getAmountRange(data.amount)
break
case 'technical':
if (data.service) metadata.service = data.service
if (data.environment) metadata.environment = data.environment
if (data.severity) metadata.severity = data.severity
break
}
// Add detection confidence
metadata.detectionConfidence = this.calculateConfidence(data, domain)
return metadata
}
/**
* Calculate detection confidence
*/
private calculateConfidence(data: any, domain: string): 'high' | 'medium' | 'low' {
// If domain was explicitly specified
if (data.domain === domain) return 'high'
// Check how many patterns matched
const pattern = [...this.customPatterns, ...this.domainPatterns]
.find(p => p.domain === domain)
if (!pattern) return 'low'
const score = this.scorePattern(data, pattern)
if (score >= 5) return 'high'
if (score >= 2) return 'medium'
return 'low'
}
/**
* Categorize price ranges
*/
private getPriceRange(price: number): string {
if (price < 10) return 'low'
if (price < 100) return 'medium'
if (price < 1000) return 'high'
return 'premium'
}
/**
* Categorize customer value
*/
private getValueCategory(value: number): string {
if (value < 100) return 'low'
if (value < 1000) return 'medium'
if (value < 10000) return 'high'
return 'vip'
}
/**
* Categorize amount ranges
*/
private getAmountRange(amount: number): string {
if (amount < 100) return 'micro'
if (amount < 1000) return 'small'
if (amount < 10000) return 'medium'
if (amount < 100000) return 'large'
return 'enterprise'
}
/**
* Add custom domain pattern
* @param pattern - Custom domain pattern to add
*/
addCustomPattern(pattern: DomainPattern): void {
// Remove existing pattern for same domain if exists
this.customPatterns = this.customPatterns.filter(p => p.domain !== pattern.domain)
this.customPatterns.push(pattern)
}
/**
* Remove custom domain pattern
* @param domain - Domain to remove pattern for
*/
removeCustomPattern(domain: string): void {
this.customPatterns = this.customPatterns.filter(p => p.domain !== domain)
}
/**
* Update domain statistics
*/
private updateStats(domain: string): void {
const count = this.domainStats.get(domain) || 0
this.domainStats.set(domain, count + 1)
}
/**
* Get domain statistics
* @returns Map of domain to count
*/
getDomainStats(): Map<string, number> {
return new Map(this.domainStats)
}
/**
* Clear domain statistics
*/
clearStats(): void {
this.domainStats.clear()
}
/**
* Get all configured domains
* @returns Array of domain names
*/
getConfiguredDomains(): string[] {
const domains = new Set<string>()
for (const pattern of [...this.domainPatterns, ...this.customPatterns]) {
domains.add(pattern.domain)
}
return Array.from(domains).sort()
}
}

170
src/distributed/hashPartitioner.ts
View file

@ -1,170 +0,0 @@
/**
* Hash-based Partitioner
* Provides deterministic partitioning for distributed writes
*/
import { getPartitionHash } from '../utils/crypto.js'
import { SharedConfig } from '../types/distributedTypes.js'
export class HashPartitioner {
private partitionCount: number
private partitionPrefix: string = 'vectors/p'
constructor(config: SharedConfig) {
this.partitionCount = config.settings.partitionCount || 100
}
/**
* Get partition for a given vector ID using deterministic hashing
* @param vectorId - The unique identifier of the vector
* @returns The partition path
*/
getPartition(vectorId: string): string {
const hash = this.hashString(vectorId)
const partitionIndex = hash % this.partitionCount
return `${this.partitionPrefix}${partitionIndex.toString().padStart(3, '0')}`
}
/**
* Get partition with domain metadata (domain stored as metadata, not in path)
* @param vectorId - The unique identifier of the vector
* @param domain - The domain identifier (for metadata only)
* @returns The partition path
*/
getPartitionWithDomain(vectorId: string, domain?: string): string {
// Domain doesn't affect partitioning - it's just metadata
return this.getPartition(vectorId)
}
/**
* Get all partition paths
* @returns Array of all partition paths
*/
getAllPartitions(): string[] {
const partitions: string[] = []
for (let i = 0; i < this.partitionCount; i++) {
partitions.push(`${this.partitionPrefix}${i.toString().padStart(3, '0')}`)
}
return partitions
}
/**
* Get partition index from partition path
* @param partitionPath - The partition path
* @returns The partition index
*/
getPartitionIndex(partitionPath: string): number {
const match = partitionPath.match(/p(\d+)$/)
if (match) {
return parseInt(match[1], 10)
}
throw new Error(`Invalid partition path: ${partitionPath}`)
}
/**
* Hash a string to a number for consistent partitioning
* @param str - The string to hash
* @returns A positive integer hash
*/
private hashString(str: string): number {
// Use our cross-platform hash function
return getPartitionHash(str)
}
/**
* Get partitions for batch operations
* Groups vector IDs by their target partition
* @param vectorIds - Array of vector IDs
* @returns Map of partition to vector IDs
*/
getPartitionsForBatch(vectorIds: string[]): Map<string, string[]> {
const partitionMap = new Map<string, string[]>()
for (const id of vectorIds) {
const partition = this.getPartition(id)
if (!partitionMap.has(partition)) {
partitionMap.set(partition, [])
}
partitionMap.get(partition)!.push(id)
}
return partitionMap
}
}
/**
* Affinity-based Partitioner
* Extends HashPartitioner to prefer certain partitions for a writer
* while maintaining correctness
*/
export class AffinityPartitioner extends HashPartitioner {
private preferredPartitions: Set<number>
private instanceId: string
constructor(config: SharedConfig, instanceId: string) {
super(config)
this.instanceId = instanceId
this.preferredPartitions = this.calculatePreferredPartitions(config)
}
/**
* Calculate preferred partitions for this instance
*/
private calculatePreferredPartitions(config: SharedConfig): Set<number> {
const partitionCount = config.settings.partitionCount || 100
const writers = Object.entries(config.instances)
.filter(([_, inst]) => inst.role === 'writer')
.map(([id, _]) => id)
.sort() // Ensure consistent ordering
const writerIndex = writers.indexOf(this.instanceId)
if (writerIndex === -1) {
// Not a writer or not found, no preferences
return new Set()
}
const writerCount = writers.length
const partitionsPerWriter = Math.ceil(partitionCount / writerCount)
const preferred = new Set<number>()
const start = writerIndex * partitionsPerWriter
const end = Math.min(start + partitionsPerWriter, partitionCount)
for (let i = start; i < end; i++) {
preferred.add(i)
}
return preferred
}
/**
* Check if a partition is preferred for this instance
* @param partitionPath - The partition path
* @returns Whether this partition is preferred
*/
isPreferredPartition(partitionPath: string): boolean {
try {
const index = this.getPartitionIndex(partitionPath)
return this.preferredPartitions.has(index)
} catch {
return false
}
}
/**
* Get all preferred partitions for this instance
* @returns Array of preferred partition paths
*/
getPreferredPartitions(): string[] {
return Array.from(this.preferredPartitions)
.map(index => `vectors/p${index.toString().padStart(3, '0')}`)
}
/**
* Update preferred partitions based on new config
* @param config - The updated shared configuration
*/
updatePreferences(config: SharedConfig): void {
this.preferredPartitions = this.calculatePreferredPartitions(config)
}
}

301
src/distributed/healthMonitor.ts
View file

@ -1,301 +0,0 @@
/**
* Health Monitor
* Monitors and reports instance health in distributed deployments
*/
import { DistributedConfigManager } from './configManager.js'
import { InstanceInfo } from '../types/distributedTypes.js'
export interface HealthMetrics {
vectorCount: number
cacheHitRate: number
memoryUsage: number
cpuUsage?: number
requestsPerSecond?: number
averageLatency?: number
errorRate?: number
}
export interface HealthStatus {
status: 'healthy' | 'degraded' | 'unhealthy'
instanceId: string
role: string
uptime: number
lastCheck: string
metrics: HealthMetrics
warnings?: string[]
errors?: string[]
}
export class HealthMonitor {
private configManager: DistributedConfigManager
private startTime: number
private requestCount: number = 0
private errorCount: number = 0
private totalLatency: number = 0
private cacheHits: number = 0
private cacheMisses: number = 0
private vectorCount: number = 0
private checkInterval: number = 30000 // 30 seconds
private healthCheckTimer?: NodeJS.Timeout
private metricsWindow: number[] = [] // Sliding window for RPS calculation
private latencyWindow: number[] = [] // Sliding window for latency
private windowSize: number = 60000 // 1 minute window
constructor(configManager: DistributedConfigManager) {
this.configManager = configManager
this.startTime = Date.now()
}
/**
* Start health monitoring
*/
start(): void {
// Initial health update
this.updateHealth()
// Schedule periodic health checks
this.healthCheckTimer = setInterval(() => {
this.updateHealth()
}, this.checkInterval)
}
/**
* Stop health monitoring
*/
stop(): void {
if (this.healthCheckTimer) {
clearInterval(this.healthCheckTimer)
this.healthCheckTimer = undefined
}
}
/**
* Update health status and metrics
*/
private async updateHealth(): Promise<void> {
const metrics = this.collectMetrics()
// Update config with latest metrics
await this.configManager.updateMetrics({
vectorCount: metrics.vectorCount,
cacheHitRate: metrics.cacheHitRate,
memoryUsage: metrics.memoryUsage,
cpuUsage: metrics.cpuUsage
})
// Clean sliding windows
this.cleanWindows()
}
/**
* Collect current metrics
*/
private collectMetrics(): HealthMetrics {
const memUsage = process.memoryUsage()
return {
vectorCount: this.vectorCount,
cacheHitRate: this.calculateCacheHitRate(),
memoryUsage: memUsage.heapUsed,
cpuUsage: this.getCPUUsage(),
requestsPerSecond: this.calculateRPS(),
averageLatency: this.calculateAverageLatency(),
errorRate: this.calculateErrorRate()
}
}
/**
* Calculate cache hit rate
*/
private calculateCacheHitRate(): number {
const total = this.cacheHits + this.cacheMisses
if (total === 0) return 0
return this.cacheHits / total
}
/**
* Calculate requests per second
*/
private calculateRPS(): number {
const now = Date.now()
const recentRequests = this.metricsWindow.filter(
timestamp => now - timestamp < this.windowSize
)
return recentRequests.length / (this.windowSize / 1000)
}
/**
* Calculate average latency
*/
private calculateAverageLatency(): number {
if (this.latencyWindow.length === 0) return 0
const sum = this.latencyWindow.reduce((a, b) => a + b, 0)
return sum / this.latencyWindow.length
}
/**
* Calculate error rate
*/
private calculateErrorRate(): number {
if (this.requestCount === 0) return 0
return this.errorCount / this.requestCount
}
/**
* Get CPU usage (simplified)
*/
private getCPUUsage(): number {
// Simplified CPU usage based on process time
const usage = process.cpuUsage()
const total = usage.user + usage.system
const seconds = (Date.now() - this.startTime) / 1000
return Math.min(100, (total / 1000000 / seconds) * 100)
}
/**
* Clean old entries from sliding windows
*/
private cleanWindows(): void {
const now = Date.now()
const cutoff = now - this.windowSize
this.metricsWindow = this.metricsWindow.filter(t => t > cutoff)
// Keep only recent latency measurements
if (this.latencyWindow.length > 100) {
this.latencyWindow = this.latencyWindow.slice(-100)
}
}
/**
* Record a request
* @param latency - Request latency in milliseconds
* @param error - Whether the request resulted in an error
*/
recordRequest(latency: number, error: boolean = false): void {
this.requestCount++
this.metricsWindow.push(Date.now())
this.latencyWindow.push(latency)
if (error) {
this.errorCount++
}
}
/**
* Record cache access
* @param hit - Whether it was a cache hit
*/
recordCacheAccess(hit: boolean): void {
if (hit) {
this.cacheHits++
} else {
this.cacheMisses++
}
}
/**
* Update vector count
* @param count - New vector count
*/
updateVectorCount(count: number): void {
this.vectorCount = count
}
/**
* Get current health status
* @returns Health status object
*/
getHealthStatus(): HealthStatus {
const metrics = this.collectMetrics()
const uptime = Date.now() - this.startTime
const warnings: string[] = []
const errors: string[] = []
// Check for warnings
if (metrics.memoryUsage > 1024 * 1024 * 1024) { // > 1GB
warnings.push('High memory usage detected')
}
if (metrics.cacheHitRate < 0.5) {
warnings.push('Low cache hit rate')
}
if (metrics.errorRate && metrics.errorRate > 0.05) {
warnings.push('High error rate detected')
}
if (metrics.averageLatency && metrics.averageLatency > 1000) {
warnings.push('High latency detected')
}
// Check for errors
if (metrics.memoryUsage > 2 * 1024 * 1024 * 1024) { // > 2GB
errors.push('Critical memory usage')
}
if (metrics.errorRate && metrics.errorRate > 0.2) {
errors.push('Critical error rate')
}
// Determine overall status
let status: 'healthy' | 'degraded' | 'unhealthy' = 'healthy'
if (errors.length > 0) {
status = 'unhealthy'
} else if (warnings.length > 0) {
status = 'degraded'
}
return {
status,
instanceId: this.configManager.getInstanceId(),
role: this.configManager.getRole(),
uptime,
lastCheck: new Date().toISOString(),
metrics,
warnings: warnings.length > 0 ? warnings : undefined,
errors: errors.length > 0 ? errors : undefined
}
}
/**
* Get health check endpoint data
* @returns JSON-serializable health data
*/
getHealthEndpointData(): Record<string, any> {
const status = this.getHealthStatus()
return {
status: status.status,
instanceId: status.instanceId,
role: status.role,
uptime: Math.floor(status.uptime / 1000), // Convert to seconds
lastCheck: status.lastCheck,
metrics: {
vectorCount: status.metrics.vectorCount,
cacheHitRate: Math.round(status.metrics.cacheHitRate * 100) / 100,
memoryUsageMB: Math.round(status.metrics.memoryUsage / 1024 / 1024),
cpuUsagePercent: Math.round(status.metrics.cpuUsage || 0),
requestsPerSecond: Math.round(status.metrics.requestsPerSecond || 0),
averageLatencyMs: Math.round(status.metrics.averageLatency || 0),
errorRate: Math.round((status.metrics.errorRate || 0) * 100) / 100
},
warnings: status.warnings,
errors: status.errors
}
}
/**
* Reset metrics (useful for testing)
*/
resetMetrics(): void {
this.requestCount = 0
this.errorCount = 0
this.totalLatency = 0
this.cacheHits = 0
this.cacheMisses = 0
this.metricsWindow = []
this.latencyWindow = []
}
}

547
src/distributed/httpTransport.ts
View file

@ -1,547 +0,0 @@
/**
* HTTP + SSE Transport for Zero-Config Distributed Brainy
* Simple, reliable, works everywhere - no WebSocket complexity!
* REAL PRODUCTION CODE - Handles millions of operations
*/
import * as http from 'node:http'
import * as https from 'node:https'
import { EventEmitter } from 'node:events'
import * as net from 'node:net'
import { URL } from 'node:url'
export interface TransportMessage {
id: string
method: string
params: any
timestamp: number
from: string
to?: string
}
export interface TransportResponse {
id: string
result?: any
error?: {
code: number
message: string
data?: any
}
timestamp: number
}
export interface SSEClient {
id: string
response: http.ServerResponse
lastPing: number
}
export class HTTPTransport extends EventEmitter {
private server: http.Server | null = null
private port: number = 0
private nodeId: string
private endpoints: Map<string, string> = new Map() // nodeId -> endpoint
private pendingRequests: Map<string, {
resolve: (value: any) => void
reject: (error: any) => void
timeout: NodeJS.Timeout
}> = new Map()
private sseClients: Map<string, SSEClient> = new Map()
private messageHandlers: Map<string, (params: any, from: string) => Promise<any>> = new Map()
private isRunning: boolean = false
private readonly REQUEST_TIMEOUT = 30000 // 30 seconds
private readonly SSE_HEARTBEAT_INTERVAL = 15000 // 15 seconds
private sseHeartbeatTimer: NodeJS.Timeout | null = null
constructor(nodeId: string) {
super()
this.nodeId = nodeId
}
/**
* Start HTTP server with automatic port selection
*/
async start(): Promise<number> {
if (this.isRunning) return this.port
// Create HTTP server with all handlers
this.server = http.createServer(async (req, res) => {
// Enable CORS for browser compatibility
res.setHeader('Access-Control-Allow-Origin', '*')
res.setHeader('Access-Control-Allow-Methods', 'GET, POST, OPTIONS')
res.setHeader('Access-Control-Allow-Headers', 'Content-Type')
if (req.method === 'OPTIONS') {
res.writeHead(204)
res.end()
return
}
const url = new URL(req.url || '/', `http://${req.headers.host}`)
try {
// Route requests
if (url.pathname === '/health') {
await this.handleHealth(req, res)
} else if (url.pathname === '/rpc') {
await this.handleRPC(req, res)
} else if (url.pathname === '/events') {
await this.handleSSE(req, res)
} else if (url.pathname.startsWith('/stream/')) {
await this.handleStream(req, res, url)
} else {
res.writeHead(404)
res.end('Not Found')
}
} catch (err) {
console.error(`[${this.nodeId}] Request error:`, err)
res.writeHead(500)
res.end('Internal Server Error')
}
})
// Find available port
this.port = await this.findAvailablePort()
// Start server
await new Promise<void>((resolve, reject) => {
this.server!.listen(this.port, () => {
console.log(`[${this.nodeId}] HTTP transport listening on port ${this.port}`)
resolve()
})
this.server!.on('error', reject)
})
this.isRunning = true
// Start SSE heartbeat
this.startSSEHeartbeat()
this.emit('started', this.port)
return this.port
}
/**
* Stop HTTP server
*/
async stop(): Promise<void> {
if (!this.isRunning) return
this.isRunning = false
// Stop SSE heartbeat
if (this.sseHeartbeatTimer) {
clearInterval(this.sseHeartbeatTimer)
this.sseHeartbeatTimer = null
}
// Close all SSE connections
for (const client of this.sseClients.values()) {
client.response.end()
}
this.sseClients.clear()
// Cancel pending requests
for (const pending of this.pendingRequests.values()) {
clearTimeout(pending.timeout)
pending.reject(new Error('Transport shutting down'))
}
this.pendingRequests.clear()
// Close server
if (this.server) {
await new Promise<void>(resolve => {
this.server!.close(() => resolve())
})
this.server = null
}
this.emit('stopped')
}
/**
* Register a node endpoint
*/
registerEndpoint(nodeId: string, endpoint: string): void {
this.endpoints.set(nodeId, endpoint)
this.emit('endpointRegistered', { nodeId, endpoint })
}
/**
* Register RPC method handler
*/
registerHandler(method: string, handler: (params: any, from: string) => Promise<any>): void {
this.messageHandlers.set(method, handler)
}
/**
* Call RPC method on remote node
*/
async call(nodeId: string, method: string, params: any): Promise<any> {
const endpoint = this.endpoints.get(nodeId)
if (!endpoint) {
throw new Error(`No endpoint registered for node ${nodeId}`)
}
const message: TransportMessage = {
id: this.generateId(),
method,
params,
timestamp: Date.now(),
from: this.nodeId,
to: nodeId
}
// Send HTTP request
const response = await this.sendHTTPRequest(endpoint, '/rpc', message)
if (response.error) {
throw new Error(response.error.message)
}
return response.result
}
/**
* Broadcast to all SSE clients
*/
broadcast(event: string, data: any): void {
const message = JSON.stringify({ event, data, timestamp: Date.now() })
for (const [clientId, client] of this.sseClients.entries()) {
try {
client.response.write(`data: ${message}\n\n`)
} catch (err) {
// Client disconnected
console.debug(`[${this.nodeId}] SSE client ${clientId} disconnected`)
this.sseClients.delete(clientId)
}
}
}
/**
* Handle health check
*/
private async handleHealth(req: http.IncomingMessage, res: http.ServerResponse): Promise<void> {
res.writeHead(200, { 'Content-Type': 'application/json' })
res.end(JSON.stringify({
status: 'healthy',
nodeId: this.nodeId,
uptime: process.uptime(),
memory: process.memoryUsage(),
connections: this.sseClients.size
}))
}
/**
* Handle RPC requests
*/
private async handleRPC(req: http.IncomingMessage, res: http.ServerResponse): Promise<void> {
if (req.method !== 'POST') {
res.writeHead(405)
res.end('Method Not Allowed')
return
}
// Read body
const body = await this.readBody(req)
let message: TransportMessage
try {
message = JSON.parse(body)
} catch (err) {
res.writeHead(400)
res.end('Invalid JSON')
return
}
// Handle the RPC call
const response: TransportResponse = {
id: message.id,
timestamp: Date.now()
}
try {
const handler = this.messageHandlers.get(message.method)
if (!handler) {
throw new Error(`Method ${message.method} not found`)
}
response.result = await handler(message.params, message.from)
} catch (err: any) {
response.error = {
code: -32603,
message: err.message,
data: err.stack
}
}
res.writeHead(200, { 'Content-Type': 'application/json' })
res.end(JSON.stringify(response))
}
/**
* Handle SSE connections for real-time updates
*/
private async handleSSE(req: http.IncomingMessage, res: http.ServerResponse): Promise<void> {
// Setup SSE headers
res.writeHead(200, {
'Content-Type': 'text/event-stream',
'Cache-Control': 'no-cache',
'Connection': 'keep-alive',
'X-Accel-Buffering': 'no' // Disable Nginx buffering
})
// Send initial connection event
const clientId = this.generateId()
res.write(`data: ${JSON.stringify({
event: 'connected',
clientId,
nodeId: this.nodeId
})}\n\n`)
// Store client
this.sseClients.set(clientId, {
id: clientId,
response: res,
lastPing: Date.now()
})
// Handle client disconnect
req.on('close', () => {
this.sseClients.delete(clientId)
this.emit('sseDisconnected', clientId)
})
this.emit('sseConnected', clientId)
}
/**
* Handle streaming data (for shard migration)
*/
private async handleStream(
req: http.IncomingMessage,
res: http.ServerResponse,
url: URL
): Promise<void> {
const streamId = url.pathname.split('/').pop()
if (req.method === 'POST') {
// Receiving stream
await this.handleStreamUpload(req, res, streamId!)
} else if (req.method === 'GET') {
// Sending stream
await this.handleStreamDownload(req, res, streamId!)
} else {
res.writeHead(405)
res.end('Method Not Allowed')
}
}
/**
* Handle stream upload (receiving data)
*/
private async handleStreamUpload(
req: http.IncomingMessage,
res: http.ServerResponse,
streamId: string
): Promise<void> {
const chunks: Buffer[] = []
let totalSize = 0
req.on('data', (chunk: Buffer) => {
chunks.push(chunk)
totalSize += chunk.length
// Emit progress
this.emit('streamProgress', {
streamId,
type: 'upload',
bytes: totalSize
})
})
req.on('end', () => {
const data = Buffer.concat(chunks)
// Process the streamed data
this.emit('streamComplete', {
streamId,
type: 'upload',
data,
size: totalSize
})
res.writeHead(200, { 'Content-Type': 'application/json' })
res.end(JSON.stringify({
success: true,
streamId,
size: totalSize
}))
})
req.on('error', (err) => {
console.error(`[${this.nodeId}] Stream upload error:`, err)
res.writeHead(500)
res.end('Stream Error')
})
}
/**
* Handle stream download (sending data)
*/
private async handleStreamDownload(
req: http.IncomingMessage,
res: http.ServerResponse,
streamId: string
): Promise<void> {
// Stream download not yet implemented
// Return error response instead of fake data
res.writeHead(501, {
'Content-Type': 'application/json'
})
res.end(JSON.stringify({
error: 'Stream download not implemented',
message: 'This feature is not yet available in the current version',
streamId
}))
this.emit('streamError', {
streamId,
type: 'download',
error: 'Not implemented'
})
}
/**
* Send HTTP request to another node
*/
private async sendHTTPRequest(
endpoint: string,
path: string,
data: any
): Promise<TransportResponse> {
return new Promise((resolve, reject) => {
const url = new URL(path, endpoint)
const options = {
method: 'POST',
headers: {
'Content-Type': 'application/json'
}
}
const proto = url.protocol === 'https:' ? https : http
const req = proto.request(url, options, (res) => {
let body = ''
res.on('data', chunk => body += chunk)
res.on('end', () => {
try {
const response = JSON.parse(body)
resolve(response)
} catch (err) {
reject(new Error(`Invalid response: ${body}`))
}
})
})
req.on('error', reject)
req.on('timeout', () => {
req.destroy()
reject(new Error('Request timeout'))
})
req.setTimeout(this.REQUEST_TIMEOUT)
req.write(JSON.stringify(data))
req.end()
})
}
/**
* Read request body
*/
private readBody(req: http.IncomingMessage): Promise<string> {
return new Promise((resolve, reject) => {
let body = ''
req.on('data', chunk => body += chunk)
req.on('end', () => resolve(body))
req.on('error', reject)
})
}
/**
* Find an available port
*/
private async findAvailablePort(startPort: number = 7947): Promise<number> {
const checkPort = (port: number): Promise<boolean> => {
return new Promise(resolve => {
const server = net.createServer()
server.once('error', () => resolve(false))
server.once('listening', () => {
server.close()
resolve(true)
})
server.listen(port)
})
}
// Try preferred port first
if (await checkPort(startPort)) {
return startPort
}
// Find random available port
return new Promise((resolve, reject) => {
const server = net.createServer()
server.once('error', reject)
server.once('listening', () => {
const port = (server.address() as net.AddressInfo).port
server.close(() => resolve(port))
})
server.listen(0) // 0 = random available port
})
}
/**
* Start SSE heartbeat to keep connections alive
*/
private startSSEHeartbeat(): void {
this.sseHeartbeatTimer = setInterval(() => {
const now = Date.now()
const ping = JSON.stringify({ event: 'ping', timestamp: now })
for (const [clientId, client] of this.sseClients.entries()) {
try {
client.response.write(`: ping\n\n`) // SSE comment for keep-alive
client.lastPing = now
} catch (err) {
// Client disconnected
this.sseClients.delete(clientId)
}
}
}, this.SSE_HEARTBEAT_INTERVAL)
}
/**
* Generate unique ID
*/
private generateId(): string {
return `${Date.now()}-${Math.random().toString(36).substring(2, 11)}`
}
/**
* Get connected nodes count
*/
getConnectionCount(): number {
return this.endpoints.size
}
/**
* Get SSE client count
*/
getSSEClientCount(): number {
return this.sseClients.size
}
}

54
src/distributed/index.ts
View file

@ -1,54 +0,0 @@
/**
* Distributed module exports
*/
export { DistributedConfigManager } from './configManager.js'
export { HashPartitioner, AffinityPartitioner } from './hashPartitioner.js'
export {
BaseOperationalMode,
ReaderMode,
WriterMode,
HybridMode,
OperationalModeFactory
} from './operationalModes.js'
export { DomainDetector } from './domainDetector.js'
export { HealthMonitor } from './healthMonitor.js'
// New distributed scaling components
export { DistributedCoordinator, createCoordinator } from './coordinator.js'
export { ShardManager, createShardManager } from './shardManager.js'
export { CacheSync, createCacheSync } from './cacheSync.js'
export { ReadWriteSeparation, createReadWriteSeparation } from './readWriteSeparation.js'
export type {
HealthMetrics,
HealthStatus
} from './healthMonitor.js'
export type {
DomainPattern
} from './domainDetector.js'
export type {
NodeInfo,
CoordinatorConfig,
ConsensusState
} from './coordinator.js'
export type {
ShardConfig,
Shard,
ShardAssignment
} from './shardManager.js'
export type {
CacheSyncConfig,
CacheEntry,
SyncMessage
} from './cacheSync.js'
export type {
ReplicationConfig,
WriteOperation,
ReplicationLog
} from './readWriteSeparation.js'

739
src/distributed/networkTransport.ts
View file

@ -1,739 +0,0 @@
/**
* Network Transport Layer for Distributed Brainy
* Uses WebSocket + HTTP for maximum compatibility
*/
import * as http from 'node:http'
import { EventEmitter } from 'node:events'
import { WebSocket } from 'ws'
// Use dynamic imports for Node.js specific modules
let WebSocketServer: any
// Default ports
const HTTP_PORT = process.env.BRAINY_HTTP_PORT || 7947
const WS_PORT = process.env.BRAINY_WS_PORT || 7948
export interface NetworkMessage {
type: string
from: string
to?: string
data: any
timestamp: number
id: string
}
export interface NodeEndpoint {
nodeId: string
host: string
httpPort: number
wsPort: number
lastSeen: number
}
export interface NetworkConfig {
nodeId?: string
host?: string
httpPort?: number
wsPort?: number
seeds?: string[] // Known node addresses for bootstrap
discoveryMethod?: 'seeds' | 'dns' | 'kubernetes' | 'auto'
enableUDP?: boolean // Optional UDP discovery for LAN
}
/**
* Production-ready network transport
*/
export class NetworkTransport extends EventEmitter {
private nodeId: string
private config: NetworkConfig
private httpServer?: http.Server
private wsServer: any
private peers: Map<string, NodeEndpoint> = new Map()
private connections: Map<string, WebSocket> = new Map()
private messageHandlers: Map<string, (msg: NetworkMessage) => Promise<any>> = new Map()
private responseHandlers: Map<string, (response: any) => void> = new Map()
private isRunning = false
constructor(config: NetworkConfig) {
super()
this.nodeId = config.nodeId || this.generateNodeId()
this.config = {
host: '0.0.0.0',
httpPort: Number(HTTP_PORT),
wsPort: Number(WS_PORT),
discoveryMethod: 'auto',
enableUDP: false, // Disabled by default for cloud compatibility
...config
}
}
/**
* Start network transport
*/
async start(): Promise<void> {
if (this.isRunning) return
const ws = await import('ws')
// The ws package has exported its server class as `WebSocketServer`
// (v8+), `Server` (older releases) and via the default export — and the
// available type declarations disagree on which of those are value
// exports, so probe all three through an optional-keyed module view.
const wsModule = ws as typeof ws & { WebSocketServer?: unknown; Server?: unknown }
WebSocketServer = wsModule.WebSocketServer || wsModule.Server || ws.default
await this.startHTTPServer()
await this.startWebSocketServer()
await this.discoverPeers()
this.isRunning = true
this.emit('started', { nodeId: this.nodeId })
// Start heartbeat
this.startHeartbeat()
}
/**
* Stop network transport
*/
async stop(): Promise<void> {
if (!this.isRunning) return
this.isRunning = false
// Close all connections
for (const ws of this.connections.values()) {
ws.close()
}
this.connections.clear()
// Stop servers
if (this.httpServer) {
await new Promise<void>((resolve) => {
this.httpServer!.close(() => resolve())
})
}
if (this.wsServer) {
await new Promise<void>((resolve) => {
this.wsServer.close(() => resolve())
})
}
this.emit('stopped')
}
/**
* Start HTTP server for REST API and health checks
*/
private async startHTTPServer(): Promise<void> {
return new Promise((resolve, reject) => {
this.httpServer = http.createServer(async (req, res) => {
// Enable CORS
res.setHeader('Access-Control-Allow-Origin', '*')
res.setHeader('Access-Control-Allow-Methods', 'GET, POST, OPTIONS')
res.setHeader('Access-Control-Allow-Headers', 'Content-Type')
if (req.method === 'OPTIONS') {
res.writeHead(200)
res.end()
return
}
if (req.url === '/health') {
// Health check endpoint
res.writeHead(200, { 'Content-Type': 'application/json' })
res.end(JSON.stringify({
status: 'healthy',
nodeId: this.nodeId,
peers: Array.from(this.peers.keys()),
connections: Array.from(this.connections.keys())
}))
return
}
if (req.url === '/peers') {
// Peer discovery endpoint
res.writeHead(200, { 'Content-Type': 'application/json' })
res.end(JSON.stringify({
nodeId: this.nodeId,
endpoint: {
host: this.config.host,
httpPort: this.config.httpPort,
wsPort: this.config.wsPort
},
peers: Array.from(this.peers.values())
}))
return
}
if (req.method === 'POST' && req.url === '/message') {
// Message handling endpoint
let body = ''
req.on('data', (chunk) => {
body += chunk.toString()
})
req.on('end', async () => {
try {
const message: NetworkMessage = JSON.parse(body)
// Handle message
const handler = this.messageHandlers.get(message.type)
if (handler) {
const response = await handler(message)
res.writeHead(200, { 'Content-Type': 'application/json' })
res.end(JSON.stringify({ success: true, data: response }))
} else {
res.writeHead(404)
res.end(JSON.stringify({ success: false, error: 'Unknown message type' }))
}
} catch (err: any) {
res.writeHead(500)
res.end(JSON.stringify({ success: false, error: err.message }))
}
})
} else {
res.writeHead(404)
res.end(JSON.stringify({ error: 'Not found' }))
}
})
this.httpServer.listen(this.config.httpPort, '0.0.0.0', () => {
console.log(`[Network] HTTP server listening on :${this.config.httpPort}`)
resolve()
})
this.httpServer.on('error', reject)
})
}
/**
* Start WebSocket server for real-time communication
*/
private async startWebSocketServer(): Promise<void> {
if (!WebSocketServer) {
console.warn('[Network] WebSocket not available in this environment')
return
}
this.wsServer = new WebSocketServer({ port: this.config.wsPort })
this.wsServer.on('connection', (ws: WebSocket) => {
let peerId: string | null = null
ws.on('message', async (data: Buffer | string) => {
try {
const message: NetworkMessage = JSON.parse(data.toString())
// Handle handshake
if (message.type === 'HANDSHAKE') {
peerId = message.from
this.connections.set(peerId, ws)
// Add to peers
const endpoint: NodeEndpoint = {
nodeId: peerId,
host: message.data.host || 'unknown',
httpPort: message.data.httpPort || 0,
wsPort: message.data.wsPort || 0,
lastSeen: Date.now()
}
this.peers.set(peerId, endpoint)
// Send handshake response
ws.send(JSON.stringify({
type: 'HANDSHAKE_ACK',
from: this.nodeId,
to: peerId,
data: {
nodeId: this.nodeId,
host: this.config.host,
httpPort: this.config.httpPort,
wsPort: this.config.wsPort
},
timestamp: Date.now(),
id: this.generateMessageId()
}))
this.emit('peerConnected', peerId)
return
}
// Handle response messages
if (message.type.endsWith('_RESPONSE')) {
const handler = this.responseHandlers.get(message.id)
if (handler) {
handler(message.data)
this.responseHandlers.delete(message.id)
}
return
}
// Handle regular messages
const handler = this.messageHandlers.get(message.type)
if (handler) {
const response = await handler(message)
if (response !== undefined) {
ws.send(JSON.stringify({
type: `${message.type}_RESPONSE`,
from: this.nodeId,
to: message.from,
data: response,
timestamp: Date.now(),
id: message.id
}))
}
}
} catch (err) {
console.error('[Network] WebSocket message error:', err)
}
})
ws.on('close', () => {
if (peerId) {
this.connections.delete(peerId)
this.emit('peerDisconnected', peerId)
}
})
ws.on('error', (err: Error) => {
console.error(`[Network] WebSocket error:`, err.message)
})
})
console.log(`[Network] WebSocket server listening on :${this.config.wsPort}`)
}
/**
* Discover peers based on configuration
*/
private async discoverPeers(): Promise<void> {
const method = this.config.discoveryMethod
if (method === 'seeds' || (method === 'auto' && this.config.seeds)) {
// Use seed nodes
await this.connectToSeeds()
} else if (method === 'dns' || (method === 'auto' && process.env.BRAINY_DNS)) {
// Use DNS discovery
await this.discoverViaDNS()
} else if (method === 'kubernetes' || (method === 'auto' && process.env.KUBERNETES_SERVICE_HOST)) {
// Use Kubernetes service discovery
await this.discoverViaKubernetes()
}
// Fallback: try localhost for development
if (this.peers.size === 0 && process.env.NODE_ENV !== 'production') {
await this.connectToNode('localhost', this.config.httpPort!)
}
}
/**
* Connect to seed nodes
*/
private async connectToSeeds(): Promise<void> {
if (!this.config.seeds) return
for (const seed of this.config.seeds) {
try {
// Parse seed address (host:port or just host)
const [host, port] = seed.split(':')
await this.connectToNode(host, Number(port) || this.config.httpPort!)
} catch (err) {
console.error(`[Network] Failed to connect to seed ${seed}:`, err)
}
}
}
/**
* Discover peers via DNS
*/
private async discoverViaDNS(): Promise<void> {
const dnsName = process.env.BRAINY_DNS || 'brainy-cluster.local'
try {
const dns = await import('dns')
const addresses = await new Promise<string[]>((resolve, reject) => {
dns.resolve4(dnsName, (err, addresses) => {
if (err) reject(err)
else resolve(addresses || [])
})
})
for (const address of addresses) {
await this.connectToNode(address, this.config.httpPort!)
}
} catch (err) {
console.log(`[Network] DNS discovery failed for ${dnsName}:`, err)
}
}
/**
* Discover peers via Kubernetes
*/
private async discoverViaKubernetes(): Promise<void> {
const serviceName = process.env.BRAINY_SERVICE || 'brainy'
const namespace = process.env.BRAINY_NAMESPACE || 'default'
const apiServer = 'https://kubernetes.default.svc'
const token = process.env.KUBERNETES_TOKEN || ''
try {
// Query Kubernetes API for pod endpoints
const https = await import('node:https')
const response = await new Promise<any>((resolve, reject) => {
https.get(
`${apiServer}/api/v1/namespaces/${namespace}/endpoints/${serviceName}`,
{
headers: {
'Authorization': `Bearer ${token}`
}
},
(res) => {
let data = ''
res.on('data', (chunk) => data += chunk)
res.on('end', () => resolve(JSON.parse(data)))
}
).on('error', reject)
})
// Connect to each pod
if (response.subsets) {
for (const subset of response.subsets) {
for (const address of subset.addresses || []) {
await this.connectToNode(address.ip, this.config.httpPort!)
}
}
}
} catch (err) {
console.log('[Network] Kubernetes discovery failed:', err)
}
}
/**
* Connect to a specific node
*/
private async connectToNode(host: string, httpPort: number): Promise<void> {
try {
// First, get node info via HTTP
const nodeInfo = await this.getNodeInfo(host, httpPort)
if (nodeInfo.nodeId === this.nodeId) {
return // Don't connect to self
}
// Add to peers
const endpoint: NodeEndpoint = {
nodeId: nodeInfo.nodeId,
host,
httpPort,
wsPort: nodeInfo.endpoint.wsPort,
lastSeen: Date.now()
}
this.peers.set(nodeInfo.nodeId, endpoint)
// Connect via WebSocket
await this.connectWebSocket(endpoint)
// Get their peer list
for (const peer of nodeInfo.peers || []) {
if (!this.peers.has(peer.nodeId) && peer.nodeId !== this.nodeId) {
this.peers.set(peer.nodeId, peer)
// Optionally connect to them too
if (this.peers.size < 10) { // Limit connections
await this.connectWebSocket(peer)
}
}
}
} catch (err) {
// Node might be down or not ready
console.debug(`[Network] Could not connect to ${host}:${httpPort}`)
}
}
/**
* Get node information via HTTP
*/
private async getNodeInfo(host: string, port: number): Promise<any> {
return new Promise((resolve, reject) => {
const req = http.get(`http://${host}:${port}/peers`, (res) => {
let data = ''
res.on('data', (chunk) => data += chunk)
res.on('end', () => {
try {
resolve(JSON.parse(data))
} catch (err) {
reject(err)
}
})
})
req.on('error', reject)
req.setTimeout(2000, () => {
req.destroy()
reject(new Error('Timeout'))
})
})
}
/**
* Connect to peer via WebSocket
*/
private async connectWebSocket(endpoint: NodeEndpoint): Promise<void> {
if (this.connections.has(endpoint.nodeId)) return
try {
const ws = new WebSocket(`ws://${endpoint.host}:${endpoint.wsPort}`)
ws.on('open', () => {
// Send handshake
ws.send(JSON.stringify({
type: 'HANDSHAKE',
from: this.nodeId,
data: {
nodeId: this.nodeId,
host: this.config.host,
httpPort: this.config.httpPort,
wsPort: this.config.wsPort
},
timestamp: Date.now(),
id: this.generateMessageId()
}))
this.connections.set(endpoint.nodeId, ws)
})
ws.on('message', async (data: Buffer | string) => {
try {
const message: NetworkMessage = JSON.parse(data.toString())
// Handle responses
if (message.type.endsWith('_RESPONSE')) {
const handler = this.responseHandlers.get(message.id)
if (handler) {
handler(message.data)
this.responseHandlers.delete(message.id)
}
return
}
// Handle messages
const handler = this.messageHandlers.get(message.type)
if (handler) {
await handler(message)
}
} catch (err) {
console.error('[Network] Message handling error:', err)
}
})
ws.on('close', () => {
this.connections.delete(endpoint.nodeId)
})
ws.on('error', (err: Error) => {
console.debug(`[Network] WebSocket error with ${endpoint.nodeId}:`, err.message)
})
} catch (err) {
console.debug(`[Network] Failed to connect to ${endpoint.nodeId}`)
}
}
/**
* Start heartbeat to maintain connections
*/
private startHeartbeat(): void {
setInterval(() => {
if (!this.isRunning) return
// Send heartbeat to all connected peers
for (const [nodeId, ws] of this.connections.entries()) {
if (ws.readyState === WebSocket.OPEN) {
ws.send(JSON.stringify({
type: 'HEARTBEAT',
from: this.nodeId,
to: nodeId,
timestamp: Date.now(),
id: this.generateMessageId()
}))
} else {
// Connection lost, remove it
this.connections.delete(nodeId)
}
}
// Clean up stale peers
const now = Date.now()
for (const [nodeId, peer] of this.peers.entries()) {
if (now - peer.lastSeen > 60000) { // 60 seconds
this.peers.delete(nodeId)
}
}
}, 10000) // Every 10 seconds
}
/**
* Send message to specific node
*/
async sendToNode(nodeId: string, type: string, data: any): Promise<any> {
const ws = this.connections.get(nodeId)
if (ws && ws.readyState === WebSocket.OPEN) {
// Use WebSocket
return new Promise((resolve, reject) => {
const messageId = this.generateMessageId()
const timeout = setTimeout(() => {
this.responseHandlers.delete(messageId)
reject(new Error(`Timeout waiting for response from ${nodeId}`))
}, 5000)
this.responseHandlers.set(messageId, (response) => {
clearTimeout(timeout)
resolve(response)
})
ws.send(JSON.stringify({
type,
from: this.nodeId,
to: nodeId,
data,
timestamp: Date.now(),
id: messageId
}))
})
} else {
// Use HTTP fallback
const endpoint = this.peers.get(nodeId)
if (!endpoint) {
throw new Error(`Unknown node: ${nodeId}`)
}
return this.sendViaHTTP(endpoint, type, data)
}
}
/**
* Send via HTTP
*/
private async sendViaHTTP(endpoint: NodeEndpoint, type: string, data: any): Promise<any> {
return new Promise((resolve, reject) => {
const message: NetworkMessage = {
type,
from: this.nodeId,
to: endpoint.nodeId,
data,
timestamp: Date.now(),
id: this.generateMessageId()
}
const postData = JSON.stringify(message)
const req = http.request({
hostname: endpoint.host,
port: endpoint.httpPort,
path: '/message',
method: 'POST',
headers: {
'Content-Type': 'application/json',
'Content-Length': Buffer.byteLength(postData)
}
}, (res) => {
let responseData = ''
res.on('data', (chunk) => {
responseData += chunk
})
res.on('end', () => {
try {
const response = JSON.parse(responseData)
if (response.success) {
resolve(response.data)
} else {
reject(new Error(response.error))
}
} catch (err) {
reject(err)
}
})
})
req.on('error', reject)
req.setTimeout(5000, () => {
req.destroy()
reject(new Error('HTTP timeout'))
})
req.write(postData)
req.end()
})
}
/**
* Broadcast to all peers
*/
async broadcast(type: string, data: any): Promise<void> {
const promises = []
for (const nodeId of this.peers.keys()) {
promises.push(
this.sendToNode(nodeId, type, data).catch(() => {
// Ignore broadcast failures
})
)
}
await Promise.all(promises)
}
/**
* Register message handler
*/
onMessage(type: string, handler: (msg: NetworkMessage) => Promise<any>): void {
this.messageHandlers.set(type, handler)
}
/**
* Get connected peers
*/
getPeers(): NodeEndpoint[] {
return Array.from(this.peers.values())
}
/**
* Check if connected
*/
isConnected(nodeId: string): boolean {
const ws = this.connections.get(nodeId)
return ws !== undefined && ws.readyState === WebSocket.OPEN
}
/**
* Generate node ID
*/
private generateNodeId(): string {
return `node-${Date.now()}-${Math.random().toString(36).substring(2, 11)}`
}
/**
* Generate message ID
*/
private generateMessageId(): string {
return `${this.nodeId}-${Date.now()}-${Math.random().toString(36).substring(2, 9)}`
}
/**
* Get node ID
*/
getNodeId(): string {
return this.nodeId
}
}
/**
* Create network transport
*/
export function createNetworkTransport(config: NetworkConfig): NetworkTransport {
return new NetworkTransport(config)
}

220
src/distributed/operationalModes.ts
View file

@ -1,220 +0,0 @@
/**
* Operational Modes for Distributed Brainy
* Defines different modes with optimized caching strategies
*/
import {
OperationalMode,
CacheStrategy,
InstanceRole
} from '../types/distributedTypes.js'
/**
* Base operational mode
*/
export abstract class BaseOperationalMode implements OperationalMode {
abstract canRead: boolean
abstract canWrite: boolean
abstract canDelete: boolean
abstract cacheStrategy: CacheStrategy
/**
* Validate operation is allowed in this mode
*/
validateOperation(operation: 'read' | 'write' | 'delete'): void {
switch (operation) {
case 'read':
if (!this.canRead) {
throw new Error('Read operations are not allowed in write-only mode')
}
break
case 'write':
if (!this.canWrite) {
throw new Error('Write operations are not allowed in read-only mode')
}
break
case 'delete':
if (!this.canDelete) {
throw new Error('Delete operations are not allowed in this mode')
}
break
}
}
}
/**
* Read-only mode optimized for query performance
*/
export class ReaderMode extends BaseOperationalMode {
canRead = true
canWrite = false
canDelete = false
cacheStrategy: CacheStrategy = {
hotCacheRatio: 0.8, // 80% of memory for read cache
prefetchAggressive: true, // Aggressively prefetch related vectors
ttl: 3600000, // 1 hour cache TTL
compressionEnabled: true, // Trade CPU for more cache capacity
writeBufferSize: 0, // No write buffer needed
batchWrites: false, // No writes
adaptive: true // Adapt to query patterns
}
/**
* Get optimized cache configuration for readers
*/
getCacheConfig() {
return {
hotCacheMaxSize: 1000000, // Large hot cache
hotCacheEvictionThreshold: 0.9, // Keep cache full
warmCacheTTL: 3600000, // 1 hour warm cache
batchSize: 100, // Large batch reads
autoTune: true, // Auto-tune for read patterns
autoTuneInterval: 60000, // Tune every minute
readOnly: true // Enable read-only optimizations
}
}
}
/**
* Write-only mode optimized for ingestion
*/
export class WriterMode extends BaseOperationalMode {
canRead = false
canWrite = true
canDelete = true
cacheStrategy: CacheStrategy = {
hotCacheRatio: 0.2, // Only 20% for cache, rest for write buffer
prefetchAggressive: false, // No prefetching needed
ttl: 60000, // Short TTL (1 minute)
compressionEnabled: false, // Speed over memory efficiency
writeBufferSize: 10000, // Large write buffer for batching
batchWrites: true, // Enable write batching
adaptive: false // Fixed strategy for consistent writes
}
/**
* Get optimized cache configuration for writers
*/
getCacheConfig() {
return {
hotCacheMaxSize: 100000, // Small hot cache
hotCacheEvictionThreshold: 0.5, // Aggressive eviction
warmCacheTTL: 60000, // 1 minute warm cache
batchSize: 1000, // Large batch writes
autoTune: false, // Fixed configuration
writeOnly: true // Enable write-only optimizations
}
}
}
/**
* Hybrid mode that can both read and write
*/
export class HybridMode extends BaseOperationalMode {
canRead = true
canWrite = true
canDelete = true
cacheStrategy: CacheStrategy = {
hotCacheRatio: 0.5, // Balanced cache/buffer allocation
prefetchAggressive: false, // Moderate prefetching
ttl: 600000, // 10 minute TTL
compressionEnabled: true, // Compress when beneficial
writeBufferSize: 5000, // Moderate write buffer
batchWrites: true, // Batch writes when possible
adaptive: true // Adapt to workload mix
}
private readWriteRatio: number = 0.5 // Track read/write ratio
/**
* Get balanced cache configuration
*/
getCacheConfig() {
return {
hotCacheMaxSize: 500000, // Medium cache size
hotCacheEvictionThreshold: 0.7, // Balanced eviction
warmCacheTTL: 600000, // 10 minute warm cache
batchSize: 500, // Medium batch size
autoTune: true, // Auto-tune based on workload
autoTuneInterval: 300000 // Tune every 5 minutes
}
}
/**
* Update cache strategy based on workload
* @param readCount - Number of recent reads
* @param writeCount - Number of recent writes
*/
updateWorkloadBalance(readCount: number, writeCount: number): void {
const total = readCount + writeCount
if (total === 0) return
this.readWriteRatio = readCount / total
// Adjust cache strategy based on workload
if (this.readWriteRatio > 0.8) {
// Read-heavy workload
this.cacheStrategy.hotCacheRatio = 0.7
this.cacheStrategy.prefetchAggressive = true
this.cacheStrategy.writeBufferSize = 2000
} else if (this.readWriteRatio < 0.2) {
// Write-heavy workload
this.cacheStrategy.hotCacheRatio = 0.3
this.cacheStrategy.prefetchAggressive = false
this.cacheStrategy.writeBufferSize = 8000
} else {
// Balanced workload
this.cacheStrategy.hotCacheRatio = 0.5
this.cacheStrategy.prefetchAggressive = false
this.cacheStrategy.writeBufferSize = 5000
}
}
}
/**
* Factory for creating operational modes
*/
export class OperationalModeFactory {
/**
* Create operational mode based on role
* @param role - The instance role
* @returns The appropriate operational mode
*/
static createMode(role: InstanceRole): BaseOperationalMode {
switch (role) {
case 'reader':
return new ReaderMode()
case 'writer':
return new WriterMode()
case 'hybrid':
return new HybridMode()
default:
// Default to reader for safety
return new ReaderMode()
}
}
/**
* Create mode with custom cache strategy
* @param role - The instance role
* @param customStrategy - Custom cache strategy overrides
* @returns The operational mode with custom strategy
*/
static createModeWithStrategy(
role: InstanceRole,
customStrategy: Partial<CacheStrategy>
): BaseOperationalMode {
const mode = this.createMode(role)
// Apply custom strategy overrides
mode.cacheStrategy = {
...mode.cacheStrategy,
...customStrategy
}
return mode
}
}

425
src/distributed/queryPlanner.ts
View file

@ -1,425 +0,0 @@
/**
* Distributed Query Planner for Brainy 3.0
*
* Intelligently plans and executes distributed queries across shards
* Optimizes for data locality, parallelism, and network efficiency
*/
import type { StorageAdapter } from '../coreTypes.js'
import type { DistributedCoordinator } from './coordinator.js'
import type { ShardManager } from './shardManager.js'
import type { HTTPTransport } from './httpTransport.js'
import type { TripleIntelligenceSystem } from '../triple/TripleIntelligenceSystem.js'
export interface QueryPlan {
/**
* Shards that need to be queried
*/
shards: string[]
/**
* Nodes responsible for each shard
*/
nodeAssignments: Map<string, string[]>
/**
* Whether query can be parallelized
*/
parallel: boolean
/**
* Estimated cost (0-1000)
*/
cost: number
/**
* Query strategy
*/
strategy: 'broadcast' | 'targeted' | 'scatter-gather' | 'local-only'
}
export interface DistributedQueryResult {
results: any[]
totalCount: number
executionTime: number
nodeStats: Map<string, {
resultsReturned: number
executionTime: number
errors?: string[]
}>
}
export class DistributedQueryPlanner {
private nodeId: string
private coordinator: DistributedCoordinator
private shardManager: ShardManager
private transport: HTTPTransport
private tripleEngine?: TripleIntelligenceSystem
private storage: StorageAdapter
constructor(
nodeId: string,
coordinator: DistributedCoordinator,
shardManager: ShardManager,
transport: HTTPTransport,
storage: StorageAdapter,
tripleEngine?: TripleIntelligenceSystem
) {
this.nodeId = nodeId
this.coordinator = coordinator
this.shardManager = shardManager
this.transport = transport
this.storage = storage
this.tripleEngine = tripleEngine
}
/**
* Plan a distributed query
*/
async planQuery(query: any): Promise<QueryPlan> {
// Determine query type and scope
const queryType = this.getQueryType(query)
const affectedShards = await this.determineAffectedShards(query)
// Get current shard assignments
const assignments = new Map<string, string[]>()
for (const shardId of affectedShards) {
const nodes = await this.shardManager.getNodesForShard(shardId)
assignments.set(shardId, nodes)
}
// Determine strategy based on query characteristics
let strategy: QueryPlan['strategy'] = 'scatter-gather'
let cost = 0
if (affectedShards.length === 0) {
// Local only query
strategy = 'local-only'
cost = 1
} else if (affectedShards.length === this.shardManager.getTotalShards()) {
// Full table scan
strategy = 'broadcast'
cost = 1000
} else if (affectedShards.length <= 3) {
// Targeted query
strategy = 'targeted'
cost = affectedShards.length * 10
} else {
// Scatter-gather for medium queries
strategy = 'scatter-gather'
cost = affectedShards.length * 50
}
// Add network cost
const remoteShards = affectedShards.filter(shardId => {
const nodes = assignments.get(shardId) || []
return !nodes.includes(this.nodeId)
})
cost += remoteShards.length * 20
return {
shards: affectedShards,
nodeAssignments: assignments,
parallel: affectedShards.length > 1,
cost,
strategy
}
}
/**
* Execute a distributed query based on plan
*/
async executeQuery(query: any, plan: QueryPlan): Promise<DistributedQueryResult> {
const startTime = Date.now()
const nodeStats = new Map<string, any>()
// Group shards by node for efficient batching
const nodeToShards = new Map<string, string[]>()
for (const [shardId, nodes] of plan.nodeAssignments) {
// Pick the first available node (could be optimized)
const targetNode = nodes[0]
if (!nodeToShards.has(targetNode)) {
nodeToShards.set(targetNode, [])
}
nodeToShards.get(targetNode)!.push(shardId)
}
// Execute queries in parallel
const promises: Promise<any>[] = []
for (const [targetNode, shards] of nodeToShards) {
if (targetNode === this.nodeId) {
// Local execution
promises.push(this.executeLocalQuery(query, shards))
} else {
// Remote execution
promises.push(this.executeRemoteQuery(targetNode, query, shards))
}
}
// Wait for all results
const results = await Promise.allSettled(promises)
// Aggregate results
const allResults: any[] = []
let totalCount = 0
let nodeIndex = 0
for (const [targetNode, shards] of nodeToShards) {
const result = results[nodeIndex]
const nodeTime = Date.now() - startTime
if (result.status === 'fulfilled') {
const nodeResult = result.value
allResults.push(...(nodeResult.results || []))
totalCount += nodeResult.count || 0
nodeStats.set(targetNode, {
resultsReturned: nodeResult.count || 0,
executionTime: nodeTime,
shards
})
} else {
nodeStats.set(targetNode, {
resultsReturned: 0,
executionTime: nodeTime,
errors: [result.reason?.message || 'Unknown error'],
shards
})
}
nodeIndex++
}
// Merge and rank results using Triple Intelligence if available
const mergedResults = await this.mergeResults(allResults, query)
return {
results: mergedResults,
totalCount,
executionTime: Date.now() - startTime,
nodeStats
}
}
/**
* Execute query on local shards
*/
private async executeLocalQuery(query: any, shards: string[]): Promise<any> {
const results: any[] = []
for (const shardId of shards) {
// Get data from storage for this shard
const shardData = await this.getShardData(shardId, query)
results.push(...shardData)
}
return {
results,
count: results.length
}
}
/**
* Execute query on remote node
*/
private async executeRemoteQuery(
targetNode: string,
query: any,
shards: string[]
): Promise<any> {
try {
const response = await this.transport.call(targetNode, 'query', {
query,
shards
})
return response || { results: [], count: 0 }
} catch (error) {
console.error(`Failed to query node ${targetNode}:`, error)
throw error
}
}
/**
* Get data from a specific shard
*/
private async getShardData(shardId: string, query: any): Promise<any[]> {
// This would interact with the actual storage adapter
// For now, return empty array since storage adapters don't have direct shard access
// In a real implementation, this would use storage-specific methods
try {
// Would need to implement shard-aware storage methods
// For now, return empty to allow compilation
return []
} catch (error) {
console.error(`Failed to get shard ${shardId} data:`, error)
return []
}
}
/**
* Filter data based on query criteria
*/
private filterData(data: any, query: any): any[] {
if (!Array.isArray(data)) return []
// Apply query filters
let filtered = data
if (query.filter) {
filtered = filtered.filter((item: any) => {
// Apply filter logic
return this.matchesFilter(item, query.filter)
})
}
if (query.limit) {
filtered = filtered.slice(0, query.limit)
}
return filtered
}
/**
* Check if item matches filter
*/
private matchesFilter(item: any, filter: any): boolean {
// Simple filter matching
for (const [key, value] of Object.entries(filter)) {
if (item[key] !== value) {
return false
}
}
return true
}
/**
* Merge results from multiple nodes using Triple Intelligence
*/
private async mergeResults(results: any[], query: any): Promise<any[]> {
if (!this.tripleEngine) {
// Simple merge without Triple Intelligence
return this.deduplicateResults(results)
}
// Use Triple Intelligence for intelligent merging
// Merge results by combining scores and maintaining order
const mergedMap = new Map<string, any>()
for (const result of results) {
const id = result.id || result.entity?.id
if (!id) continue
if (mergedMap.has(id)) {
// Merge duplicate results by averaging scores
const existing = mergedMap.get(id)
existing.score = (existing.score + (result.score || 0)) / 2
// Preserve highest confidence metadata
if (result.confidence > existing.confidence) {
existing.metadata = { ...existing.metadata, ...result.metadata }
}
} else {
mergedMap.set(id, { ...result })
}
}
// Sort by score and return
return Array.from(mergedMap.values())
.sort((a, b) => (b.score || 0) - (a.score || 0))
}
/**
* Simple deduplication of results
*/
private deduplicateResults(results: any[]): any[] {
const seen = new Set<string>()
const deduplicated: any[] = []
for (const result of results) {
const key = this.getResultKey(result)
if (!seen.has(key)) {
seen.add(key)
deduplicated.push(result)
}
}
return deduplicated
}
/**
* Get unique key for result
*/
private getResultKey(result: any): string {
if (result.id) return result.id
if (result.uuid) return result.uuid
return JSON.stringify(result)
}
/**
* Determine query type
*/
private getQueryType(query: any): string {
if (query.vector) return 'vector'
if (query.triple) return 'triple'
if (query.filter) return 'filter'
return 'scan'
}
/**
* Determine which shards are affected by query
*/
private async determineAffectedShards(query: any): Promise<string[]> {
const totalShards = this.shardManager.getTotalShards()
const affectedShards: string[] = []
// If query has specific entity/key, determine shard
if (query.entity || query.key) {
const key = query.entity || query.key
const assignment = this.shardManager.getShardForKey(key)
if (assignment) {
return [assignment.shardId]
}
}
// If query has partition hint
if (query.partition) {
return [query.partition]
}
// Otherwise, query all shards (broadcast)
for (let i = 0; i < totalShards; i++) {
affectedShards.push(`shard-${i}`)
}
return affectedShards
}
/**
* Optimize query plan based on statistics
*/
async optimizePlan(plan: QueryPlan): Promise<QueryPlan> {
// Get node health and latency stats
const nodeHealth = await this.coordinator.getHealth()
// Re-assign shards to healthier nodes if needed
const optimizedAssignments = new Map<string, string[]>()
for (const [shardId, nodes] of plan.nodeAssignments) {
// Sort nodes by health score (simple heuristic)
const sortedNodes = nodes.sort((a, b) => {
// Higher health score = better node
// For now, use a simple approach
return 0
})
optimizedAssignments.set(shardId, sortedNodes)
}
return {
...plan,
nodeAssignments: optimizedAssignments
}
}
}

457
src/distributed/readWriteSeparation.ts
View file

@ -1,457 +0,0 @@
/**
* Read/Write Separation for Distributed Scaling
* Implements primary-replica architecture for scalable reads
*/
import { EventEmitter } from 'node:events'
import { DistributedCoordinator } from './coordinator.js'
import { ShardManager } from './shardManager.js'
import { CacheSync } from './cacheSync.js'
export interface ReplicationConfig {
nodeId: string
role?: 'primary' | 'replica' | 'auto'
primaryUrl?: string
replicaUrls?: string[]
syncInterval?: number
readPreference?: 'primary' | 'replica' | 'nearest'
consistencyLevel?: 'eventual' | 'strong' | 'bounded'
maxStaleness?: number
}
export interface WriteOperation {
id: string
type: 'add' | 'update' | 'delete'
data: any
timestamp: number
version: number
}
export interface ReplicationLog {
operations: WriteOperation[]
lastSequence: number
primaryVersion: number
}
/**
* Read/Write Separation Manager
*/
export class ReadWriteSeparation extends EventEmitter {
private nodeId: string
private role: 'primary' | 'replica'
private coordinator: DistributedCoordinator
private cacheSync: CacheSync
private replicationLog: ReplicationLog
private replicas: Map<string, ReplicaConnection> = new Map()
private primaryConnection?: PrimaryConnection
private config: ReplicationConfig
private syncTimer?: NodeJS.Timeout
private isRunning: boolean = false
constructor(
config: ReplicationConfig,
coordinator: DistributedCoordinator,
_shardManager: ShardManager,
cacheSync: CacheSync
) {
super()
this.config = config
this.nodeId = config.nodeId
this.role = config.role === 'auto' ? this.determineRole() : (config.role || 'replica')
this.coordinator = coordinator
this.cacheSync = cacheSync
this.replicationLog = {
operations: [],
lastSequence: 0,
primaryVersion: 0
}
// Setup connections based on role
this.setupConnections()
}
/**
* Start read/write separation
*/
async start(): Promise<void> {
if (this.isRunning) return
this.isRunning = true
// Start components
await this.coordinator.start()
// Setup role-specific behavior
if (this.role === 'primary') {
this.startAsPrimary()
} else {
this.startAsReplica()
}
this.emit('started', { nodeId: this.nodeId, role: this.role })
}
/**
* Stop read/write separation
*/
async stop(): Promise<void> {
if (!this.isRunning) return
this.isRunning = false
if (this.syncTimer) {
clearInterval(this.syncTimer)
this.syncTimer = undefined
}
await this.coordinator.stop()
this.emit('stopped', { nodeId: this.nodeId })
}
/**
* Execute a write operation (primary only)
*/
async write(operation: Omit<WriteOperation, 'id' | 'timestamp' | 'version'>): Promise<string> {
if (this.role !== 'primary') {
// Forward to primary
if (this.primaryConnection) {
return this.primaryConnection.forwardWrite(operation)
}
throw new Error('Cannot write: not connected to primary')
}
// Generate operation metadata
const writeOp: WriteOperation = {
id: this.generateOperationId(),
...operation,
timestamp: Date.now(),
version: ++this.replicationLog.primaryVersion
}
// Add to replication log
this.replicationLog.operations.push(writeOp)
this.replicationLog.lastSequence++
// Propagate to replicas
this.propagateToReplicas(writeOp)
// Update cache
if (operation.type !== 'delete') {
this.cacheSync.set(writeOp.id, operation.data)
} else {
this.cacheSync.delete(writeOp.id)
}
this.emit('write', writeOp)
return writeOp.id
}
/**
* Execute a read operation
*/
async read(key: string, options?: { consistency?: 'eventual' | 'strong' }): Promise<any> {
const consistency = options?.consistency || this.config.consistencyLevel || 'eventual'
if (consistency === 'strong' && this.role === 'replica') {
// For strong consistency, read from primary
if (this.primaryConnection) {
return this.primaryConnection.read(key)
}
throw new Error('Cannot guarantee strong consistency: not connected to primary')
}
// Check cache first
const cached = this.cacheSync.get(key)
if (cached !== undefined) {
return cached
}
// Read from appropriate source based on preference
if (this.config.readPreference === 'primary' && this.primaryConnection) {
return this.primaryConnection.read(key)
}
// Read locally (replica or primary)
return this.readLocal(key)
}
/**
* Get replication lag (replica only)
*/
getReplicationLag(): number {
if (this.role === 'primary') return 0
if (this.primaryConnection) {
return Date.now() - this.primaryConnection.lastSync
}
return -1 // Unknown
}
/**
* Setup connections based on role
*/
private setupConnections(): void {
if (this.role === 'primary') {
// Setup replica connections
if (this.config.replicaUrls) {
for (const url of this.config.replicaUrls) {
const replica = new ReplicaConnection(url)
this.replicas.set(url, replica)
}
}
} else {
// Setup primary connection
if (this.config.primaryUrl) {
this.primaryConnection = new PrimaryConnection(this.config.primaryUrl)
}
}
}
/**
* Start as primary node
*/
private startAsPrimary(): void {
// Start accepting writes
this.emit('roleEstablished', { role: 'primary' })
// Start replication timer
this.syncTimer = setInterval(() => {
this.syncReplicas()
}, this.config.syncInterval || 1000)
}
/**
* Start as replica node
*/
private startAsReplica(): void {
// Start syncing from primary
this.emit('roleEstablished', { role: 'replica' })
// Start sync timer
this.syncTimer = setInterval(() => {
this.syncFromPrimary()
}, this.config.syncInterval || 1000)
}
/**
* Sync replicas (primary only)
*/
private async syncReplicas(): Promise<void> {
const batch = this.replicationLog.operations.slice(-100) // Last 100 ops
for (const [url, replica] of this.replicas) {
try {
await replica.sync(batch, this.replicationLog.primaryVersion)
} catch (error) {
this.emit('replicaSyncError', { url, error })
}
}
}
/**
* Sync from primary (replica only)
*/
private async syncFromPrimary(): Promise<void> {
if (!this.primaryConnection) return
try {
const updates = await this.primaryConnection.getUpdates(
this.replicationLog.lastSequence
)
// Apply updates
for (const op of updates) {
await this.applyOperation(op)
}
this.emit('synced', { operations: updates.length })
} catch (error) {
this.emit('primarySyncError', { error })
}
}
/**
* Apply a replicated operation
*/
private async applyOperation(op: WriteOperation): Promise<void> {
// Update local state
this.replicationLog.operations.push(op)
this.replicationLog.lastSequence = Math.max(
this.replicationLog.lastSequence,
op.version
)
// Update cache
switch (op.type) {
case 'add':
case 'update':
this.cacheSync.set(op.id, op.data)
break
case 'delete':
this.cacheSync.delete(op.id)
break
}
this.emit('operationApplied', op)
}
/**
* Propagate operation to replicas
*/
private propagateToReplicas(op: WriteOperation): void {
for (const replica of this.replicas.values()) {
replica.sendOperation(op).catch(error => {
this.emit('replicationError', { replica, error })
})
}
}
/**
* Determine role automatically
*/
private determineRole(): 'primary' | 'replica' {
// Use coordinator's leader election
return this.coordinator.isLeader() ? 'primary' : 'replica'
}
/**
* Read from local storage
*/
private async readLocal(key: string): Promise<any> {
// This would connect to actual storage
// For now, return from cache or undefined
return this.cacheSync.get(key)
}
/**
* Generate unique operation ID
*/
private generateOperationId(): string {
return `${this.nodeId}-${Date.now()}-${Math.random().toString(36).substring(2, 11)}`
}
/**
* Get replication statistics
*/
getStats(): {
role: string
replicas: number
replicationLag: number
operationsInLog: number
primaryVersion: number
} {
return {
role: this.role,
replicas: this.replicas.size,
replicationLag: this.getReplicationLag(),
operationsInLog: this.replicationLog.operations.length,
primaryVersion: this.replicationLog.primaryVersion
}
}
/**
* Check if node can accept writes
*/
canWrite(): boolean {
return this.role === 'primary'
}
/**
* Check if node can serve reads
*/
canRead(): boolean {
if (this.config.consistencyLevel === 'strong') {
return this.role === 'primary' || this.primaryConnection !== undefined
}
return true
}
/**
* Set whether this node is primary (for leader election integration)
*/
setPrimary(isPrimary: boolean): void {
const newRole = isPrimary ? 'primary' : 'replica'
if (this.role !== newRole) {
this.role = newRole
this.emit('roleChange', { oldRole: this.role, newRole })
if (isPrimary) {
// Became primary - stop syncing from old primary
this.primaryConnection = undefined
} else {
// Became replica - connect to new primary if URL is known
if (this.config.primaryUrl) {
this.primaryConnection = new PrimaryConnection(this.config.primaryUrl)
}
}
}
}
}
/**
* Connection to a replica (used by primary)
*/
class ReplicaConnection {
constructor(private url: string) {
// Store URL for connection
void this.url
}
async sync(_operations: WriteOperation[], _version: number): Promise<void> {
// In real implementation, this would use HTTP/gRPC to this.url
// For now, simulate network call
await new Promise(resolve => setTimeout(resolve, 10))
}
async sendOperation(_op: WriteOperation): Promise<void> {
// Send single operation to replica at this.url
await new Promise(resolve => setTimeout(resolve, 5))
}
}
/**
* Connection to primary (used by replicas)
*/
class PrimaryConnection {
lastSync: number = Date.now()
constructor(private url: string) {
// Store URL for connection
void this.url
}
async getUpdates(_fromSequence: number): Promise<WriteOperation[]> {
// In real implementation, fetch from primary at this.url
this.lastSync = Date.now()
return []
}
async forwardWrite(_operation: any): Promise<string> {
// Forward write to primary at this.url
await new Promise(resolve => setTimeout(resolve, 20))
return `forwarded-${Date.now()}`
}
async read(_key: string): Promise<any> {
// Read from primary at this.url for strong consistency
await new Promise(resolve => setTimeout(resolve, 10))
return undefined
}
}
/**
* Create read/write separation manager
*/
export function createReadWriteSeparation(
config: ReplicationConfig,
coordinator: DistributedCoordinator,
shardManager: ShardManager,
cacheSync: CacheSync
): ReadWriteSeparation {
return new ReadWriteSeparation(config, coordinator, shardManager, cacheSync)
}

453
src/distributed/shardManager.ts
View file

@ -1,453 +0,0 @@
/**
* Shard Manager for Horizontal Scaling
* Implements consistent hashing for data distribution across shards
*/
import { createHash } from 'node:crypto'
import { EventEmitter } from 'node:events'
export interface ShardConfig {
shardCount?: number
replicationFactor?: number
virtualNodes?: number
autoRebalance?: boolean
}
export interface Shard {
id: string
nodeId: string
virtualNodes: string[]
itemCount: number
sizeBytes: number
status: 'active' | 'rebalancing' | 'offline'
}
export interface ShardAssignment {
shardId: string
nodeId: string
replicas: string[]
}
/**
* Consistent Hash Ring for shard distribution
*/
class ConsistentHashRing {
private ring: Map<number, string> = new Map()
private virtualNodes: number
private sortedKeys: number[] = []
constructor(virtualNodes: number = 150) {
this.virtualNodes = virtualNodes
}
/**
* Add a node to the hash ring
*/
addNode(nodeId: string): void {
for (let i = 0; i < this.virtualNodes; i++) {
const virtualNodeId = `${nodeId}:${i}`
const hash = this.hash(virtualNodeId)
this.ring.set(hash, nodeId)
}
this.updateSortedKeys()
}
/**
* Remove a node from the hash ring
*/
removeNode(nodeId: string): void {
const keysToRemove: number[] = []
for (const [hash, node] of this.ring) {
if (node === nodeId) {
keysToRemove.push(hash)
}
}
for (const key of keysToRemove) {
this.ring.delete(key)
}
this.updateSortedKeys()
}
/**
* Get the node responsible for a given key
*/
getNode(key: string): string | null {
if (this.ring.size === 0) return null
const hash = this.hash(key)
// Find the first node with hash >= key hash
for (const nodeHash of this.sortedKeys) {
if (nodeHash >= hash) {
return this.ring.get(nodeHash) || null
}
}
// Wrap around to the first node
return this.ring.get(this.sortedKeys[0]) || null
}
/**
* Get N nodes for replication
*/
getNodes(key: string, count: number): string[] {
if (this.ring.size === 0) return []
const nodes = new Set<string>()
const hash = this.hash(key)
// Start from the primary node position
let startIdx = 0
for (let i = 0; i < this.sortedKeys.length; i++) {
if (this.sortedKeys[i] >= hash) {
startIdx = i
break
}
}
// Collect unique nodes
let idx = startIdx
while (nodes.size < count && nodes.size < this.getUniqueNodeCount()) {
const nodeHash = this.sortedKeys[idx % this.sortedKeys.length]
const node = this.ring.get(nodeHash)
if (node) {
nodes.add(node)
}
idx++
}
return Array.from(nodes)
}
/**
* Get unique node count
*/
private getUniqueNodeCount(): number {
return new Set(this.ring.values()).size
}
/**
* Update sorted keys for efficient lookup
*/
private updateSortedKeys(): void {
this.sortedKeys = Array.from(this.ring.keys()).sort((a, b) => a - b)
}
/**
* Hash function for consistent hashing
*/
private hash(key: string): number {
const hash = createHash('md5').update(key).digest()
return hash.readUInt32BE(0)
}
/**
* Get all nodes in the ring
*/
getAllNodes(): string[] {
return Array.from(new Set(this.ring.values()))
}
}
/**
* Shard Manager for distributing data across multiple nodes
*/
export class ShardManager extends EventEmitter {
private hashRing: ConsistentHashRing
private shards: Map<string, Shard> = new Map()
private nodeToShards: Map<string, Set<string>> = new Map()
private shardCount: number
private replicationFactor: number
private autoRebalance: boolean
constructor(config: ShardConfig = {}) {
super()
this.shardCount = config.shardCount || 64
this.replicationFactor = config.replicationFactor || 3
this.autoRebalance = config.autoRebalance ?? true
this.hashRing = new ConsistentHashRing(config.virtualNodes || 150)
// Initialize shards
this.initializeShards()
}
/**
* Initialize shard configuration
*/
private initializeShards(): void {
for (let i = 0; i < this.shardCount; i++) {
const shardId = `shard-${i.toString().padStart(3, '0')}`
this.shards.set(shardId, {
id: shardId,
nodeId: '',
virtualNodes: [],
itemCount: 0,
sizeBytes: 0,
status: 'offline'
})
}
}
/**
* Add a node to the cluster
*/
addNode(nodeId: string): void {
this.hashRing.addNode(nodeId)
this.nodeToShards.set(nodeId, new Set())
// Assign shards to the new node
this.rebalanceShards()
this.emit('nodeAdded', { nodeId })
}
/**
* Remove a node from the cluster
*/
removeNode(nodeId: string): void {
const affectedShards = this.nodeToShards.get(nodeId) || new Set()
this.hashRing.removeNode(nodeId)
this.nodeToShards.delete(nodeId)
// Reassign affected shards
for (const shardId of affectedShards) {
const shard = this.shards.get(shardId)
if (shard) {
shard.status = 'rebalancing'
}
}
this.rebalanceShards()
this.emit('nodeRemoved', { nodeId, affectedShards: Array.from(affectedShards) })
}
/**
* Get shard assignment for a key
*/
getShardForKey(key: string): ShardAssignment | null {
const shardId = this.getShardId(key)
const nodes = this.hashRing.getNodes(shardId, this.replicationFactor)
if (nodes.length === 0) return null
return {
shardId,
nodeId: nodes[0],
replicas: nodes.slice(1)
}
}
/**
* Get nodes responsible for a shard
*/
getNodesForShard(shardId: string): string[] {
const shard = this.shards.get(shardId)
if (!shard) return []
// Return primary node and replicas
const nodes = this.hashRing.getNodes(shardId, this.replicationFactor)
return nodes
}
/**
* Get total number of shards
*/
getTotalShards(): number {
return this.shardCount
}
/**
* Update shard assignment to a new node
*/
updateShardAssignment(shardId: string, newNodeId: string): void {
const shard = this.shards.get(shardId)
if (!shard) {
throw new Error(`Shard ${shardId} not found`)
}
// Remove from old node
if (shard.nodeId) {
const oldNodeShards = this.nodeToShards.get(shard.nodeId)
if (oldNodeShards) {
oldNodeShards.delete(shardId)
}
}
// Add to new node
shard.nodeId = newNodeId
const newNodeShards = this.nodeToShards.get(newNodeId)
if (newNodeShards) {
newNodeShards.add(shardId)
} else {
this.nodeToShards.set(newNodeId, new Set([shardId]))
}
this.emit('shardReassigned', { shardId, newNodeId })
}
/**
* Get shard ID for a key
*/
private getShardId(key: string): string {
const hash = createHash('md5').update(key).digest()
const shardIndex = hash.readUInt16BE(0) % this.shardCount
return `shard-${shardIndex.toString().padStart(3, '0')}`
}
/**
* Rebalance shards across nodes
*/
private rebalanceShards(): void {
if (!this.autoRebalance) return
const nodes = this.hashRing.getAllNodes()
if (nodes.length === 0) return
// Clear current assignments
for (const nodeSet of this.nodeToShards.values()) {
nodeSet.clear()
}
// Reassign each shard
for (const [shardId, shard] of this.shards) {
const assignedNodes = this.hashRing.getNodes(shardId, 1)
if (assignedNodes.length > 0) {
shard.nodeId = assignedNodes[0]
shard.status = 'active'
const nodeShards = this.nodeToShards.get(assignedNodes[0])
if (nodeShards) {
nodeShards.add(shardId)
}
} else {
shard.status = 'offline'
}
}
this.emit('rebalanced', { nodes, shardCount: this.shardCount })
}
/**
* Get shard assignment for all shards
*/
getShardAssignments(): ShardAssignment[] {
const assignments: ShardAssignment[] = []
for (const [shardId, shard] of this.shards) {
if (shard.nodeId) {
assignments.push({
shardId,
nodeId: shard.nodeId,
replicas: this.hashRing.getNodes(shardId, this.replicationFactor).slice(1)
})
}
}
return assignments
}
/**
* Get shard statistics
*/
getShardStats(): {
totalShards: number
activeShards: number
rebalancingShards: number
offlineShards: number
averageItemsPerShard: number
} {
let activeShards = 0
let rebalancingShards = 0
let offlineShards = 0
let totalItems = 0
for (const shard of this.shards.values()) {
switch (shard.status) {
case 'active':
activeShards++
break
case 'rebalancing':
rebalancingShards++
break
case 'offline':
offlineShards++
break
}
totalItems += shard.itemCount
}
return {
totalShards: this.shardCount,
activeShards,
rebalancingShards,
offlineShards,
averageItemsPerShard: this.shardCount > 0 ? Math.floor(totalItems / this.shardCount) : 0
}
}
/**
* Update shard metrics
*/
updateShardMetrics(shardId: string, itemCount: number, sizeBytes: number): void {
const shard = this.shards.get(shardId)
if (shard) {
shard.itemCount = itemCount
shard.sizeBytes = sizeBytes
}
}
/**
* Get replication nodes for a shard
*/
getReplicationNodes(shardId: string): string[] {
return this.hashRing.getNodes(shardId, this.replicationFactor)
}
/**
* Check if rebalancing is needed
*/
needsRebalancing(): boolean {
const stats = this.getShardStats()
return stats.offlineShards > 0 || stats.rebalancingShards > 0
}
/**
* Get cluster health
*/
getHealth(): {
healthy: boolean
nodes: number
shards: {
total: number
active: number
inactive: number
}
} {
const nodes = this.hashRing.getAllNodes()
const stats = this.getShardStats()
return {
healthy: stats.activeShards >= this.shardCount * 0.9, // 90% shards active
nodes: nodes.length,
shards: {
total: this.shardCount,
active: stats.activeShards,
inactive: stats.offlineShards + stats.rebalancingShards
}
}
}
}
/**
* Create a shard manager instance
*/
export function createShardManager(config?: ShardConfig): ShardManager {
return new ShardManager(config)
}

398
src/distributed/shardMigration.ts
View file

@ -1,398 +0,0 @@
/**
* Shard Migration System for Brainy 3.0
*
* Handles zero-downtime migration of data between nodes
* Uses streaming for efficient transfer of large datasets
*/
import { EventEmitter } from 'node:events'
import type { StorageAdapter } from '../coreTypes.js'
import type { ShardManager } from './shardManager.js'
import type { HTTPTransport } from './httpTransport.js'
import type { DistributedCoordinator } from './coordinator.js'
export interface MigrationTask {
id: string
shardId: string
sourceNode: string
targetNode: string
status: 'pending' | 'transferring' | 'validating' | 'switching' | 'completed' | 'failed'
progress: number // 0-100
itemsTransferred: number
totalItems: number
startTime: number
endTime?: number
error?: string
}
export interface MigrationOptions {
batchSize?: number
validateData?: boolean
maxRetries?: number
timeout?: number
}
export class ShardMigrationManager extends EventEmitter {
private storage: StorageAdapter
private shardManager: ShardManager
private transport: HTTPTransport
private coordinator: DistributedCoordinator
private nodeId: string
private activeMigrations = new Map<string, MigrationTask>()
private migrationQueue: MigrationTask[] = []
private maxConcurrentMigrations = 2
constructor(
nodeId: string,
storage: StorageAdapter,
shardManager: ShardManager,
transport: HTTPTransport,
coordinator: DistributedCoordinator
) {
super()
this.nodeId = nodeId
this.storage = storage
this.shardManager = shardManager
this.transport = transport
this.coordinator = coordinator
}
/**
* Initiate migration of a shard to a new node
*/
async migrateShard(
shardId: string,
targetNode: string,
options: MigrationOptions = {}
): Promise<MigrationTask> {
const task: MigrationTask = {
id: `migration-${Date.now()}-${Math.random().toString(36).substring(7)}`,
shardId,
sourceNode: this.nodeId,
targetNode,
status: 'pending',
progress: 0,
itemsTransferred: 0,
totalItems: 0,
startTime: Date.now()
}
// Add to queue
this.migrationQueue.push(task)
this.processMigrationQueue()
return task
}
/**
* Process migration queue
*/
private async processMigrationQueue(): Promise<void> {
while (this.migrationQueue.length > 0 &&
this.activeMigrations.size < this.maxConcurrentMigrations) {
const task = this.migrationQueue.shift()!
this.activeMigrations.set(task.id, task)
// Execute migration in background
this.executeMigration(task).catch(error => {
console.error(`Migration ${task.id} failed:`, error)
task.status = 'failed'
task.error = error.message
this.emit('migrationFailed', task)
})
}
}
/**
* Execute a single migration task
*/
private async executeMigration(task: MigrationTask): Promise<void> {
try {
this.emit('migrationStarted', task)
// Phase 1: Start transferring data
task.status = 'transferring'
await this.transferData(task)
// Phase 2: Validate transferred data
task.status = 'validating'
await this.validateData(task)
// Phase 3: Switch ownership atomically
task.status = 'switching'
await this.switchOwnership(task)
// Phase 4: Cleanup source
task.status = 'completed'
task.endTime = Date.now()
task.progress = 100
this.activeMigrations.delete(task.id)
this.emit('migrationCompleted', task)
// Process next in queue
this.processMigrationQueue()
} catch (error) {
task.status = 'failed'
task.error = (error as Error).message
this.activeMigrations.delete(task.id)
throw error
}
}
/**
* Transfer data from source to target node
*/
private async transferData(task: MigrationTask): Promise<void> {
const batchSize = 1000
let offset = 0
// Get total count
const totalItems = await this.getShardItemCount(task.shardId)
task.totalItems = totalItems
while (offset < totalItems) {
// Get batch of items
const items = await this.getShardItems(task.shardId, offset, batchSize)
if (items.length === 0) break
// Send batch to target node
await this.transport.call(task.targetNode, 'receiveMigrationBatch', {
migrationId: task.id,
shardId: task.shardId,
items,
offset,
total: totalItems
})
offset += items.length
task.itemsTransferred = offset
task.progress = Math.floor((offset / totalItems) * 80) // 80% for transfer
this.emit('migrationProgress', task)
}
}
/**
* Get items from a shard
*/
private async getShardItems(
shardId: string,
offset: number,
limit: number
): Promise<any[]> {
// Get all noun IDs for this shard
const nounKey = `shard:${shardId}:nouns`
const verbKey = `shard:${shardId}:verbs`
const items: any[] = []
try {
// Get nouns
const nouns = await this.storage.getNounsByNounType('*')
const shardNouns = nouns.filter(n => {
const assignment = this.shardManager.getShardForKey(n.id)
return assignment?.shardId === shardId
}).slice(offset, offset + limit)
items.push(...shardNouns.map(n => ({ type: 'noun', data: n })))
// Get verbs if we have room
if (items.length < limit) {
const verbs = await this.storage.getVerbsByType('*')
const shardVerbs = verbs.filter(v => {
const assignment = this.shardManager.getShardForKey(v.id)
return assignment?.shardId === shardId
}).slice(0, limit - items.length)
items.push(...shardVerbs.map(v => ({ type: 'verb', data: v })))
}
} catch (error) {
console.error(`Failed to get shard items for ${shardId}:`, error)
}
return items
}
/**
* Get count of items in a shard
*/
private async getShardItemCount(shardId: string): Promise<number> {
// For now, estimate based on total items / shard count
// In production, maintain accurate per-shard counts
const status = await this.storage.getStorageStatus()
const totalShards = this.shardManager.getTotalShards()
return Math.ceil(status.used / totalShards)
}
/**
* Validate transferred data
*/
private async validateData(task: MigrationTask): Promise<void> {
// Request validation from target node
const response = await this.transport.call(task.targetNode, 'validateMigration', {
migrationId: task.id,
shardId: task.shardId,
expectedCount: task.totalItems
})
if (!response.valid) {
throw new Error(`Validation failed: ${response.error}`)
}
task.progress = 90 // 90% after validation
this.emit('migrationProgress', task)
}
/**
* Switch shard ownership atomically
*/
private async switchOwnership(task: MigrationTask): Promise<void> {
// Coordinate with all nodes to update shard assignment
await this.coordinator.proposeMigration({
shardId: task.shardId,
fromNode: task.sourceNode,
toNode: task.targetNode,
migrationId: task.id
})
// Wait for consensus
await this.waitForConsensus(task.id)
// Update local shard manager
this.shardManager.updateShardAssignment(task.shardId, task.targetNode)
task.progress = 95 // 95% after ownership switch
this.emit('migrationProgress', task)
// Cleanup local data
await this.cleanupShardData(task.shardId)
}
/**
* Wait for consensus on migration
*/
private async waitForConsensus(migrationId: string): Promise<void> {
const maxWait = 30000 // 30 seconds
const startTime = Date.now()
while (Date.now() - startTime < maxWait) {
const status = await this.coordinator.getMigrationStatus(migrationId)
if (status === 'committed') {
return
} else if (status === 'rejected') {
throw new Error('Migration rejected by cluster')
}
// Wait a bit before checking again
await new Promise(resolve => setTimeout(resolve, 1000))
}
throw new Error('Consensus timeout')
}
/**
* Cleanup local shard data after migration
*/
private async cleanupShardData(shardId: string): Promise<void> {
// Mark shard data for deletion
// Don't delete immediately in case of rollback
const cleanupKey = `cleanup:${shardId}:${Date.now()}`
await this.storage.saveMetadata(cleanupKey, {
noun: 'Document',
shardId,
scheduledFor: Date.now() + 3600000 // Delete after 1 hour
})
}
/**
* Handle incoming migration batch (when we're the target)
*/
async receiveMigrationBatch(data: {
migrationId: string
shardId: string
items: any[]
offset: number
total: number
}): Promise<void> {
// Store items
for (const item of data.items) {
if (item.type === 'noun') {
await this.storage.saveNoun(item.data)
} else if (item.type === 'verb') {
await this.storage.saveVerb(item.data)
}
}
// Track progress
const progress = {
noun: 'Document',
migrationId: data.migrationId,
shardId: data.shardId,
received: data.offset + data.items.length,
total: data.total
}
await this.storage.saveMetadata(`migration:${data.migrationId}:progress`, progress)
}
/**
* Validate received migration data
*/
async validateMigration(data: {
migrationId: string
shardId: string
expectedCount: number
}): Promise<{ valid: boolean; error?: string }> {
// Check if we received all expected items
const progressKey = `migration:${data.migrationId}:progress`
const progress = await this.storage.getMetadata(progressKey)
if (!progress) {
return { valid: false, error: 'No migration progress found' }
}
if (progress.received !== data.expectedCount) {
return {
valid: false,
error: `Expected ${data.expectedCount} items, received ${progress.received}`
}
}
// Verify data integrity (could add checksums)
return { valid: true }
}
/**
* Get status of all active migrations
*/
getActiveMigrations(): MigrationTask[] {
return Array.from(this.activeMigrations.values())
}
/**
* Cancel a migration
*/
async cancelMigration(migrationId: string): Promise<void> {
const task = this.activeMigrations.get(migrationId)
if (!task) {
throw new Error(`Migration ${migrationId} not found`)
}
task.status = 'failed'
task.error = 'Cancelled by user'
this.activeMigrations.delete(migrationId)
// Notify target node
await this.transport.call(task.targetNode, 'cancelMigration', {
migrationId
})
this.emit('migrationCancelled', task)
}
}

687
src/distributed/storageDiscovery.ts
View file

@ -1,687 +0,0 @@
/**
* Storage-based Discovery for Zero-Config Distributed Brainy
* Uses shared storage (S3/GCS/R2) as coordination point
* REAL PRODUCTION CODE - No mocks, no stubs!
*/
import { EventEmitter } from 'node:events'
import * as os from 'node:os'
import { NounMetadata, StorageAdapter } from '../coreTypes.js'
export interface NodeInfo {
id: string
endpoint: string
hostname: string
started: number
lastSeen: number
role: 'primary' | 'replica' | 'candidate'
shards: string[]
capacity: {
cpu: number
memory: number
storage: number
}
stats: {
nouns: number
verbs: number
queries: number
latency: number
}
}
export interface ClusterConfig {
version: number
created: number
updated: number
leader: string | null
nodes: Record<string, NodeInfo>
shards: {
count: number
assignments: Record<string, string[]> // shardId -> [primaryNode, ...replicas]
}
settings: {
replicationFactor: number
shardCount: number
autoRebalance: boolean
minNodes: number
maxNodesPerShard: number
}
}
export class StorageDiscovery extends EventEmitter {
private nodeId: string
private storage: StorageAdapter
private nodeInfo: NodeInfo
private clusterConfig: ClusterConfig | null = null
private heartbeatInterval: NodeJS.Timeout | null = null
private discoveryInterval: NodeJS.Timeout | null = null
private endpoint: string = ''
private isRunning: boolean = false
private readonly HEARTBEAT_INTERVAL = 5000 // 5 seconds
private readonly DISCOVERY_INTERVAL = 2000 // 2 seconds
private readonly NODE_TIMEOUT = 30000 // 30 seconds until node considered dead
private readonly CLUSTER_PATH = '_cluster'
constructor(storage: StorageAdapter, nodeId?: string) {
super()
this.storage = storage
this.nodeId = nodeId || this.generateNodeId()
// Initialize node info with REAL system data
this.nodeInfo = {
id: this.nodeId,
endpoint: '', // Will be set when HTTP server starts
hostname: os.hostname(),
started: Date.now(),
lastSeen: Date.now(),
role: 'candidate',
shards: [],
capacity: {
cpu: os.cpus().length,
memory: Math.floor(os.totalmem() / 1024 / 1024), // MB
storage: 0 // Will be updated based on actual usage
},
stats: {
nouns: 0,
verbs: 0,
queries: 0,
latency: 0
}
}
}
/**
* Start discovery and registration
*/
async start(httpPort: number): Promise<ClusterConfig> {
if (this.isRunning) return this.clusterConfig!
this.isRunning = true
// Set our endpoint
this.endpoint = await this.detectEndpoint(httpPort)
this.nodeInfo.endpoint = this.endpoint
// Try to load existing cluster config
this.clusterConfig = await this.loadClusterConfig()
if (!this.clusterConfig) {
// We're the first node - initialize cluster
await this.initializeCluster()
} else {
// Join existing cluster
await this.joinCluster()
}
// Start heartbeat to keep our node alive
this.startHeartbeat()
// Start discovery to find other nodes
this.startDiscovery()
this.emit('started', this.nodeInfo)
return this.clusterConfig!
}
/**
* Stop discovery and unregister
*/
async stop(): Promise<void> {
if (!this.isRunning) return
this.isRunning = false
// Stop intervals
if (this.heartbeatInterval) {
clearInterval(this.heartbeatInterval)
this.heartbeatInterval = null
}
if (this.discoveryInterval) {
clearInterval(this.discoveryInterval)
this.discoveryInterval = null
}
// Remove ourselves from cluster
await this.leaveCluster()
this.emit('stopped')
}
/**
* Initialize a new cluster (we're the first node)
*/
private async initializeCluster(): Promise<void> {
console.log(`[${this.nodeId}] Initializing new cluster as first node`)
this.nodeInfo.role = 'primary'
this.clusterConfig = {
version: 1,
created: Date.now(),
updated: Date.now(),
leader: this.nodeId,
nodes: {
[this.nodeId]: this.nodeInfo
},
shards: {
count: 64, // Default shard count
assignments: {}
},
settings: {
replicationFactor: 3,
shardCount: 64,
autoRebalance: true,
minNodes: 1,
maxNodesPerShard: 5
}
}
// Assign all shards to ourselves initially
for (let i = 0; i < this.clusterConfig.shards.count; i++) {
const shardId = `shard-${i.toString().padStart(3, '0')}`
this.clusterConfig.shards.assignments[shardId] = [this.nodeId]
this.nodeInfo.shards.push(shardId)
}
// Save cluster config
await this.saveClusterConfig()
// Register ourselves
await this.registerNode()
this.emit('clusterInitialized', this.clusterConfig)
}
/**
* Join an existing cluster
*/
private async joinCluster(): Promise<void> {
console.log(`[${this.nodeId}] Joining existing cluster`)
if (!this.clusterConfig) throw new Error('No cluster config')
// Add ourselves to the cluster
this.clusterConfig.nodes[this.nodeId] = this.nodeInfo
// Determine our role based on cluster state
const nodeCount = Object.keys(this.clusterConfig.nodes).length
if (!this.clusterConfig.leader || !this.clusterConfig.nodes[this.clusterConfig.leader]) {
// No leader or leader is gone - trigger election
await this.triggerLeaderElection()
} else {
// Become replica
this.nodeInfo.role = 'replica'
}
// Register ourselves
await this.registerNode()
// Request shard assignment if auto-rebalance is enabled
if (this.clusterConfig.settings.autoRebalance) {
await this.requestShardAssignment()
}
this.emit('clusterJoined', this.clusterConfig)
}
/**
* Leave cluster cleanly
*/
private async leaveCluster(): Promise<void> {
if (!this.clusterConfig) return
console.log(`[${this.nodeId}] Leaving cluster`)
// Remove ourselves from node registry
try {
// Mark as deleted rather than actually deleting
const deadNode = { noun: 'Document', ...this.nodeInfo, lastSeen: 0, status: 'inactive' as const }
await this.storage.saveMetadata(`${this.CLUSTER_PATH}/nodes/${this.nodeId}.json`, deadNode)
} catch (err) {
// Ignore errors during shutdown
}
// If we're the leader, trigger new election
if (this.clusterConfig.leader === this.nodeId) {
this.clusterConfig.leader = null
await this.saveClusterConfig()
}
this.emit('clusterLeft')
}
/**
* Register node in storage
*/
private async registerNode(): Promise<void> {
const path = `${this.CLUSTER_PATH}/nodes/${this.nodeId}.json`
await this.storage.saveMetadata(path, { noun: 'Document', ...this.nodeInfo })
// Also update registry
await this.updateNodeRegistry(this.nodeId)
}
/**
* Heartbeat to keep node alive
*/
private startHeartbeat(): void {
this.heartbeatInterval = setInterval(async () => {
try {
this.nodeInfo.lastSeen = Date.now()
await this.registerNode()
// Also update cluster config if we're the leader
if (this.clusterConfig && this.clusterConfig.leader === this.nodeId) {
await this.saveClusterConfig()
}
} catch (err) {
console.error(`[${this.nodeId}] Heartbeat failed:`, err)
}
}, this.HEARTBEAT_INTERVAL)
}
/**
* Discover other nodes and monitor health
*/
private startDiscovery(): void {
this.discoveryInterval = setInterval(async () => {
try {
await this.discoverNodes()
await this.checkNodeHealth()
// Check if we need to rebalance
if (this.shouldRebalance()) {
await this.triggerRebalance()
}
} catch (err) {
console.error(`[${this.nodeId}] Discovery failed:`, err)
}
}, this.DISCOVERY_INTERVAL)
}
/**
* Discover nodes from storage
*/
private async discoverNodes(): Promise<void> {
try {
// Since we can't list arbitrary paths, we'll use a registry approach
// Each node registers in a central registry file
const registry = await this.loadNodeRegistry()
const now = Date.now()
let updated = false
for (const nodeId of registry) {
if (nodeId === this.nodeId) continue
try {
const nodeInfoData = await this.storage.getMetadata(
`${this.CLUSTER_PATH}/nodes/${nodeId}.json`
)
const nodeInfo = nodeInfoData as unknown as NodeInfo
// Check if node is alive
if (now - nodeInfo.lastSeen < this.NODE_TIMEOUT) {
if (!this.clusterConfig!.nodes[nodeId]) {
// New node discovered!
console.log(`[${this.nodeId}] Discovered new node: ${nodeId}`)
this.clusterConfig!.nodes[nodeId] = nodeInfo
updated = true
this.emit('nodeDiscovered', nodeInfo)
} else {
// Update existing node info
this.clusterConfig!.nodes[nodeId] = nodeInfo
}
}
} catch (err) {
// Node file might be corrupted or deleted
console.warn(`[${this.nodeId}] Failed to read node ${nodeId}:`, err)
}
}
if (updated) {
this.clusterConfig!.version++
this.clusterConfig!.updated = Date.now()
}
} catch (err) {
// Storage might be unavailable
console.error(`[${this.nodeId}] Failed to discover nodes:`, err)
}
}
/**
* Load node registry from storage
*/
private async loadNodeRegistry(): Promise<string[]> {
try {
// The registry document is written by updateNodeRegistry() below with a
// `nodes` string-ID array on top of the base NounMetadata shape.
const registry = await this.storage.getMetadata(
`${this.CLUSTER_PATH}/registry.json`
) as (NounMetadata & { nodes?: string[] }) | null
return registry?.nodes || []
} catch (err) {
return []
}
}
/**
* Update node registry in storage
*/
private async updateNodeRegistry(add?: string, remove?: string): Promise<void> {
try {
let registry = await this.loadNodeRegistry()
if (add && !registry.includes(add)) {
registry.push(add)
}
if (remove) {
registry = registry.filter(id => id !== remove)
}
await this.storage.saveMetadata(`${this.CLUSTER_PATH}/registry.json`, {
noun: 'Document',
nodes: registry,
updated: Date.now()
})
} catch (err) {
console.error(`[${this.nodeId}] Failed to update registry:`, err)
}
}
/**
* Check health of known nodes
*/
private async checkNodeHealth(): Promise<void> {
if (!this.clusterConfig) return
const now = Date.now()
const deadNodes: string[] = []
for (const [nodeId, nodeInfo] of Object.entries(this.clusterConfig.nodes)) {
if (nodeId === this.nodeId) continue
if (now - nodeInfo.lastSeen > this.NODE_TIMEOUT) {
console.log(`[${this.nodeId}] Node ${nodeId} is dead (last seen ${now - nodeInfo.lastSeen}ms ago)`)
deadNodes.push(nodeId)
}
}
// Remove dead nodes
for (const nodeId of deadNodes) {
delete this.clusterConfig.nodes[nodeId]
this.emit('nodeLost', nodeId)
// If dead node was leader, trigger election
if (this.clusterConfig.leader === nodeId) {
await this.triggerLeaderElection()
}
}
if (deadNodes.length > 0) {
// Trigger rebalance to reassign shards from dead nodes
await this.triggerRebalance()
}
}
/**
* Load cluster configuration from storage
*/
private async loadClusterConfig(): Promise<ClusterConfig | null> {
try {
const config = await this.storage.getMetadata(`${this.CLUSTER_PATH}/config.json`)
return config as unknown as ClusterConfig
} catch (err) {
// No cluster config exists yet
return null
}
}
/**
* Save cluster configuration to storage
*/
private async saveClusterConfig(): Promise<void> {
if (!this.clusterConfig) return
await this.storage.saveMetadata(
`${this.CLUSTER_PATH}/config.json`,
{ noun: 'Document', ...this.clusterConfig }
)
}
/**
* Trigger leader election (simplified - not full Raft)
*/
private async triggerLeaderElection(): Promise<void> {
console.log(`[${this.nodeId}] Triggering leader election`)
// Simple election: node with lowest ID wins
// In production, use proper Raft consensus
const activeNodes = Object.entries(this.clusterConfig!.nodes)
.filter(([_, info]) => Date.now() - info.lastSeen < this.NODE_TIMEOUT)
.sort(([a], [b]) => a.localeCompare(b))
if (activeNodes.length > 0) {
const [leaderId, leaderInfo] = activeNodes[0]
this.clusterConfig!.leader = leaderId
if (leaderId === this.nodeId) {
console.log(`[${this.nodeId}] Became leader`)
this.nodeInfo.role = 'primary'
this.emit('becameLeader')
} else {
console.log(`[${this.nodeId}] Node ${leaderId} is the new leader`)
this.nodeInfo.role = 'replica'
this.emit('leaderElected', leaderId)
}
await this.saveClusterConfig()
}
}
/**
* Request shard assignment for this node
*/
private async requestShardAssignment(): Promise<void> {
if (!this.clusterConfig) return
// Calculate how many shards each node should have
const nodeCount = Object.keys(this.clusterConfig.nodes).length
const shardsPerNode = Math.ceil(this.clusterConfig.shards.count / nodeCount)
// Find shards that need assignment
const unassignedShards: string[] = []
for (let i = 0; i < this.clusterConfig.shards.count; i++) {
const shardId = `shard-${i.toString().padStart(3, '0')}`
if (!this.clusterConfig.shards.assignments[shardId] ||
this.clusterConfig.shards.assignments[shardId].length === 0) {
unassignedShards.push(shardId)
}
}
// Assign some shards to ourselves
const ourShare = unassignedShards.slice(0, shardsPerNode)
for (const shardId of ourShare) {
this.clusterConfig.shards.assignments[shardId] = [this.nodeId]
this.nodeInfo.shards.push(shardId)
}
if (ourShare.length > 0) {
console.log(`[${this.nodeId}] Assigned ${ourShare.length} shards`)
await this.saveClusterConfig()
}
}
/**
* Check if rebalancing is needed
*/
private shouldRebalance(): boolean {
if (!this.clusterConfig || !this.clusterConfig.settings.autoRebalance) {
return false
}
// Check if shards are evenly distributed
const nodeCount = Object.keys(this.clusterConfig.nodes).length
if (nodeCount <= 1) return false
const targetShardsPerNode = Math.ceil(this.clusterConfig.shards.count / nodeCount)
const variance = 2 // Allow some variance
for (const nodeInfo of Object.values(this.clusterConfig.nodes)) {
const shardCount = nodeInfo.shards.length
if (Math.abs(shardCount - targetShardsPerNode) > variance) {
return true
}
}
return false
}
/**
* Trigger shard rebalancing
*/
private async triggerRebalance(): Promise<void> {
// Only leader can trigger rebalance
if (this.clusterConfig?.leader !== this.nodeId) return
console.log(`[${this.nodeId}] Triggering shard rebalance`)
// This will be implemented with actual data migration
// For now, just redistribute shard assignments
await this.redistributeShards()
this.emit('rebalanceTriggered')
}
/**
* Redistribute shards among active nodes
*/
private async redistributeShards(): Promise<void> {
if (!this.clusterConfig) return
const activeNodes = Object.keys(this.clusterConfig.nodes)
.filter(id => Date.now() - this.clusterConfig!.nodes[id].lastSeen < this.NODE_TIMEOUT)
if (activeNodes.length === 0) return
const shardsPerNode = Math.ceil(this.clusterConfig.shards.count / activeNodes.length)
const newAssignments: Record<string, string[]> = {}
// Clear current shard assignments from nodes
for (const nodeInfo of Object.values(this.clusterConfig.nodes)) {
nodeInfo.shards = []
}
// Redistribute shards
let nodeIndex = 0
for (let i = 0; i < this.clusterConfig.shards.count; i++) {
const shardId = `shard-${i.toString().padStart(3, '0')}`
const primaryNode = activeNodes[nodeIndex % activeNodes.length]
// Assign primary
newAssignments[shardId] = [primaryNode]
this.clusterConfig.nodes[primaryNode].shards.push(shardId)
// Assign replicas
const replicas: string[] = []
for (let r = 1; r < Math.min(this.clusterConfig.settings.replicationFactor, activeNodes.length); r++) {
const replicaNode = activeNodes[(nodeIndex + r) % activeNodes.length]
if (replicaNode !== primaryNode) {
replicas.push(replicaNode)
}
}
if (replicas.length > 0) {
newAssignments[shardId].push(...replicas)
}
nodeIndex++
}
this.clusterConfig.shards.assignments = newAssignments
this.clusterConfig.version++
this.clusterConfig.updated = Date.now()
await this.saveClusterConfig()
console.log(`[${this.nodeId}] Rebalanced ${this.clusterConfig.shards.count} shards across ${activeNodes.length} nodes`)
}
/**
* Detect our public endpoint
*/
private async detectEndpoint(port: number): Promise<string> {
// Try to detect public IP
const interfaces = os.networkInterfaces()
let ip = '127.0.0.1'
// Find first non-internal IPv4 address
for (const iface of Object.values(interfaces)) {
if (!iface) continue
for (const addr of iface) {
if (addr.family === 'IPv4' && !addr.internal) {
ip = addr.address
break
}
}
}
// In cloud environments, might need to detect public IP differently
if (process.env.PUBLIC_IP) {
ip = process.env.PUBLIC_IP
} else if (process.env.KUBERNETES_SERVICE_HOST) {
// In Kubernetes, use pod IP
ip = process.env.POD_IP || ip
}
return `http://${ip}:${port}`
}
/**
* Generate unique node ID
*/
private generateNodeId(): string {
const hostname = os.hostname()
const pid = process.pid
const random = Math.random().toString(36).substring(2, 8)
return `${hostname}-${pid}-${random}`
}
/**
* Get current cluster configuration
*/
getClusterConfig(): ClusterConfig | null {
return this.clusterConfig
}
/**
* Get active nodes
*/
getActiveNodes(): NodeInfo[] {
if (!this.clusterConfig) return []
const now = Date.now()
return Object.values(this.clusterConfig.nodes)
.filter(node => now - node.lastSeen < this.NODE_TIMEOUT)
}
/**
* Get shards assigned to this node
*/
getMyShards(): string[] {
return this.nodeInfo.shards
}
/**
* Update node statistics
*/
updateStats(stats: Partial<NodeInfo['stats']>): void {
Object.assign(this.nodeInfo.stats, stats)
}
}

27
src/index.ts
View file

@ -77,32 +77,9 @@ export type {
// Export Aggregation Engine
export { AggregationIndex, AggregateMaterializer, bucketTimestamp, parseBucketRange } from './aggregation/index.js'
// Export zero-configuration types and enums
// Export zero-configuration input type
export {
// Preset names
PresetName,
// Model configuration
ModelPrecision,
// Storage configuration
StorageOption,
// Feature configuration
FeatureSet,
// Distributed roles
DistributedRole,
// Categories
PresetCategory,
// Config type
BrainyZeroConfig,
// Extensibility
StorageProvider,
registerStorageAugmentation,
registerPresetAugmentation,
// Preset utilities
getPreset,
isValidPreset,
getPresetsByCategory,
getAllPresetNames,
getPresetDescription
BrainyZeroConfig
} from './config/index.js'
// Export Neural Import (AI data understanding)

25
src/storage/adapters/baseStorageAdapter.ts
View file

@ -1159,13 +1159,13 @@ export abstract class BaseStorageAdapter implements StorageAdapter {
}
/**
* Increment count for entity type - O(1) operation
* Protected by storage-specific mechanisms (mutex, distributed consensus, etc.)
* Increment count for entity type - O(1) operation.
* Concurrency is handled by the process-global mutex
* ({@link incrementEntityCountSafe}); this raw form is for callers that
* already hold it.
* @param type The entity type
*/
protected incrementEntityCount(type: string): void {
// For distributed scenarios, this is aggregated across shards
// For single-node, this is protected by storage-specific locking
this.entityCounts.set(type, (this.entityCounts.get(type) || 0) + 1)
this.totalNounCount++
// Update cache
@ -1176,11 +1176,11 @@ export abstract class BaseStorageAdapter implements StorageAdapter {
}
/**
* Thread-safe increment for concurrent scenarios
* Uses mutex for single-node, distributed consensus for multi-node
* Thread-safe increment for concurrent scenarios.
* A process-global mutex serialises the read-modify-write so concurrent
* writers in the same process cannot lose updates.
*/
protected async incrementEntityCountSafe(type: string): Promise<void> {
// Single-node mutex protection (distributed mode handled by coordinator)
const mutex = getGlobalMutex()
await mutex.runExclusive(`count-entity-${type}`, async () => {
this.incrementEntityCount(type)
@ -1225,8 +1225,10 @@ export abstract class BaseStorageAdapter implements StorageAdapter {
}
/**
* Increment verb count - O(1) operation (now synchronous)
* Protected by storage-specific mechanisms (mutex, distributed consensus, etc.)
* Increment verb count - O(1) operation (now synchronous).
* Concurrency is handled by the process-global mutex
* ({@link incrementVerbCountSafe}); this raw form is for callers that already
* hold it.
* @param type The verb type
*/
protected incrementVerbCount(type: string): void {
@ -1240,8 +1242,9 @@ export abstract class BaseStorageAdapter implements StorageAdapter {
}
/**
* Thread-safe increment for verb counts
* Uses mutex for single-node, distributed consensus for multi-node
* Thread-safe increment for verb counts.
* A process-global mutex serialises the read-modify-write so concurrent
* writers in the same process cannot lose updates.
* @param type The verb type
*/
protected async incrementVerbCountSafe(type: string): Promise<void> {

10
src/storage/adapters/memoryStorage.ts
View file

@ -407,13 +407,9 @@ export class MemoryStorage extends BaseStorage {
// Include services if present
...(statistics.services && {
services: statistics.services.map(s => ({...s}))
}),
// Include distributedConfig if present
...(statistics.distributedConfig && {
distributedConfig: JSON.parse(JSON.stringify(statistics.distributedConfig))
})
}
// Since this is in-memory, there's no need for time-based partitioning
// or legacy file handling
}
@ -459,10 +455,6 @@ export class MemoryStorage extends BaseStorage {
// Include services if present
...(this.statistics.services && {
services: this.statistics.services.map(s => ({...s}))
}),
// Include distributedConfig if present
...(this.statistics.distributedConfig && {
distributedConfig: JSON.parse(JSON.stringify(this.statistics.distributedConfig))
})
}

8
src/storage/baseStorage.ts
View file

@ -144,7 +144,6 @@ const SINGLETON_SYSTEM_KEYS = new Set([
const SINGLETON_SYSTEM_PREFIXES = [
'statistics_',
'distributed_',
]
function isSingletonSystemKey(key: string): boolean {
@ -2292,11 +2291,11 @@ export abstract class BaseStorage extends BaseStorageAdapter {
await this.ensureInitialized()
const keyInfo = this.analyzeKey(id, 'system')
// Try new distributed path first
// Try the new shard-prefixed path first
const data = await this.readCanonicalObject(keyInfo.fullPath)
if (data !== null) return data
// Backward compat: if key was distributed, fall back to legacy flat path
// Backward compat: if the key was sharded, fall back to the legacy flat path
if (keyInfo.shardId !== null) {
const legacyPath = `${SYSTEM_DIR}/${id}.json`
return this.readCanonicalObject(legacyPath)
@ -2428,10 +2427,9 @@ export abstract class BaseStorage extends BaseStorageAdapter {
* @returns Metadata or null if not found
*
* @performance
* - O(1) direct ID lookup - always 1 read (~500ms on cloud, ~10ms local)
* - O(1) direct ID lookup - always 1 read (~10ms on local disk)
* - No caching complexity
* - No type search fallbacks
* - Works in distributed systems without sync issues
*
* Type-first paths (removed)
* Promoted to fast path for brain.get() optimization

73
src/storage/operationalModes.ts Normal file
View file

@ -0,0 +1,73 @@
/**
* @module storage/operationalModes
* @description Operational modes that gate which operations a Brainy instance may
* perform. Brainy's multi-process model runs one writer process plus any number of
* reader processes against a single shared on-disk store; the mode object is the
* in-process guard that enforces that contract.
*
* - {@link HybridMode} (the default, used by writer instances) permits reads,
* writes, and deletes — a writer must be able to read its own data.
* - {@link ReaderMode} (used by `mode: 'reader'` / `Brainy.openReadOnly()` and by
* `asOf()` historical snapshots) permits reads only; every mutation throws.
*
* `validateOperation()` is called at the top of each mutation path, and the
* `canWrite` flag drives the public read-only checks on the instance.
*/
/**
* @description Base class for operational modes. Concrete modes declare which of
* read/write/delete they permit; {@link BaseOperationalMode.validateOperation}
* turns a disallowed operation into an explicit error.
*/
export abstract class BaseOperationalMode {
abstract canRead: boolean
abstract canWrite: boolean
abstract canDelete: boolean
/**
* @description Throw if the requested operation is not allowed in this mode.
* Called at the top of every mutation method so read-only instances fail loudly
* rather than silently dropping writes.
* @param operation - The operation being attempted.
* @throws {Error} When the mode does not permit the operation.
*/
validateOperation(operation: 'read' | 'write' | 'delete'): void {
switch (operation) {
case 'read':
if (!this.canRead) {
throw new Error('Read operations are not allowed in write-only mode')
}
break
case 'write':
if (!this.canWrite) {
throw new Error('Write operations are not allowed in read-only mode')
}
break
case 'delete':
if (!this.canDelete) {
throw new Error('Delete operations are not allowed in this mode')
}
break
}
}
}
/**
* @description Read-only mode. Reads are permitted; writes and deletes throw.
* Used by reader processes, `Brainy.openReadOnly()`, and historical snapshots.
*/
export class ReaderMode extends BaseOperationalMode {
canRead = true
canWrite = false
canDelete = false
}
/**
* @description Read-write mode and the default for writer instances. Permits
* reads, writes, and deletes — a writer needs to read its own data.
*/
export class HybridMode extends BaseOperationalMode {
canRead = true
canWrite = true
canDelete = true
}

530
src/storage/readOnlyOptimizations.ts
View file

@ -1,530 +0,0 @@
/**
* Read-Only Storage Optimizations for Production Deployments
* Implements compression, memory-mapping, and pre-built index segments
*/
import { HNSWNoun, Vector } from '../coreTypes.js'
// Compression types supported
enum CompressionType {
NONE = 'none',
GZIP = 'gzip',
QUANTIZATION = 'quantization',
HYBRID = 'hybrid'
// BROTLI removed - was not actually implemented
}
// Vector quantization methods
enum QuantizationType {
SCALAR = 'scalar', // 8-bit scalar quantization
PRODUCT = 'product', // Product quantization
BINARY = 'binary' // Binary quantization
}
interface CompressionConfig {
vectorCompression: CompressionType
metadataCompression: CompressionType
quantizationType?: QuantizationType
quantizationBits?: number
compressionLevel?: number
}
interface ReadOnlyConfig {
prebuiltIndexPath?: string
memoryMapped?: boolean
compression: CompressionConfig
segmentSize?: number // For index segmentation
prefetchSegments?: number
cacheIndexInMemory?: boolean
}
interface IndexSegment {
id: string
nodeCount: number
vectorDimension: number
compression: CompressionType
s3Key?: string
localPath?: string
loadedInMemory: boolean
lastAccessed: number
}
/**
* Read-only storage optimizations for high-performance production deployments
*/
export class ReadOnlyOptimizations {
private config: Required<ReadOnlyConfig>
private segments: Map<string, IndexSegment> = new Map()
private compressionStats = {
originalSize: 0,
compressedSize: 0,
compressionRatio: 0,
decompressionTime: 0
}
// Quantization codebooks for vector compression
private quantizationCodebooks: Map<string, Float32Array> = new Map()
// Memory-mapped buffers for large datasets
private memoryMappedBuffers: Map<string, ArrayBuffer> = new Map()
constructor(config: Partial<ReadOnlyConfig> = {}) {
this.config = {
prebuiltIndexPath: '',
memoryMapped: true,
compression: {
vectorCompression: CompressionType.QUANTIZATION,
metadataCompression: CompressionType.GZIP,
quantizationType: QuantizationType.SCALAR,
quantizationBits: 8,
compressionLevel: 6
},
segmentSize: 10000, // 10k nodes per segment
prefetchSegments: 3,
cacheIndexInMemory: false,
...config
}
if (config.compression) {
this.config.compression = { ...this.config.compression, ...config.compression }
}
}
/**
* Compress vector data using specified compression method
*/
public async compressVector(vector: Vector, segmentId: string): Promise<ArrayBuffer> {
const startTime = Date.now()
let compressedData: ArrayBuffer
switch (this.config.compression.vectorCompression) {
case CompressionType.QUANTIZATION:
compressedData = await this.quantizeVector(vector, segmentId)
break
case CompressionType.GZIP:
const gzipBuffer = new Float32Array(vector).buffer
compressedData = await this.gzipCompress(gzipBuffer.slice(0))
break
// Brotli removed - was not implemented
case CompressionType.HYBRID:
// First quantize, then compress
const quantized = await this.quantizeVector(vector, segmentId)
compressedData = await this.gzipCompress(quantized)
break
default:
const defaultBuffer = new Float32Array(vector).buffer
compressedData = defaultBuffer.slice(0)
break
}
// Update compression statistics
const originalSize = vector.length * 4 // 4 bytes per float32
this.compressionStats.originalSize += originalSize
this.compressionStats.compressedSize += compressedData.byteLength
this.compressionStats.decompressionTime += Date.now() - startTime
this.updateCompressionRatio()
return compressedData
}
/**
* Decompress vector data
*/
public async decompressVector(
compressedData: ArrayBuffer,
segmentId: string,
originalDimension: number
): Promise<Vector> {
switch (this.config.compression.vectorCompression) {
case CompressionType.QUANTIZATION:
return this.dequantizeVector(compressedData, segmentId, originalDimension)
case CompressionType.GZIP:
const gzipDecompressed = await this.gzipDecompress(compressedData)
return Array.from(new Float32Array(gzipDecompressed))
// Brotli removed - was not implemented
case CompressionType.HYBRID:
const gzipStage = await this.gzipDecompress(compressedData)
return this.dequantizeVector(gzipStage, segmentId, originalDimension)
default:
return Array.from(new Float32Array(compressedData))
}
}
/**
* Scalar quantization of vectors to 8-bit integers
*/
private async quantizeVector(vector: Vector, segmentId: string): Promise<ArrayBuffer> {
let codebook = this.quantizationCodebooks.get(segmentId)
if (!codebook) {
// Create codebook (min/max values for scaling)
const min = Math.min(...vector)
const max = Math.max(...vector)
codebook = new Float32Array([min, max])
this.quantizationCodebooks.set(segmentId, codebook)
}
const [min, max] = codebook
const scale = (max - min) / 255 // 8-bit quantization
const quantized = new Uint8Array(vector.length)
for (let i = 0; i < vector.length; i++) {
quantized[i] = Math.round((vector[i] - min) / scale)
}
// Store codebook with quantized data
const result = new ArrayBuffer(quantized.byteLength + codebook.byteLength)
const resultView = new Uint8Array(result)
// First 8 bytes: codebook (min, max as float32)
resultView.set(new Uint8Array(codebook.buffer), 0)
// Remaining bytes: quantized vector
resultView.set(quantized, codebook.byteLength)
return result
}
/**
* Dequantize 8-bit vectors back to float32
*/
private dequantizeVector(
quantizedData: ArrayBuffer,
segmentId: string,
dimension: number
): Vector {
const dataView = new Uint8Array(quantizedData)
// Extract codebook (first 8 bytes)
const codebookBytes = dataView.slice(0, 8)
const codebook = new Float32Array(codebookBytes.buffer)
const [min, max] = codebook
// Extract quantized vector
const quantized = dataView.slice(8)
const scale = (max - min) / 255
const result: Vector = []
for (let i = 0; i < dimension; i++) {
result[i] = min + quantized[i] * scale
}
return result
}
/**
* GZIP compression using browser/Node.js APIs
*/
private async gzipCompress(data: ArrayBuffer): Promise<ArrayBuffer> {
if (typeof CompressionStream !== 'undefined') {
// Browser environment
const stream = new CompressionStream('gzip')
const writer = stream.writable.getWriter()
const reader = stream.readable.getReader()
writer.write(new Uint8Array(data))
writer.close()
const chunks: Uint8Array[] = []
let result = await reader.read()
while (!result.done) {
chunks.push(result.value)
result = await reader.read()
}
// Combine chunks
const totalLength = chunks.reduce((sum, chunk) => sum + chunk.length, 0)
const combined = new Uint8Array(totalLength)
let offset = 0
for (const chunk of chunks) {
combined.set(chunk, offset)
offset += chunk.length
}
return combined.buffer
} else {
// Node.js environment - would use zlib
console.warn('GZIP compression not available, returning original data')
return data
}
}
/**
* GZIP decompression
*/
private async gzipDecompress(compressedData: ArrayBuffer): Promise<ArrayBuffer> {
if (typeof DecompressionStream !== 'undefined') {
// Browser environment
const stream = new DecompressionStream('gzip')
const writer = stream.writable.getWriter()
const reader = stream.readable.getReader()
writer.write(new Uint8Array(compressedData))
writer.close()
const chunks: Uint8Array[] = []
let result = await reader.read()
while (!result.done) {
chunks.push(result.value)
result = await reader.read()
}
// Combine chunks
const totalLength = chunks.reduce((sum, chunk) => sum + chunk.length, 0)
const combined = new Uint8Array(totalLength)
let offset = 0
for (const chunk of chunks) {
combined.set(chunk, offset)
offset += chunk.length
}
return combined.buffer
} else {
console.warn('GZIP decompression not available, returning original data')
return compressedData
}
}
// Brotli methods removed - were not implemented
/**
* Create prebuilt index segments for faster loading
*/
public async createPrebuiltSegments(
nodes: HNSWNoun[],
outputPath: string
): Promise<IndexSegment[]> {
const segments: IndexSegment[] = []
const segmentSize = this.config.segmentSize
console.log(`Creating ${Math.ceil(nodes.length / segmentSize)} prebuilt segments`)
for (let i = 0; i < nodes.length; i += segmentSize) {
const segmentNodes = nodes.slice(i, i + segmentSize)
const segmentId = `segment_${Math.floor(i / segmentSize)}`
const segment: IndexSegment = {
id: segmentId,
nodeCount: segmentNodes.length,
vectorDimension: segmentNodes[0]?.vector.length || 0,
compression: this.config.compression.vectorCompression,
localPath: `${outputPath}/${segmentId}.dat`,
loadedInMemory: false,
lastAccessed: 0
}
// Compress and serialize segment data
const compressedData = await this.compressSegment(segmentNodes)
// In a real implementation, you would write this to disk/S3
console.log(`Created segment ${segmentId} with ${compressedData.byteLength} bytes`)
segments.push(segment)
this.segments.set(segmentId, segment)
}
return segments
}
/**
* Compress an entire segment of nodes
*/
private async compressSegment(nodes: HNSWNoun[]): Promise<ArrayBuffer> {
const serialized = JSON.stringify(nodes.map(node => ({
id: node.id,
vector: node.vector,
connections: this.serializeConnections(node.connections)
})))
const encoder = new TextEncoder()
const data = encoder.encode(serialized)
// Apply metadata compression
switch (this.config.compression.metadataCompression) {
case CompressionType.GZIP:
return this.gzipCompress(data.buffer.slice(0) as ArrayBuffer)
// Brotli removed - was not implemented
default:
return data.buffer.slice(0) as ArrayBuffer
}
}
/**
* Load a segment from storage with caching
*/
public async loadSegment(segmentId: string): Promise<HNSWNoun[]> {
const segment = this.segments.get(segmentId)
if (!segment) {
throw new Error(`Segment ${segmentId} not found`)
}
segment.lastAccessed = Date.now()
// Check if segment is already loaded in memory
if (segment.loadedInMemory && this.memoryMappedBuffers.has(segmentId)) {
return this.deserializeSegment(this.memoryMappedBuffers.get(segmentId)!)
}
// Load from storage (S3, disk, etc.)
const compressedData = await this.loadSegmentFromStorage(segment)
// Cache in memory if configured
if (this.config.cacheIndexInMemory) {
this.memoryMappedBuffers.set(segmentId, compressedData)
segment.loadedInMemory = true
}
return this.deserializeSegment(compressedData)
}
/**
* Load segment data from storage
*/
private async loadSegmentFromStorage(segment: IndexSegment): Promise<ArrayBuffer> {
// Load segment from memory-mapped buffer if available
const cached = this.memoryMappedBuffers.get(segment.id)
if (cached) {
return cached
}
// This feature requires actual storage backend integration (S3, file system, etc)
// Return empty buffer as this is an optional optimization feature
console.warn(`Segment loading optimization not available for segment ${segment.id}. Using standard storage.`)
return new ArrayBuffer(0)
}
/**
* Deserialize and decompress segment data
*/
private async deserializeSegment(compressedData: ArrayBuffer): Promise<HNSWNoun[]> {
// Decompress metadata
let decompressed: ArrayBuffer
switch (this.config.compression.metadataCompression) {
case CompressionType.GZIP:
decompressed = await this.gzipDecompress(compressedData)
break
// Brotli removed - was not implemented
default:
decompressed = compressedData
break
}
// Parse JSON
const decoder = new TextDecoder()
const jsonStr = decoder.decode(decompressed)
const parsed = JSON.parse(jsonStr)
// Reconstruct HNSWNoun objects
return parsed.map((item: any) => ({
id: item.id,
vector: item.vector,
connections: this.deserializeConnections(item.connections)
}))
}
/**
* Serialize connections Map for storage
*/
private serializeConnections(connections: Map<number, Set<string>>): Record<string, string[]> {
const result: Record<string, string[]> = {}
for (const [level, nodeIds] of connections.entries()) {
result[level.toString()] = Array.from(nodeIds)
}
return result
}
/**
* Deserialize connections from storage format
*/
private deserializeConnections(serialized: Record<string, string[]>): Map<number, Set<string>> {
const result = new Map<number, Set<string>>()
for (const [levelStr, nodeIds] of Object.entries(serialized)) {
result.set(parseInt(levelStr), new Set(nodeIds))
}
return result
}
/**
* Prefetch segments based on access patterns
*/
public async prefetchSegments(currentSegmentId: string): Promise<void> {
const segment = this.segments.get(currentSegmentId)
if (!segment) return
// Simple prefetching strategy - load adjacent segments
const segmentNumber = parseInt(currentSegmentId.split('_')[1])
const toPrefetch: string[] = []
for (let i = 1; i <= this.config.prefetchSegments; i++) {
const nextId = `segment_${segmentNumber + i}`
const prevId = `segment_${segmentNumber - i}`
if (this.segments.has(nextId) && !this.memoryMappedBuffers.has(nextId)) {
toPrefetch.push(nextId)
}
if (this.segments.has(prevId) && !this.memoryMappedBuffers.has(prevId)) {
toPrefetch.push(prevId)
}
}
// Prefetch in background
for (const segmentId of toPrefetch) {
this.loadSegment(segmentId).catch(error => {
console.warn(`Failed to prefetch segment ${segmentId}:`, error)
})
}
}
/**
* Update compression statistics
*/
private updateCompressionRatio(): void {
if (this.compressionStats.originalSize > 0) {
this.compressionStats.compressionRatio =
this.compressionStats.compressedSize / this.compressionStats.originalSize
}
}
/**
* Get compression statistics
*/
public getCompressionStats(): typeof this.compressionStats & {
segmentCount: number
memoryUsage: number
} {
const memoryUsage = Array.from(this.memoryMappedBuffers.values())
.reduce((sum, buffer) => sum + buffer.byteLength, 0)
return {
...this.compressionStats,
segmentCount: this.segments.size,
memoryUsage
}
}
/**
* Cleanup memory-mapped buffers
*/
public cleanup(): void {
this.memoryMappedBuffers.clear()
this.quantizationCodebooks.clear()
// Mark all segments as not loaded
for (const segment of this.segments.values()) {
segment.loadedInMemory = false
}
}
}

19
src/types/brainy.types.ts
View file

@ -1206,25 +1206,6 @@ export interface BrainyConfig {
}
| StorageAdapter
/**
* Distributed-cluster configuration. **Explicit opt-in only** — Brainy
* never auto-enables cluster mode from environment heuristics (a single
* production process is the 90th-percentile deployment). Set
* `enabled: true` (or the `BRAINY_DISTRIBUTED=true` env var) to activate;
* the remaining fields then default sensibly (64 shards, 3 replicas,
* raft consensus, http transport, hostname-derived nodeId).
*/
distributed?: {
enabled: boolean
nodeId?: string
nodes?: string[] // Other nodes in cluster
coordinatorUrl?: string // Coordinator endpoint
shardCount?: number // Number of shards (default: 64)
replicationFactor?: number // Number of replicas (default: 3)
consensus?: 'raft' | 'none' // Consensus mechanism
transport?: 'tcp' | 'http' | 'udp'
}
/**
* Disable the automatic index rebuild check during `init()`. By default
* Brainy auto-decides from dataset size: small datasets rebuild missing

236
src/types/distributedTypes.ts
View file

@ -1,236 +0,0 @@
/**
* Distributed types for Brainy
* Defines types for distributed operations across multiple instances
*/
export type InstanceRole = 'reader' | 'writer' | 'hybrid'
export type PartitionStrategy = 'hash' | 'semantic' | 'manual'
export interface DistributedConfig {
/**
* Enable distributed mode
* Can be boolean for auto-detection or specific configuration
*/
enabled?: boolean | 'auto'
/**
* Role of this instance in the distributed system
* - reader: Read-only access, optimized for queries
* - writer: Write-focused, handles data ingestion
* - hybrid: Can both read and write (requires coordination)
*/
role?: InstanceRole
/**
* Unique identifier for this instance
* Auto-generated if not provided
*/
instanceId?: string
/**
* Path to shared configuration file in S3
* Default: '_brainy/config.json'
*/
configPath?: string
/**
* Heartbeat interval in milliseconds
* Default: 30000 (30 seconds)
*/
heartbeatInterval?: number
/**
* Config check interval in milliseconds
* Default: 10000 (10 seconds)
*/
configCheckInterval?: number
/**
* Instance timeout in milliseconds
* Instances not seen for this duration are considered dead
* Default: 60000 (60 seconds)
*/
instanceTimeout?: number
}
export interface SharedConfig {
/**
* Configuration version for compatibility checking
*/
version: number
/**
* Last update timestamp
*/
updated: string
/**
* Global settings that must be consistent across all instances
*/
settings: {
/**
* Partitioning strategy
* - hash: Deterministic hash-based partitioning (recommended for multi-writer)
* - semantic: Group similar vectors (single writer only)
* - manual: Explicit partition assignment
*/
partitionStrategy: PartitionStrategy
/**
* Number of partitions (for hash strategy)
*/
partitionCount: number
/**
* Embedding model name (must be consistent)
*/
embeddingModel: string
/**
* Vector dimensions
*/
dimensions: number
/**
* Distance metric
*/
distanceMetric: 'cosine' | 'euclidean' | 'manhattan'
/**
* HNSW parameters (must be consistent for index compatibility)
*/
hnswParams?: {
M: number
efConstruction: number
maxElements?: number
}
}
/**
* Active instances in the distributed system
*/
instances: {
[instanceId: string]: InstanceInfo
}
/**
* Partition assignments (for manual strategy)
*/
partitionAssignments?: {
[instanceId: string]: string[]
}
}
export interface InstanceInfo {
/**
* Instance role
*/
role: InstanceRole
/**
* Instance status
*/
status: 'active' | 'inactive' | 'unhealthy'
/**
* Last heartbeat timestamp
*/
lastHeartbeat: string
/**
* Optional endpoint for health checks
*/
endpoint?: string
/**
* Instance metrics
*/
metrics?: {
vectorCount?: number
cacheHitRate?: number
memoryUsage?: number
cpuUsage?: number
}
/**
* Assigned partitions (for manual assignment)
*/
assignedPartitions?: string[]
/**
* Preferred partitions (for affinity)
*/
preferredPartitions?: number[]
}
export interface DomainMetadata {
/**
* Domain identifier for logical data separation
*/
domain?: string
/**
* Additional domain-specific metadata
*/
domainMetadata?: Record<string, any>
}
export interface CacheStrategy {
/**
* Percentage of memory allocated to hot cache (0-1)
*/
hotCacheRatio: number
/**
* Enable aggressive prefetching
*/
prefetchAggressive?: boolean
/**
* Cache time-to-live in milliseconds
*/
ttl?: number
/**
* Enable compression to trade CPU for memory
*/
compressionEnabled?: boolean
/**
* Write buffer size for batching
*/
writeBufferSize?: number
/**
* Enable write batching
*/
batchWrites?: boolean
/**
* Adaptive caching based on workload
*/
adaptive?: boolean
}
export interface OperationalMode {
/**
* Whether this mode can read
*/
canRead: boolean
/**
* Whether this mode can write
*/
canWrite: boolean
/**
* Whether this mode can delete
*/
canDelete: boolean
/**
* Cache strategy for this mode
*/
cacheStrategy: CacheStrategy
}

4
src/utils/autoConfiguration.ts
View file

@ -25,7 +25,6 @@ export interface AutoConfigResult {
targetSearchLatency: number
enablePartitioning: boolean
enableCompression: boolean
enableDistributedSearch: boolean
enablePredictiveCaching: boolean
partitionStrategy: 'semantic' | 'hash'
maxNodesPerPartition: number
@ -121,7 +120,6 @@ export class AutoConfiguration {
// Learn from search performance
if (metrics.averageSearchTime > 200) {
// Too slow - optimize for speed
adjustments.enableDistributedSearch = true
adjustments.maxNodesPerPartition = Math.max(10000, (this.cachedConfig?.recommendedConfig.maxNodesPerPartition || 50000) * 0.8)
} else if (metrics.averageSearchTime < 50) {
// Very fast - can optimize for quality
@ -282,7 +280,6 @@ export class AutoConfiguration {
targetSearchLatency: 150,
enablePartitioning: datasetSize > 25000,
enableCompression: memoryBudget < 2 * 1024 * 1024 * 1024,
enableDistributedSearch: resources.cpuCores > 2 && datasetSize > 50000,
enablePredictiveCaching: true,
partitionStrategy: 'semantic' as const,
maxNodesPerPartition: 50000,
@ -316,7 +313,6 @@ export class AutoConfiguration {
config.maxNodesPerPartition = 100000
} else if (datasetSize < 10000) {
config.enablePartitioning = false
config.enableDistributedSearch = false
config.partitionStrategy = 'semantic' // Keep semantic but disable partitioning
}

1
src/utils/metadataNamespace.ts
View file

@ -25,7 +25,6 @@ export interface BrainyInternalMetadata {
updated: number // Unix timestamp
// Optional internal fields
partition?: number // For distributed mode
domain?: string // Domain classification
priority?: number // Query priority hint
ttl?: number // Time to live

3
src/utils/statisticsCollector.ts
View file

@ -391,7 +391,8 @@ export class StatisticsCollector {
}
/**
* Merge statistics from storage (for distributed systems)
* Merge persisted statistics from storage into the in-memory collector
* (used when restoring counts on startup).
*/
mergeFromStorage(stored: Partial<StatisticsData>): void {
// Merge content types