feat(distributed): add distributed mode with multi-instance coordination
Implements Phase 1 and Phase 2 of distributed enhancements for horizontal scaling: Phase 1 - Zero-Config Distributed Mode: - Add DistributedConfigManager for shared S3 configuration coordination - Implement explicit role configuration (reader/writer/hybrid) for safety - Add instance registration with heartbeat and health monitoring - Create hash-based partitioner for deterministic data distribution Phase 2 - Intelligent Data Management: - Add DomainDetector for automatic data categorization (medical, legal, product, etc.) - Implement domain-aware search filtering for improved relevance - Create role-based operational modes with specific optimizations - Add HealthMonitor for comprehensive metrics tracking Key Features: - Multi-writer support with consistent hash partitioning - Reader instances optimize for 80% cache utilization - Writer instances optimize for batched writes - Automatic domain detection and tagging - Real-time health monitoring across all instances - Cross-platform crypto utilities for browser compatibility Safety Improvements: - Require explicit role configuration (no automatic assignment) - Validate role compatibility on startup - Track instance health and performance metrics Testing: - Add comprehensive test suite for distributed features - All 25 distributed tests passing - Fixed domain filtering in search functionality Documentation: - Update README with distributed mode highlights - Add examples showing reader/writer setup - Document new capabilities and benefits 🤖 Generated with Claude Code https://claude.ai/code Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
parent
29625cc1af
commit
26e9c26852
12 changed files with 3301 additions and 3 deletions
|
|
@ -47,6 +47,14 @@ import {
|
|||
prepareJsonForVectorization,
|
||||
extractFieldFromJson
|
||||
} from './utils/jsonProcessing.js'
|
||||
import { DistributedConfig } from './types/distributedTypes.js'
|
||||
import {
|
||||
DistributedConfigManager,
|
||||
HashPartitioner,
|
||||
OperationalModeFactory,
|
||||
DomainDetector,
|
||||
HealthMonitor
|
||||
} from './distributed/index.js'
|
||||
|
||||
export interface BrainyDataConfig {
|
||||
/**
|
||||
|
|
@ -260,6 +268,12 @@ export interface BrainyDataConfig {
|
|||
updateIndex?: boolean
|
||||
}
|
||||
|
||||
/**
|
||||
* Distributed mode configuration
|
||||
* Enables coordination across multiple Brainy instances
|
||||
*/
|
||||
distributed?: DistributedConfig | boolean
|
||||
|
||||
/**
|
||||
* Cache configuration for optimizing search performance
|
||||
* Controls how the system caches data for faster access
|
||||
|
|
@ -378,6 +392,14 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
|
|||
private remoteServerConfig: BrainyDataConfig['remoteServer'] | null = null
|
||||
private serverSearchConduit: ServerSearchConduitAugmentation | null = null
|
||||
private serverConnection: WebSocketConnection | null = null
|
||||
|
||||
// Distributed mode properties
|
||||
private distributedConfig: DistributedConfig | null = null
|
||||
private configManager: DistributedConfigManager | null = null
|
||||
private partitioner: HashPartitioner | null = null
|
||||
private operationalMode: any = null
|
||||
private domainDetector: DomainDetector | null = null
|
||||
private healthMonitor: HealthMonitor | null = null
|
||||
|
||||
/**
|
||||
* Get the vector dimensions
|
||||
|
|
@ -513,6 +535,19 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
|
|||
...config.cache
|
||||
}
|
||||
}
|
||||
|
||||
// Store distributed configuration
|
||||
if (config.distributed) {
|
||||
if (typeof config.distributed === 'boolean') {
|
||||
// Auto-mode enabled
|
||||
this.distributedConfig = {
|
||||
enabled: true
|
||||
}
|
||||
} else {
|
||||
// Explicit configuration
|
||||
this.distributedConfig = config.distributed
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -1005,6 +1040,11 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
|
|||
|
||||
// Initialize storage
|
||||
await this.storage!.init()
|
||||
|
||||
// Initialize distributed mode if configured
|
||||
if (this.distributedConfig) {
|
||||
await this.initializeDistributedMode()
|
||||
}
|
||||
|
||||
// If using optimized index, set the storage adapter
|
||||
if (this.useOptimizedIndex && this.index instanceof HNSWIndexOptimized) {
|
||||
|
|
@ -1076,6 +1116,113 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
|
|||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Initialize distributed mode
|
||||
* Sets up configuration management, partitioning, and operational modes
|
||||
*/
|
||||
private async initializeDistributedMode(): Promise<void> {
|
||||
if (!this.storage) {
|
||||
throw new Error('Storage must be initialized before distributed mode')
|
||||
}
|
||||
|
||||
// Create configuration manager with mode hints
|
||||
this.configManager = new DistributedConfigManager(
|
||||
this.storage,
|
||||
this.distributedConfig || undefined,
|
||||
{ readOnly: this.readOnly, writeOnly: this.writeOnly }
|
||||
)
|
||||
|
||||
// Initialize configuration
|
||||
const sharedConfig = await this.configManager.initialize()
|
||||
|
||||
// Create partitioner based on strategy
|
||||
if (sharedConfig.settings.partitionStrategy === 'hash') {
|
||||
this.partitioner = new HashPartitioner(sharedConfig)
|
||||
} else {
|
||||
// Default to hash partitioner for now
|
||||
this.partitioner = new HashPartitioner(sharedConfig)
|
||||
}
|
||||
|
||||
// Create operational mode based on role
|
||||
const role = this.configManager.getRole()
|
||||
this.operationalMode = OperationalModeFactory.createMode(role)
|
||||
|
||||
// Validate that role matches the configured mode
|
||||
// Don't override explicitly set readOnly/writeOnly
|
||||
if (role === 'reader' && !this.readOnly) {
|
||||
console.warn('Distributed role is "reader" but readOnly is not set. Setting readOnly=true for consistency.')
|
||||
this.readOnly = true
|
||||
this.writeOnly = false
|
||||
} else if (role === 'writer' && !this.writeOnly) {
|
||||
console.warn('Distributed role is "writer" but writeOnly is not set. Setting writeOnly=true for consistency.')
|
||||
this.readOnly = false
|
||||
this.writeOnly = true
|
||||
} else if (role === 'hybrid' && (this.readOnly || this.writeOnly)) {
|
||||
console.warn('Distributed role is "hybrid" but readOnly or writeOnly is set. Clearing both for hybrid mode.')
|
||||
this.readOnly = false
|
||||
this.writeOnly = false
|
||||
}
|
||||
|
||||
// Apply cache configuration from operational mode
|
||||
const modeCache = this.operationalMode.cacheStrategy
|
||||
if (modeCache) {
|
||||
this.cacheConfig = {
|
||||
...this.cacheConfig,
|
||||
hotCacheMaxSize: modeCache.hotCacheRatio * 1000000, // Convert ratio to size
|
||||
hotCacheEvictionThreshold: modeCache.hotCacheRatio,
|
||||
warmCacheTTL: modeCache.ttl,
|
||||
batchSize: modeCache.writeBufferSize || 100
|
||||
}
|
||||
|
||||
// Update storage cache config if it supports it
|
||||
if (this.storage && 'updateCacheConfig' in this.storage) {
|
||||
(this.storage as any).updateCacheConfig(this.cacheConfig)
|
||||
}
|
||||
}
|
||||
|
||||
// Initialize domain detector
|
||||
this.domainDetector = new DomainDetector()
|
||||
|
||||
// Initialize health monitor
|
||||
this.healthMonitor = new HealthMonitor(this.configManager)
|
||||
this.healthMonitor.start()
|
||||
|
||||
// Set up config update listener
|
||||
this.configManager.setOnConfigUpdate((config) => {
|
||||
this.handleDistributedConfigUpdate(config)
|
||||
})
|
||||
|
||||
if (this.loggingConfig?.verbose) {
|
||||
console.log(`Distributed mode initialized as ${role} with ${sharedConfig.settings.partitionStrategy} partitioning`)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Handle distributed configuration updates
|
||||
*/
|
||||
private handleDistributedConfigUpdate(config: any): void {
|
||||
// Update partitioner if needed
|
||||
if (this.partitioner && config.settings) {
|
||||
this.partitioner = new HashPartitioner(config)
|
||||
}
|
||||
|
||||
// Log configuration update
|
||||
if (this.loggingConfig?.verbose) {
|
||||
console.log('Distributed configuration updated:', config.version)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get distributed health status
|
||||
* @returns Health status if distributed mode is enabled
|
||||
*/
|
||||
public getHealthStatus(): any {
|
||||
if (this.healthMonitor) {
|
||||
return this.healthMonitor.getHealthEndpointData()
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
/**
|
||||
* Connect to a remote Brainy server for search operations
|
||||
* @param serverUrl WebSocket URL of the remote Brainy server
|
||||
|
|
@ -1338,6 +1485,34 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
|
|||
if (metadata && typeof metadata === 'object') {
|
||||
// Always make a copy without adding the ID
|
||||
metadataToSave = { ...metadata }
|
||||
|
||||
// Add domain metadata if distributed mode is enabled
|
||||
if (this.domainDetector) {
|
||||
// First check if domain is already in metadata
|
||||
if ((metadataToSave as any).domain) {
|
||||
// Domain already specified, keep it
|
||||
const domainInfo = this.domainDetector.detectDomain(metadataToSave)
|
||||
if (domainInfo.domainMetadata) {
|
||||
(metadataToSave as any).domainMetadata = domainInfo.domainMetadata
|
||||
}
|
||||
} else {
|
||||
// Try to detect domain from the data
|
||||
const dataToAnalyze = Array.isArray(vectorOrData) ? metadata : vectorOrData
|
||||
const domainInfo = this.domainDetector.detectDomain(dataToAnalyze)
|
||||
if (domainInfo.domain) {
|
||||
(metadataToSave as any).domain = domainInfo.domain
|
||||
if (domainInfo.domainMetadata) {
|
||||
(metadataToSave as any).domainMetadata = domainInfo.domainMetadata
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Add partition information if distributed mode is enabled
|
||||
if (this.partitioner) {
|
||||
const partition = this.partitioner.getPartition(id)
|
||||
;(metadataToSave as any).partition = partition
|
||||
}
|
||||
}
|
||||
|
||||
await this.storage!.saveMetadata(id, metadataToSave)
|
||||
|
|
@ -1350,6 +1525,12 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
|
|||
|
||||
// Update HNSW index size (excluding verbs)
|
||||
await this.storage!.updateHnswIndexSize(await this.getNounCount())
|
||||
|
||||
// Update health metrics if in distributed mode
|
||||
if (this.healthMonitor) {
|
||||
const vectorCount = await this.getNounCount()
|
||||
this.healthMonitor.updateVectorCount(vectorCount)
|
||||
}
|
||||
|
||||
// If addToRemote is true and we're connected to a remote server, add to remote as well
|
||||
if (options.addToRemote && this.isConnectedToRemoteServer()) {
|
||||
|
|
@ -1365,6 +1546,12 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
|
|||
return id
|
||||
} catch (error) {
|
||||
console.error('Failed to add vector:', error)
|
||||
|
||||
// Track error in health monitor
|
||||
if (this.healthMonitor) {
|
||||
this.healthMonitor.recordRequest(0, true)
|
||||
}
|
||||
|
||||
throw new Error(`Failed to add vector: ${error}`)
|
||||
}
|
||||
}
|
||||
|
|
@ -1865,8 +2052,10 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
|
|||
verbDirection?: 'outgoing' | 'incoming' | 'both' // Direction of verbs to consider when searching connected nouns
|
||||
service?: string // Filter results by the service that created the data
|
||||
searchField?: string // Optional specific field to search within JSON documents
|
||||
filter?: { domain?: string } // Filter results by domain
|
||||
} = {}
|
||||
): Promise<SearchResult<T>[]> {
|
||||
const startTime = Date.now()
|
||||
// Validate input is not null or undefined
|
||||
if (queryVectorOrData === null || queryVectorOrData === undefined) {
|
||||
throw new Error('Query cannot be null or undefined')
|
||||
|
|
@ -1925,7 +2114,25 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
|
|||
}
|
||||
|
||||
// Default behavior (backward compatible): search locally
|
||||
return this.searchLocal(queryVectorOrData, k, options)
|
||||
try {
|
||||
const results = await this.searchLocal(queryVectorOrData, k, options)
|
||||
|
||||
// Track successful search in health monitor
|
||||
if (this.healthMonitor) {
|
||||
const latency = Date.now() - startTime
|
||||
this.healthMonitor.recordRequest(latency, false)
|
||||
this.healthMonitor.recordCacheAccess(results.length > 0)
|
||||
}
|
||||
|
||||
return results
|
||||
} catch (error) {
|
||||
// Track error in health monitor
|
||||
if (this.healthMonitor) {
|
||||
const latency = Date.now() - startTime
|
||||
this.healthMonitor.recordRequest(latency, true)
|
||||
}
|
||||
throw error
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -1945,6 +2152,7 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
|
|||
service?: string // Filter results by the service that created the data
|
||||
searchField?: string // Optional specific field to search within JSON documents
|
||||
priorityFields?: string[] // Fields to prioritize when searching JSON documents
|
||||
filter?: { domain?: string } // Filter results by domain
|
||||
} = {}
|
||||
): Promise<SearchResult<T>[]> {
|
||||
if (!this.isInitialized) {
|
||||
|
|
@ -2024,7 +2232,16 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
|
|||
if (result.metadata && typeof result.metadata === 'object') {
|
||||
const metadata = result.metadata as Record<string, any>
|
||||
// Exclude placeholder nouns from search results
|
||||
return !metadata.isPlaceholder
|
||||
if (metadata.isPlaceholder) {
|
||||
return false
|
||||
}
|
||||
|
||||
// Apply domain filter if specified
|
||||
if (options.filter?.domain) {
|
||||
if (metadata.domain !== options.filter.domain) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
}
|
||||
return true
|
||||
})
|
||||
|
|
@ -4859,6 +5076,30 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
|
|||
// Sort by score and limit to k results
|
||||
return allResults.sort((a, b) => b.score - a.score).slice(0, k)
|
||||
}
|
||||
|
||||
/**
|
||||
* Cleanup distributed resources
|
||||
* Should be called when shutting down the instance
|
||||
*/
|
||||
public async cleanup(): Promise<void> {
|
||||
// Stop real-time updates
|
||||
if (this.updateTimerId) {
|
||||
clearInterval(this.updateTimerId)
|
||||
this.updateTimerId = null
|
||||
}
|
||||
|
||||
// Clean up distributed mode resources
|
||||
if (this.healthMonitor) {
|
||||
this.healthMonitor.stop()
|
||||
}
|
||||
|
||||
if (this.configManager) {
|
||||
await this.configManager.cleanup()
|
||||
}
|
||||
|
||||
// Clean up worker pools
|
||||
await cleanupWorkerPools()
|
||||
}
|
||||
}
|
||||
|
||||
// Export distance functions for convenience
|
||||
|
|
|
|||
382
src/distributed/configManager.ts
Normal file
382
src/distributed/configManager.ts
Normal file
|
|
@ -0,0 +1,382 @@
|
|||
/**
|
||||
* Distributed Configuration Manager
|
||||
* Manages shared configuration in S3 for distributed Brainy instances
|
||||
*/
|
||||
|
||||
import { v4 as uuidv4 } from 'uuid'
|
||||
import {
|
||||
DistributedConfig,
|
||||
SharedConfig,
|
||||
InstanceInfo,
|
||||
InstanceRole
|
||||
} from '../types/distributedTypes.js'
|
||||
import { StorageAdapter } from '../coreTypes.js'
|
||||
|
||||
export class DistributedConfigManager {
|
||||
private config: SharedConfig | null = null
|
||||
private instanceId: string
|
||||
private role: InstanceRole | undefined
|
||||
private configPath: string
|
||||
private heartbeatInterval: number
|
||||
private configCheckInterval: number
|
||||
private instanceTimeout: number
|
||||
private storage: StorageAdapter
|
||||
private heartbeatTimer?: NodeJS.Timeout
|
||||
private configWatchTimer?: NodeJS.Timeout
|
||||
private lastConfigVersion: number = 0
|
||||
private onConfigUpdate?: (config: SharedConfig) => void
|
||||
|
||||
constructor(
|
||||
storage: StorageAdapter,
|
||||
distributedConfig?: DistributedConfig,
|
||||
brainyMode?: { readOnly?: boolean; writeOnly?: boolean }
|
||||
) {
|
||||
this.storage = storage
|
||||
this.instanceId = distributedConfig?.instanceId || `instance-${uuidv4()}`
|
||||
this.configPath = distributedConfig?.configPath || '_brainy/config.json'
|
||||
this.heartbeatInterval = distributedConfig?.heartbeatInterval || 30000
|
||||
this.configCheckInterval = distributedConfig?.configCheckInterval || 10000
|
||||
this.instanceTimeout = distributedConfig?.instanceTimeout || 60000
|
||||
|
||||
// Set role from distributed config if provided
|
||||
if (distributedConfig?.role) {
|
||||
this.role = distributedConfig.role
|
||||
}
|
||||
// Infer role from Brainy's read/write mode if not explicitly set
|
||||
else if (brainyMode) {
|
||||
if (brainyMode.writeOnly) {
|
||||
this.role = 'writer'
|
||||
} else if (brainyMode.readOnly) {
|
||||
this.role = 'reader'
|
||||
}
|
||||
// If neither readOnly nor writeOnly, role must be explicitly set
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Initialize the distributed configuration
|
||||
*/
|
||||
async initialize(): Promise<SharedConfig> {
|
||||
// Load or create configuration
|
||||
this.config = await this.loadOrCreateConfig()
|
||||
|
||||
// Determine role if not explicitly set
|
||||
if (!this.role) {
|
||||
this.role = await this.determineRole()
|
||||
}
|
||||
|
||||
// Register this instance
|
||||
await this.registerInstance()
|
||||
|
||||
// Start heartbeat and config watching
|
||||
this.startHeartbeat()
|
||||
this.startConfigWatch()
|
||||
|
||||
return this.config
|
||||
}
|
||||
|
||||
/**
|
||||
* Load existing config or create new one
|
||||
*/
|
||||
private async loadOrCreateConfig(): Promise<SharedConfig> {
|
||||
try {
|
||||
// Use metadata storage with a special ID for config
|
||||
const configData = await this.storage.getMetadata('_distributed_config')
|
||||
if (configData) {
|
||||
this.lastConfigVersion = configData.version
|
||||
return configData as SharedConfig
|
||||
}
|
||||
} catch (error) {
|
||||
// Config doesn't exist yet
|
||||
}
|
||||
|
||||
// Create default config
|
||||
const newConfig: SharedConfig = {
|
||||
version: 1,
|
||||
updated: new Date().toISOString(),
|
||||
settings: {
|
||||
partitionStrategy: 'hash',
|
||||
partitionCount: 100,
|
||||
embeddingModel: 'text-embedding-ada-002',
|
||||
dimensions: 1536,
|
||||
distanceMetric: 'cosine',
|
||||
hnswParams: {
|
||||
M: 16,
|
||||
efConstruction: 200
|
||||
}
|
||||
},
|
||||
instances: {}
|
||||
}
|
||||
|
||||
await this.saveConfig(newConfig)
|
||||
return newConfig
|
||||
}
|
||||
|
||||
/**
|
||||
* Determine role based on configuration
|
||||
* IMPORTANT: Role must be explicitly set - no automatic assignment based on order
|
||||
*/
|
||||
private async determineRole(): Promise<InstanceRole> {
|
||||
// Check environment variable first
|
||||
if (process.env.BRAINY_ROLE) {
|
||||
const role = process.env.BRAINY_ROLE.toLowerCase()
|
||||
if (role === 'writer' || role === 'reader' || role === 'hybrid') {
|
||||
return role as InstanceRole
|
||||
}
|
||||
throw new Error(`Invalid BRAINY_ROLE: ${process.env.BRAINY_ROLE}. Must be 'writer', 'reader', or 'hybrid'`)
|
||||
}
|
||||
|
||||
// Check if explicitly passed in distributed config
|
||||
if (this.role) {
|
||||
return this.role
|
||||
}
|
||||
|
||||
// DO NOT auto-assign roles based on deployment order or existing instances
|
||||
// This is dangerous and can lead to data corruption or loss
|
||||
throw new Error(
|
||||
'Distributed mode requires explicit role configuration. ' +
|
||||
'Set BRAINY_ROLE environment variable or pass role in distributed config. ' +
|
||||
'Valid roles: "writer", "reader", "hybrid"'
|
||||
)
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if an instance is still alive
|
||||
*/
|
||||
private isInstanceAlive(instance: InstanceInfo): boolean {
|
||||
const lastSeen = new Date(instance.lastHeartbeat).getTime()
|
||||
const now = Date.now()
|
||||
return (now - lastSeen) < this.instanceTimeout
|
||||
}
|
||||
|
||||
/**
|
||||
* Register this instance in the shared config
|
||||
*/
|
||||
private async registerInstance(): Promise<void> {
|
||||
if (!this.config) return
|
||||
|
||||
// Role must be set by this point
|
||||
if (!this.role) {
|
||||
throw new Error('Cannot register instance without a role')
|
||||
}
|
||||
|
||||
const instanceInfo: InstanceInfo = {
|
||||
role: this.role,
|
||||
status: 'active',
|
||||
lastHeartbeat: new Date().toISOString(),
|
||||
metrics: {
|
||||
memoryUsage: process.memoryUsage().heapUsed
|
||||
}
|
||||
}
|
||||
|
||||
// Add endpoint if available
|
||||
if (process.env.SERVICE_ENDPOINT) {
|
||||
instanceInfo.endpoint = process.env.SERVICE_ENDPOINT
|
||||
}
|
||||
|
||||
this.config.instances[this.instanceId] = instanceInfo
|
||||
await this.saveConfig(this.config)
|
||||
}
|
||||
|
||||
/**
|
||||
* Save configuration with version increment
|
||||
*/
|
||||
private async saveConfig(config: SharedConfig): Promise<void> {
|
||||
config.version++
|
||||
config.updated = new Date().toISOString()
|
||||
this.lastConfigVersion = config.version
|
||||
|
||||
// Use metadata storage with a special ID for config
|
||||
await this.storage.saveMetadata('_distributed_config', config)
|
||||
|
||||
this.config = config
|
||||
}
|
||||
|
||||
/**
|
||||
* Start heartbeat to keep instance alive in config
|
||||
*/
|
||||
private startHeartbeat(): void {
|
||||
this.heartbeatTimer = setInterval(async () => {
|
||||
await this.updateHeartbeat()
|
||||
}, this.heartbeatInterval)
|
||||
}
|
||||
|
||||
/**
|
||||
* Update heartbeat and clean stale instances
|
||||
*/
|
||||
private async updateHeartbeat(): Promise<void> {
|
||||
if (!this.config) return
|
||||
|
||||
// Reload config to get latest state
|
||||
try {
|
||||
const latestConfig = await this.loadConfig()
|
||||
if (latestConfig) {
|
||||
this.config = latestConfig
|
||||
}
|
||||
} catch (error) {
|
||||
console.error('Failed to reload config:', error)
|
||||
}
|
||||
|
||||
// Update our heartbeat
|
||||
if (this.config.instances[this.instanceId]) {
|
||||
this.config.instances[this.instanceId].lastHeartbeat = new Date().toISOString()
|
||||
this.config.instances[this.instanceId].status = 'active'
|
||||
|
||||
// Update metrics if available
|
||||
this.config.instances[this.instanceId].metrics = {
|
||||
memoryUsage: process.memoryUsage().heapUsed
|
||||
}
|
||||
} else {
|
||||
// Re-register if we were removed
|
||||
await this.registerInstance()
|
||||
return
|
||||
}
|
||||
|
||||
// Clean up stale instances
|
||||
const now = Date.now()
|
||||
let hasChanges = false
|
||||
|
||||
for (const [id, instance] of Object.entries(this.config.instances)) {
|
||||
if (id === this.instanceId) continue
|
||||
|
||||
const lastSeen = new Date(instance.lastHeartbeat).getTime()
|
||||
if (now - lastSeen > this.instanceTimeout) {
|
||||
delete this.config.instances[id]
|
||||
hasChanges = true
|
||||
}
|
||||
}
|
||||
|
||||
// Save if there were changes
|
||||
if (hasChanges) {
|
||||
await this.saveConfig(this.config)
|
||||
} else {
|
||||
// Just update our heartbeat without version increment
|
||||
await this.storage.saveMetadata('_distributed_config', this.config)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Start watching for config changes
|
||||
*/
|
||||
private startConfigWatch(): void {
|
||||
this.configWatchTimer = setInterval(async () => {
|
||||
await this.checkForConfigUpdates()
|
||||
}, this.configCheckInterval)
|
||||
}
|
||||
|
||||
/**
|
||||
* Check for configuration updates
|
||||
*/
|
||||
private async checkForConfigUpdates(): Promise<void> {
|
||||
try {
|
||||
const latestConfig = await this.loadConfig()
|
||||
if (!latestConfig) return
|
||||
|
||||
if (latestConfig.version > this.lastConfigVersion) {
|
||||
this.config = latestConfig
|
||||
this.lastConfigVersion = latestConfig.version
|
||||
|
||||
// Notify listeners of config update
|
||||
if (this.onConfigUpdate) {
|
||||
this.onConfigUpdate(latestConfig)
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
console.error('Failed to check config updates:', error)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Load configuration from storage
|
||||
*/
|
||||
private async loadConfig(): Promise<SharedConfig | null> {
|
||||
try {
|
||||
const configData = await this.storage.getMetadata('_distributed_config')
|
||||
if (configData) {
|
||||
return configData as SharedConfig
|
||||
}
|
||||
} catch (error) {
|
||||
console.error('Failed to load config:', error)
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
/**
|
||||
* Get current configuration
|
||||
*/
|
||||
getConfig(): SharedConfig | null {
|
||||
return this.config
|
||||
}
|
||||
|
||||
/**
|
||||
* Get instance role
|
||||
*/
|
||||
getRole(): InstanceRole {
|
||||
if (!this.role) {
|
||||
throw new Error('Role not initialized')
|
||||
}
|
||||
return this.role
|
||||
}
|
||||
|
||||
/**
|
||||
* Get instance ID
|
||||
*/
|
||||
getInstanceId(): string {
|
||||
return this.instanceId
|
||||
}
|
||||
|
||||
/**
|
||||
* Set config update callback
|
||||
*/
|
||||
setOnConfigUpdate(callback: (config: SharedConfig) => void): void {
|
||||
this.onConfigUpdate = callback
|
||||
}
|
||||
|
||||
/**
|
||||
* Get all active instances of a specific role
|
||||
*/
|
||||
getInstancesByRole(role: InstanceRole): InstanceInfo[] {
|
||||
if (!this.config) return []
|
||||
|
||||
return Object.entries(this.config.instances)
|
||||
.filter(([_, instance]) =>
|
||||
instance.role === role &&
|
||||
this.isInstanceAlive(instance)
|
||||
)
|
||||
.map(([_, instance]) => instance)
|
||||
}
|
||||
|
||||
/**
|
||||
* Update instance metrics
|
||||
*/
|
||||
async updateMetrics(metrics: Partial<InstanceInfo['metrics']>): Promise<void> {
|
||||
if (!this.config || !this.config.instances[this.instanceId]) return
|
||||
|
||||
this.config.instances[this.instanceId].metrics = {
|
||||
...this.config.instances[this.instanceId].metrics,
|
||||
...metrics
|
||||
}
|
||||
|
||||
// Don't increment version for metric updates
|
||||
await this.storage.saveMetadata('_distributed_config', this.config)
|
||||
}
|
||||
|
||||
/**
|
||||
* Cleanup resources
|
||||
*/
|
||||
async cleanup(): Promise<void> {
|
||||
// Stop timers
|
||||
if (this.heartbeatTimer) {
|
||||
clearInterval(this.heartbeatTimer)
|
||||
}
|
||||
if (this.configWatchTimer) {
|
||||
clearInterval(this.configWatchTimer)
|
||||
}
|
||||
|
||||
// Mark instance as inactive
|
||||
if (this.config && this.config.instances[this.instanceId]) {
|
||||
this.config.instances[this.instanceId].status = 'inactive'
|
||||
await this.saveConfig(this.config)
|
||||
}
|
||||
}
|
||||
}
|
||||
323
src/distributed/domainDetector.ts
Normal file
323
src/distributed/domainDetector.ts
Normal file
|
|
@ -0,0 +1,323 @@
|
|||
/**
|
||||
* Domain Detector
|
||||
* Automatically detects and manages data domains for logical separation
|
||||
*/
|
||||
|
||||
import { DomainMetadata } from '../types/distributedTypes.js'
|
||||
|
||||
export interface DomainPattern {
|
||||
domain: string
|
||||
patterns: {
|
||||
fields?: string[]
|
||||
keywords?: string[]
|
||||
regex?: RegExp
|
||||
}
|
||||
priority?: number
|
||||
}
|
||||
|
||||
export class DomainDetector {
|
||||
private domainPatterns: DomainPattern[] = [
|
||||
{
|
||||
domain: 'medical',
|
||||
patterns: {
|
||||
fields: ['symptoms', 'diagnosis', 'treatment', 'medication', 'patient'],
|
||||
keywords: ['medical', 'health', 'disease', 'symptom', 'treatment', 'doctor', 'patient']
|
||||
},
|
||||
priority: 1
|
||||
},
|
||||
{
|
||||
domain: 'legal',
|
||||
patterns: {
|
||||
fields: ['contract', 'clause', 'litigation', 'statute', 'jurisdiction'],
|
||||
keywords: ['legal', 'law', 'contract', 'court', 'attorney', 'litigation', 'statute']
|
||||
},
|
||||
priority: 1
|
||||
},
|
||||
{
|
||||
domain: 'product',
|
||||
patterns: {
|
||||
fields: ['price', 'sku', 'inventory', 'category', 'brand'],
|
||||
keywords: ['product', 'price', 'sale', 'inventory', 'catalog', 'item', 'sku']
|
||||
},
|
||||
priority: 1
|
||||
},
|
||||
{
|
||||
domain: 'customer',
|
||||
patterns: {
|
||||
fields: ['customerId', 'email', 'phone', 'address', 'orders'],
|
||||
keywords: ['customer', 'client', 'user', 'account', 'profile', 'contact']
|
||||
},
|
||||
priority: 1
|
||||
},
|
||||
{
|
||||
domain: 'financial',
|
||||
patterns: {
|
||||
fields: ['amount', 'currency', 'transaction', 'balance', 'account'],
|
||||
keywords: ['financial', 'money', 'payment', 'transaction', 'bank', 'credit', 'debit']
|
||||
},
|
||||
priority: 1
|
||||
},
|
||||
{
|
||||
domain: 'technical',
|
||||
patterns: {
|
||||
fields: ['code', 'function', 'error', 'stack', 'api'],
|
||||
keywords: ['code', 'software', 'api', 'error', 'debug', 'function', 'class', 'method']
|
||||
},
|
||||
priority: 2
|
||||
}
|
||||
]
|
||||
|
||||
private customPatterns: DomainPattern[] = []
|
||||
private domainStats: Map<string, number> = new Map()
|
||||
|
||||
/**
|
||||
* Detect domain from data object
|
||||
* @param data - The data object to analyze
|
||||
* @returns The detected domain and metadata
|
||||
*/
|
||||
detectDomain(data: any): DomainMetadata {
|
||||
if (!data || typeof data !== 'object') {
|
||||
return { domain: 'general' }
|
||||
}
|
||||
|
||||
// Check for explicit domain field
|
||||
if (data.domain && typeof data.domain === 'string') {
|
||||
this.updateStats(data.domain)
|
||||
return {
|
||||
domain: data.domain,
|
||||
domainMetadata: this.extractDomainMetadata(data, data.domain)
|
||||
}
|
||||
}
|
||||
|
||||
// Score each domain pattern
|
||||
const scores = new Map<string, number>()
|
||||
|
||||
// Check custom patterns first (higher priority)
|
||||
for (const pattern of this.customPatterns) {
|
||||
const score = this.scorePattern(data, pattern)
|
||||
if (score > 0) {
|
||||
scores.set(pattern.domain, score * (pattern.priority || 1))
|
||||
}
|
||||
}
|
||||
|
||||
// Check default patterns
|
||||
for (const pattern of this.domainPatterns) {
|
||||
const score = this.scorePattern(data, pattern)
|
||||
if (score > 0) {
|
||||
const currentScore = scores.get(pattern.domain) || 0
|
||||
scores.set(pattern.domain, currentScore + score * (pattern.priority || 1))
|
||||
}
|
||||
}
|
||||
|
||||
// Find highest scoring domain
|
||||
let bestDomain = 'general'
|
||||
let bestScore = 0
|
||||
|
||||
for (const [domain, score] of scores.entries()) {
|
||||
if (score > bestScore) {
|
||||
bestDomain = domain
|
||||
bestScore = score
|
||||
}
|
||||
}
|
||||
|
||||
this.updateStats(bestDomain)
|
||||
|
||||
return {
|
||||
domain: bestDomain,
|
||||
domainMetadata: this.extractDomainMetadata(data, bestDomain)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Score a data object against a domain pattern
|
||||
*/
|
||||
private scorePattern(data: any, pattern: DomainPattern): number {
|
||||
let score = 0
|
||||
|
||||
// Check field matches
|
||||
if (pattern.patterns.fields) {
|
||||
const dataKeys = Object.keys(data)
|
||||
for (const field of pattern.patterns.fields) {
|
||||
if (dataKeys.some(key => key.toLowerCase().includes(field.toLowerCase()))) {
|
||||
score += 2 // Field match is strong signal
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Check keyword matches in values
|
||||
if (pattern.patterns.keywords) {
|
||||
const dataStr = JSON.stringify(data).toLowerCase()
|
||||
for (const keyword of pattern.patterns.keywords) {
|
||||
if (dataStr.includes(keyword.toLowerCase())) {
|
||||
score += 1
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Check regex patterns
|
||||
if (pattern.patterns.regex) {
|
||||
const dataStr = JSON.stringify(data)
|
||||
if (pattern.patterns.regex.test(dataStr)) {
|
||||
score += 3 // Regex match is very specific
|
||||
}
|
||||
}
|
||||
|
||||
return score
|
||||
}
|
||||
|
||||
/**
|
||||
* Extract domain-specific metadata
|
||||
*/
|
||||
private extractDomainMetadata(data: any, domain: string): Record<string, any> {
|
||||
const metadata: Record<string, any> = {}
|
||||
|
||||
switch (domain) {
|
||||
case 'medical':
|
||||
if (data.patientId) metadata.patientId = data.patientId
|
||||
if (data.condition) metadata.condition = data.condition
|
||||
if (data.severity) metadata.severity = data.severity
|
||||
break
|
||||
|
||||
case 'legal':
|
||||
if (data.caseId) metadata.caseId = data.caseId
|
||||
if (data.jurisdiction) metadata.jurisdiction = data.jurisdiction
|
||||
if (data.documentType) metadata.documentType = data.documentType
|
||||
break
|
||||
|
||||
case 'product':
|
||||
if (data.sku) metadata.sku = data.sku
|
||||
if (data.category) metadata.category = data.category
|
||||
if (data.brand) metadata.brand = data.brand
|
||||
if (data.price) metadata.priceRange = this.getPriceRange(data.price)
|
||||
break
|
||||
|
||||
case 'customer':
|
||||
if (data.customerId) metadata.customerId = data.customerId
|
||||
if (data.segment) metadata.segment = data.segment
|
||||
if (data.lifetime_value) metadata.valueCategory = this.getValueCategory(data.lifetime_value)
|
||||
break
|
||||
|
||||
case 'financial':
|
||||
if (data.accountId) metadata.accountId = data.accountId
|
||||
if (data.transactionType) metadata.transactionType = data.transactionType
|
||||
if (data.amount) metadata.amountRange = this.getAmountRange(data.amount)
|
||||
break
|
||||
|
||||
case 'technical':
|
||||
if (data.service) metadata.service = data.service
|
||||
if (data.environment) metadata.environment = data.environment
|
||||
if (data.severity) metadata.severity = data.severity
|
||||
break
|
||||
}
|
||||
|
||||
// Add detection confidence
|
||||
metadata.detectionConfidence = this.calculateConfidence(data, domain)
|
||||
|
||||
return metadata
|
||||
}
|
||||
|
||||
/**
|
||||
* Calculate detection confidence
|
||||
*/
|
||||
private calculateConfidence(data: any, domain: string): 'high' | 'medium' | 'low' {
|
||||
// If domain was explicitly specified
|
||||
if (data.domain === domain) return 'high'
|
||||
|
||||
// Check how many patterns matched
|
||||
const pattern = [...this.customPatterns, ...this.domainPatterns]
|
||||
.find(p => p.domain === domain)
|
||||
|
||||
if (!pattern) return 'low'
|
||||
|
||||
const score = this.scorePattern(data, pattern)
|
||||
if (score >= 5) return 'high'
|
||||
if (score >= 2) return 'medium'
|
||||
return 'low'
|
||||
}
|
||||
|
||||
/**
|
||||
* Categorize price ranges
|
||||
*/
|
||||
private getPriceRange(price: number): string {
|
||||
if (price < 10) return 'low'
|
||||
if (price < 100) return 'medium'
|
||||
if (price < 1000) return 'high'
|
||||
return 'premium'
|
||||
}
|
||||
|
||||
/**
|
||||
* Categorize customer value
|
||||
*/
|
||||
private getValueCategory(value: number): string {
|
||||
if (value < 100) return 'low'
|
||||
if (value < 1000) return 'medium'
|
||||
if (value < 10000) return 'high'
|
||||
return 'vip'
|
||||
}
|
||||
|
||||
/**
|
||||
* Categorize amount ranges
|
||||
*/
|
||||
private getAmountRange(amount: number): string {
|
||||
if (amount < 100) return 'micro'
|
||||
if (amount < 1000) return 'small'
|
||||
if (amount < 10000) return 'medium'
|
||||
if (amount < 100000) return 'large'
|
||||
return 'enterprise'
|
||||
}
|
||||
|
||||
/**
|
||||
* Add custom domain pattern
|
||||
* @param pattern - Custom domain pattern to add
|
||||
*/
|
||||
addCustomPattern(pattern: DomainPattern): void {
|
||||
// Remove existing pattern for same domain if exists
|
||||
this.customPatterns = this.customPatterns.filter(p => p.domain !== pattern.domain)
|
||||
this.customPatterns.push(pattern)
|
||||
}
|
||||
|
||||
/**
|
||||
* Remove custom domain pattern
|
||||
* @param domain - Domain to remove pattern for
|
||||
*/
|
||||
removeCustomPattern(domain: string): void {
|
||||
this.customPatterns = this.customPatterns.filter(p => p.domain !== domain)
|
||||
}
|
||||
|
||||
/**
|
||||
* Update domain statistics
|
||||
*/
|
||||
private updateStats(domain: string): void {
|
||||
const count = this.domainStats.get(domain) || 0
|
||||
this.domainStats.set(domain, count + 1)
|
||||
}
|
||||
|
||||
/**
|
||||
* Get domain statistics
|
||||
* @returns Map of domain to count
|
||||
*/
|
||||
getDomainStats(): Map<string, number> {
|
||||
return new Map(this.domainStats)
|
||||
}
|
||||
|
||||
/**
|
||||
* Clear domain statistics
|
||||
*/
|
||||
clearStats(): void {
|
||||
this.domainStats.clear()
|
||||
}
|
||||
|
||||
/**
|
||||
* Get all configured domains
|
||||
* @returns Array of domain names
|
||||
*/
|
||||
getConfiguredDomains(): string[] {
|
||||
const domains = new Set<string>()
|
||||
|
||||
for (const pattern of [...this.domainPatterns, ...this.customPatterns]) {
|
||||
domains.add(pattern.domain)
|
||||
}
|
||||
|
||||
return Array.from(domains).sort()
|
||||
}
|
||||
}
|
||||
170
src/distributed/hashPartitioner.ts
Normal file
170
src/distributed/hashPartitioner.ts
Normal file
|
|
@ -0,0 +1,170 @@
|
|||
/**
|
||||
* Hash-based Partitioner
|
||||
* Provides deterministic partitioning for distributed writes
|
||||
*/
|
||||
|
||||
import { getPartitionHash } from '../utils/crypto.js'
|
||||
import { SharedConfig } from '../types/distributedTypes.js'
|
||||
|
||||
export class HashPartitioner {
|
||||
private partitionCount: number
|
||||
private partitionPrefix: string = 'vectors/p'
|
||||
|
||||
constructor(config: SharedConfig) {
|
||||
this.partitionCount = config.settings.partitionCount || 100
|
||||
}
|
||||
|
||||
/**
|
||||
* Get partition for a given vector ID using deterministic hashing
|
||||
* @param vectorId - The unique identifier of the vector
|
||||
* @returns The partition path
|
||||
*/
|
||||
getPartition(vectorId: string): string {
|
||||
const hash = this.hashString(vectorId)
|
||||
const partitionIndex = hash % this.partitionCount
|
||||
return `${this.partitionPrefix}${partitionIndex.toString().padStart(3, '0')}`
|
||||
}
|
||||
|
||||
/**
|
||||
* Get partition with domain metadata (domain stored as metadata, not in path)
|
||||
* @param vectorId - The unique identifier of the vector
|
||||
* @param domain - The domain identifier (for metadata only)
|
||||
* @returns The partition path
|
||||
*/
|
||||
getPartitionWithDomain(vectorId: string, domain?: string): string {
|
||||
// Domain doesn't affect partitioning - it's just metadata
|
||||
return this.getPartition(vectorId)
|
||||
}
|
||||
|
||||
/**
|
||||
* Get all partition paths
|
||||
* @returns Array of all partition paths
|
||||
*/
|
||||
getAllPartitions(): string[] {
|
||||
const partitions: string[] = []
|
||||
for (let i = 0; i < this.partitionCount; i++) {
|
||||
partitions.push(`${this.partitionPrefix}${i.toString().padStart(3, '0')}`)
|
||||
}
|
||||
return partitions
|
||||
}
|
||||
|
||||
/**
|
||||
* Get partition index from partition path
|
||||
* @param partitionPath - The partition path
|
||||
* @returns The partition index
|
||||
*/
|
||||
getPartitionIndex(partitionPath: string): number {
|
||||
const match = partitionPath.match(/p(\d+)$/)
|
||||
if (match) {
|
||||
return parseInt(match[1], 10)
|
||||
}
|
||||
throw new Error(`Invalid partition path: ${partitionPath}`)
|
||||
}
|
||||
|
||||
/**
|
||||
* Hash a string to a number for consistent partitioning
|
||||
* @param str - The string to hash
|
||||
* @returns A positive integer hash
|
||||
*/
|
||||
private hashString(str: string): number {
|
||||
// Use our cross-platform hash function
|
||||
return getPartitionHash(str)
|
||||
}
|
||||
|
||||
/**
|
||||
* Get partitions for batch operations
|
||||
* Groups vector IDs by their target partition
|
||||
* @param vectorIds - Array of vector IDs
|
||||
* @returns Map of partition to vector IDs
|
||||
*/
|
||||
getPartitionsForBatch(vectorIds: string[]): Map<string, string[]> {
|
||||
const partitionMap = new Map<string, string[]>()
|
||||
|
||||
for (const id of vectorIds) {
|
||||
const partition = this.getPartition(id)
|
||||
if (!partitionMap.has(partition)) {
|
||||
partitionMap.set(partition, [])
|
||||
}
|
||||
partitionMap.get(partition)!.push(id)
|
||||
}
|
||||
|
||||
return partitionMap
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Affinity-based Partitioner
|
||||
* Extends HashPartitioner to prefer certain partitions for a writer
|
||||
* while maintaining correctness
|
||||
*/
|
||||
export class AffinityPartitioner extends HashPartitioner {
|
||||
private preferredPartitions: Set<number>
|
||||
private instanceId: string
|
||||
|
||||
constructor(config: SharedConfig, instanceId: string) {
|
||||
super(config)
|
||||
this.instanceId = instanceId
|
||||
this.preferredPartitions = this.calculatePreferredPartitions(config)
|
||||
}
|
||||
|
||||
/**
|
||||
* Calculate preferred partitions for this instance
|
||||
*/
|
||||
private calculatePreferredPartitions(config: SharedConfig): Set<number> {
|
||||
const partitionCount = config.settings.partitionCount || 100
|
||||
const writers = Object.entries(config.instances)
|
||||
.filter(([_, inst]) => inst.role === 'writer')
|
||||
.map(([id, _]) => id)
|
||||
.sort() // Ensure consistent ordering
|
||||
|
||||
const writerIndex = writers.indexOf(this.instanceId)
|
||||
if (writerIndex === -1) {
|
||||
// Not a writer or not found, no preferences
|
||||
return new Set()
|
||||
}
|
||||
|
||||
const writerCount = writers.length
|
||||
const partitionsPerWriter = Math.ceil(partitionCount / writerCount)
|
||||
|
||||
const preferred = new Set<number>()
|
||||
const start = writerIndex * partitionsPerWriter
|
||||
const end = Math.min(start + partitionsPerWriter, partitionCount)
|
||||
|
||||
for (let i = start; i < end; i++) {
|
||||
preferred.add(i)
|
||||
}
|
||||
|
||||
return preferred
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if a partition is preferred for this instance
|
||||
* @param partitionPath - The partition path
|
||||
* @returns Whether this partition is preferred
|
||||
*/
|
||||
isPreferredPartition(partitionPath: string): boolean {
|
||||
try {
|
||||
const index = this.getPartitionIndex(partitionPath)
|
||||
return this.preferredPartitions.has(index)
|
||||
} catch {
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get all preferred partitions for this instance
|
||||
* @returns Array of preferred partition paths
|
||||
*/
|
||||
getPreferredPartitions(): string[] {
|
||||
return Array.from(this.preferredPartitions)
|
||||
.map(index => `vectors/p${index.toString().padStart(3, '0')}`)
|
||||
}
|
||||
|
||||
/**
|
||||
* Update preferred partitions based on new config
|
||||
* @param config - The updated shared configuration
|
||||
*/
|
||||
updatePreferences(config: SharedConfig): void {
|
||||
this.preferredPartitions = this.calculatePreferredPartitions(config)
|
||||
}
|
||||
}
|
||||
301
src/distributed/healthMonitor.ts
Normal file
301
src/distributed/healthMonitor.ts
Normal file
|
|
@ -0,0 +1,301 @@
|
|||
/**
|
||||
* Health Monitor
|
||||
* Monitors and reports instance health in distributed deployments
|
||||
*/
|
||||
|
||||
import { DistributedConfigManager } from './configManager.js'
|
||||
import { InstanceInfo } from '../types/distributedTypes.js'
|
||||
|
||||
export interface HealthMetrics {
|
||||
vectorCount: number
|
||||
cacheHitRate: number
|
||||
memoryUsage: number
|
||||
cpuUsage?: number
|
||||
requestsPerSecond?: number
|
||||
averageLatency?: number
|
||||
errorRate?: number
|
||||
}
|
||||
|
||||
export interface HealthStatus {
|
||||
status: 'healthy' | 'degraded' | 'unhealthy'
|
||||
instanceId: string
|
||||
role: string
|
||||
uptime: number
|
||||
lastCheck: string
|
||||
metrics: HealthMetrics
|
||||
warnings?: string[]
|
||||
errors?: string[]
|
||||
}
|
||||
|
||||
export class HealthMonitor {
|
||||
private configManager: DistributedConfigManager
|
||||
private startTime: number
|
||||
private requestCount: number = 0
|
||||
private errorCount: number = 0
|
||||
private totalLatency: number = 0
|
||||
private cacheHits: number = 0
|
||||
private cacheMisses: number = 0
|
||||
private vectorCount: number = 0
|
||||
private checkInterval: number = 30000 // 30 seconds
|
||||
private healthCheckTimer?: NodeJS.Timeout
|
||||
private metricsWindow: number[] = [] // Sliding window for RPS calculation
|
||||
private latencyWindow: number[] = [] // Sliding window for latency
|
||||
private windowSize: number = 60000 // 1 minute window
|
||||
|
||||
constructor(configManager: DistributedConfigManager) {
|
||||
this.configManager = configManager
|
||||
this.startTime = Date.now()
|
||||
}
|
||||
|
||||
/**
|
||||
* Start health monitoring
|
||||
*/
|
||||
start(): void {
|
||||
// Initial health update
|
||||
this.updateHealth()
|
||||
|
||||
// Schedule periodic health checks
|
||||
this.healthCheckTimer = setInterval(() => {
|
||||
this.updateHealth()
|
||||
}, this.checkInterval)
|
||||
}
|
||||
|
||||
/**
|
||||
* Stop health monitoring
|
||||
*/
|
||||
stop(): void {
|
||||
if (this.healthCheckTimer) {
|
||||
clearInterval(this.healthCheckTimer)
|
||||
this.healthCheckTimer = undefined
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Update health status and metrics
|
||||
*/
|
||||
private async updateHealth(): Promise<void> {
|
||||
const metrics = this.collectMetrics()
|
||||
|
||||
// Update config with latest metrics
|
||||
await this.configManager.updateMetrics({
|
||||
vectorCount: metrics.vectorCount,
|
||||
cacheHitRate: metrics.cacheHitRate,
|
||||
memoryUsage: metrics.memoryUsage,
|
||||
cpuUsage: metrics.cpuUsage
|
||||
})
|
||||
|
||||
// Clean sliding windows
|
||||
this.cleanWindows()
|
||||
}
|
||||
|
||||
/**
|
||||
* Collect current metrics
|
||||
*/
|
||||
private collectMetrics(): HealthMetrics {
|
||||
const memUsage = process.memoryUsage()
|
||||
|
||||
return {
|
||||
vectorCount: this.vectorCount,
|
||||
cacheHitRate: this.calculateCacheHitRate(),
|
||||
memoryUsage: memUsage.heapUsed,
|
||||
cpuUsage: this.getCPUUsage(),
|
||||
requestsPerSecond: this.calculateRPS(),
|
||||
averageLatency: this.calculateAverageLatency(),
|
||||
errorRate: this.calculateErrorRate()
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Calculate cache hit rate
|
||||
*/
|
||||
private calculateCacheHitRate(): number {
|
||||
const total = this.cacheHits + this.cacheMisses
|
||||
if (total === 0) return 0
|
||||
return this.cacheHits / total
|
||||
}
|
||||
|
||||
/**
|
||||
* Calculate requests per second
|
||||
*/
|
||||
private calculateRPS(): number {
|
||||
const now = Date.now()
|
||||
const recentRequests = this.metricsWindow.filter(
|
||||
timestamp => now - timestamp < this.windowSize
|
||||
)
|
||||
return recentRequests.length / (this.windowSize / 1000)
|
||||
}
|
||||
|
||||
/**
|
||||
* Calculate average latency
|
||||
*/
|
||||
private calculateAverageLatency(): number {
|
||||
if (this.latencyWindow.length === 0) return 0
|
||||
const sum = this.latencyWindow.reduce((a, b) => a + b, 0)
|
||||
return sum / this.latencyWindow.length
|
||||
}
|
||||
|
||||
/**
|
||||
* Calculate error rate
|
||||
*/
|
||||
private calculateErrorRate(): number {
|
||||
if (this.requestCount === 0) return 0
|
||||
return this.errorCount / this.requestCount
|
||||
}
|
||||
|
||||
/**
|
||||
* Get CPU usage (simplified)
|
||||
*/
|
||||
private getCPUUsage(): number {
|
||||
// Simplified CPU usage based on process time
|
||||
const usage = process.cpuUsage()
|
||||
const total = usage.user + usage.system
|
||||
const seconds = (Date.now() - this.startTime) / 1000
|
||||
return Math.min(100, (total / 1000000 / seconds) * 100)
|
||||
}
|
||||
|
||||
/**
|
||||
* Clean old entries from sliding windows
|
||||
*/
|
||||
private cleanWindows(): void {
|
||||
const now = Date.now()
|
||||
const cutoff = now - this.windowSize
|
||||
|
||||
this.metricsWindow = this.metricsWindow.filter(t => t > cutoff)
|
||||
|
||||
// Keep only recent latency measurements
|
||||
if (this.latencyWindow.length > 100) {
|
||||
this.latencyWindow = this.latencyWindow.slice(-100)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Record a request
|
||||
* @param latency - Request latency in milliseconds
|
||||
* @param error - Whether the request resulted in an error
|
||||
*/
|
||||
recordRequest(latency: number, error: boolean = false): void {
|
||||
this.requestCount++
|
||||
this.metricsWindow.push(Date.now())
|
||||
this.latencyWindow.push(latency)
|
||||
|
||||
if (error) {
|
||||
this.errorCount++
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Record cache access
|
||||
* @param hit - Whether it was a cache hit
|
||||
*/
|
||||
recordCacheAccess(hit: boolean): void {
|
||||
if (hit) {
|
||||
this.cacheHits++
|
||||
} else {
|
||||
this.cacheMisses++
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Update vector count
|
||||
* @param count - New vector count
|
||||
*/
|
||||
updateVectorCount(count: number): void {
|
||||
this.vectorCount = count
|
||||
}
|
||||
|
||||
/**
|
||||
* Get current health status
|
||||
* @returns Health status object
|
||||
*/
|
||||
getHealthStatus(): HealthStatus {
|
||||
const metrics = this.collectMetrics()
|
||||
const uptime = Date.now() - this.startTime
|
||||
const warnings: string[] = []
|
||||
const errors: string[] = []
|
||||
|
||||
// Check for warnings
|
||||
if (metrics.memoryUsage > 1024 * 1024 * 1024) { // > 1GB
|
||||
warnings.push('High memory usage detected')
|
||||
}
|
||||
|
||||
if (metrics.cacheHitRate < 0.5) {
|
||||
warnings.push('Low cache hit rate')
|
||||
}
|
||||
|
||||
if (metrics.errorRate && metrics.errorRate > 0.05) {
|
||||
warnings.push('High error rate detected')
|
||||
}
|
||||
|
||||
if (metrics.averageLatency && metrics.averageLatency > 1000) {
|
||||
warnings.push('High latency detected')
|
||||
}
|
||||
|
||||
// Check for errors
|
||||
if (metrics.memoryUsage > 2 * 1024 * 1024 * 1024) { // > 2GB
|
||||
errors.push('Critical memory usage')
|
||||
}
|
||||
|
||||
if (metrics.errorRate && metrics.errorRate > 0.2) {
|
||||
errors.push('Critical error rate')
|
||||
}
|
||||
|
||||
// Determine overall status
|
||||
let status: 'healthy' | 'degraded' | 'unhealthy' = 'healthy'
|
||||
if (errors.length > 0) {
|
||||
status = 'unhealthy'
|
||||
} else if (warnings.length > 0) {
|
||||
status = 'degraded'
|
||||
}
|
||||
|
||||
return {
|
||||
status,
|
||||
instanceId: this.configManager.getInstanceId(),
|
||||
role: this.configManager.getRole(),
|
||||
uptime,
|
||||
lastCheck: new Date().toISOString(),
|
||||
metrics,
|
||||
warnings: warnings.length > 0 ? warnings : undefined,
|
||||
errors: errors.length > 0 ? errors : undefined
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get health check endpoint data
|
||||
* @returns JSON-serializable health data
|
||||
*/
|
||||
getHealthEndpointData(): Record<string, any> {
|
||||
const status = this.getHealthStatus()
|
||||
|
||||
return {
|
||||
status: status.status,
|
||||
instanceId: status.instanceId,
|
||||
role: status.role,
|
||||
uptime: Math.floor(status.uptime / 1000), // Convert to seconds
|
||||
lastCheck: status.lastCheck,
|
||||
metrics: {
|
||||
vectorCount: status.metrics.vectorCount,
|
||||
cacheHitRate: Math.round(status.metrics.cacheHitRate * 100) / 100,
|
||||
memoryUsageMB: Math.round(status.metrics.memoryUsage / 1024 / 1024),
|
||||
cpuUsagePercent: Math.round(status.metrics.cpuUsage || 0),
|
||||
requestsPerSecond: Math.round(status.metrics.requestsPerSecond || 0),
|
||||
averageLatencyMs: Math.round(status.metrics.averageLatency || 0),
|
||||
errorRate: Math.round((status.metrics.errorRate || 0) * 100) / 100
|
||||
},
|
||||
warnings: status.warnings,
|
||||
errors: status.errors
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Reset metrics (useful for testing)
|
||||
*/
|
||||
resetMetrics(): void {
|
||||
this.requestCount = 0
|
||||
this.errorCount = 0
|
||||
this.totalLatency = 0
|
||||
this.cacheHits = 0
|
||||
this.cacheMisses = 0
|
||||
this.metricsWindow = []
|
||||
this.latencyWindow = []
|
||||
}
|
||||
}
|
||||
24
src/distributed/index.ts
Normal file
24
src/distributed/index.ts
Normal file
|
|
@ -0,0 +1,24 @@
|
|||
/**
|
||||
* Distributed module exports
|
||||
*/
|
||||
|
||||
export { DistributedConfigManager } from './configManager.js'
|
||||
export { HashPartitioner, AffinityPartitioner } from './hashPartitioner.js'
|
||||
export {
|
||||
BaseOperationalMode,
|
||||
ReaderMode,
|
||||
WriterMode,
|
||||
HybridMode,
|
||||
OperationalModeFactory
|
||||
} from './operationalModes.js'
|
||||
export { DomainDetector } from './domainDetector.js'
|
||||
export { HealthMonitor } from './healthMonitor.js'
|
||||
|
||||
export type {
|
||||
HealthMetrics,
|
||||
HealthStatus
|
||||
} from './healthMonitor.js'
|
||||
|
||||
export type {
|
||||
DomainPattern
|
||||
} from './domainDetector.js'
|
||||
220
src/distributed/operationalModes.ts
Normal file
220
src/distributed/operationalModes.ts
Normal file
|
|
@ -0,0 +1,220 @@
|
|||
/**
|
||||
* Operational Modes for Distributed Brainy
|
||||
* Defines different modes with optimized caching strategies
|
||||
*/
|
||||
|
||||
import {
|
||||
OperationalMode,
|
||||
CacheStrategy,
|
||||
InstanceRole
|
||||
} from '../types/distributedTypes.js'
|
||||
|
||||
/**
|
||||
* Base operational mode
|
||||
*/
|
||||
export abstract class BaseOperationalMode implements OperationalMode {
|
||||
abstract canRead: boolean
|
||||
abstract canWrite: boolean
|
||||
abstract canDelete: boolean
|
||||
abstract cacheStrategy: CacheStrategy
|
||||
|
||||
/**
|
||||
* Validate operation is allowed in this mode
|
||||
*/
|
||||
validateOperation(operation: 'read' | 'write' | 'delete'): void {
|
||||
switch (operation) {
|
||||
case 'read':
|
||||
if (!this.canRead) {
|
||||
throw new Error('Read operations are not allowed in write-only mode')
|
||||
}
|
||||
break
|
||||
case 'write':
|
||||
if (!this.canWrite) {
|
||||
throw new Error('Write operations are not allowed in read-only mode')
|
||||
}
|
||||
break
|
||||
case 'delete':
|
||||
if (!this.canDelete) {
|
||||
throw new Error('Delete operations are not allowed in this mode')
|
||||
}
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Read-only mode optimized for query performance
|
||||
*/
|
||||
export class ReaderMode extends BaseOperationalMode {
|
||||
canRead = true
|
||||
canWrite = false
|
||||
canDelete = false
|
||||
|
||||
cacheStrategy: CacheStrategy = {
|
||||
hotCacheRatio: 0.8, // 80% of memory for read cache
|
||||
prefetchAggressive: true, // Aggressively prefetch related vectors
|
||||
ttl: 3600000, // 1 hour cache TTL
|
||||
compressionEnabled: true, // Trade CPU for more cache capacity
|
||||
writeBufferSize: 0, // No write buffer needed
|
||||
batchWrites: false, // No writes
|
||||
adaptive: true // Adapt to query patterns
|
||||
}
|
||||
|
||||
/**
|
||||
* Get optimized cache configuration for readers
|
||||
*/
|
||||
getCacheConfig() {
|
||||
return {
|
||||
hotCacheMaxSize: 1000000, // Large hot cache
|
||||
hotCacheEvictionThreshold: 0.9, // Keep cache full
|
||||
warmCacheTTL: 3600000, // 1 hour warm cache
|
||||
batchSize: 100, // Large batch reads
|
||||
autoTune: true, // Auto-tune for read patterns
|
||||
autoTuneInterval: 60000, // Tune every minute
|
||||
readOnly: true // Enable read-only optimizations
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Write-only mode optimized for ingestion
|
||||
*/
|
||||
export class WriterMode extends BaseOperationalMode {
|
||||
canRead = false
|
||||
canWrite = true
|
||||
canDelete = true
|
||||
|
||||
cacheStrategy: CacheStrategy = {
|
||||
hotCacheRatio: 0.2, // Only 20% for cache, rest for write buffer
|
||||
prefetchAggressive: false, // No prefetching needed
|
||||
ttl: 60000, // Short TTL (1 minute)
|
||||
compressionEnabled: false, // Speed over memory efficiency
|
||||
writeBufferSize: 10000, // Large write buffer for batching
|
||||
batchWrites: true, // Enable write batching
|
||||
adaptive: false // Fixed strategy for consistent writes
|
||||
}
|
||||
|
||||
/**
|
||||
* Get optimized cache configuration for writers
|
||||
*/
|
||||
getCacheConfig() {
|
||||
return {
|
||||
hotCacheMaxSize: 100000, // Small hot cache
|
||||
hotCacheEvictionThreshold: 0.5, // Aggressive eviction
|
||||
warmCacheTTL: 60000, // 1 minute warm cache
|
||||
batchSize: 1000, // Large batch writes
|
||||
autoTune: false, // Fixed configuration
|
||||
writeOnly: true // Enable write-only optimizations
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Hybrid mode that can both read and write
|
||||
*/
|
||||
export class HybridMode extends BaseOperationalMode {
|
||||
canRead = true
|
||||
canWrite = true
|
||||
canDelete = true
|
||||
|
||||
cacheStrategy: CacheStrategy = {
|
||||
hotCacheRatio: 0.5, // Balanced cache/buffer allocation
|
||||
prefetchAggressive: false, // Moderate prefetching
|
||||
ttl: 600000, // 10 minute TTL
|
||||
compressionEnabled: true, // Compress when beneficial
|
||||
writeBufferSize: 5000, // Moderate write buffer
|
||||
batchWrites: true, // Batch writes when possible
|
||||
adaptive: true // Adapt to workload mix
|
||||
}
|
||||
|
||||
private readWriteRatio: number = 0.5 // Track read/write ratio
|
||||
|
||||
/**
|
||||
* Get balanced cache configuration
|
||||
*/
|
||||
getCacheConfig() {
|
||||
return {
|
||||
hotCacheMaxSize: 500000, // Medium cache size
|
||||
hotCacheEvictionThreshold: 0.7, // Balanced eviction
|
||||
warmCacheTTL: 600000, // 10 minute warm cache
|
||||
batchSize: 500, // Medium batch size
|
||||
autoTune: true, // Auto-tune based on workload
|
||||
autoTuneInterval: 300000 // Tune every 5 minutes
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Update cache strategy based on workload
|
||||
* @param readCount - Number of recent reads
|
||||
* @param writeCount - Number of recent writes
|
||||
*/
|
||||
updateWorkloadBalance(readCount: number, writeCount: number): void {
|
||||
const total = readCount + writeCount
|
||||
if (total === 0) return
|
||||
|
||||
this.readWriteRatio = readCount / total
|
||||
|
||||
// Adjust cache strategy based on workload
|
||||
if (this.readWriteRatio > 0.8) {
|
||||
// Read-heavy workload
|
||||
this.cacheStrategy.hotCacheRatio = 0.7
|
||||
this.cacheStrategy.prefetchAggressive = true
|
||||
this.cacheStrategy.writeBufferSize = 2000
|
||||
} else if (this.readWriteRatio < 0.2) {
|
||||
// Write-heavy workload
|
||||
this.cacheStrategy.hotCacheRatio = 0.3
|
||||
this.cacheStrategy.prefetchAggressive = false
|
||||
this.cacheStrategy.writeBufferSize = 8000
|
||||
} else {
|
||||
// Balanced workload
|
||||
this.cacheStrategy.hotCacheRatio = 0.5
|
||||
this.cacheStrategy.prefetchAggressive = false
|
||||
this.cacheStrategy.writeBufferSize = 5000
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Factory for creating operational modes
|
||||
*/
|
||||
export class OperationalModeFactory {
|
||||
/**
|
||||
* Create operational mode based on role
|
||||
* @param role - The instance role
|
||||
* @returns The appropriate operational mode
|
||||
*/
|
||||
static createMode(role: InstanceRole): BaseOperationalMode {
|
||||
switch (role) {
|
||||
case 'reader':
|
||||
return new ReaderMode()
|
||||
case 'writer':
|
||||
return new WriterMode()
|
||||
case 'hybrid':
|
||||
return new HybridMode()
|
||||
default:
|
||||
// Default to reader for safety
|
||||
return new ReaderMode()
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Create mode with custom cache strategy
|
||||
* @param role - The instance role
|
||||
* @param customStrategy - Custom cache strategy overrides
|
||||
* @returns The operational mode with custom strategy
|
||||
*/
|
||||
static createModeWithStrategy(
|
||||
role: InstanceRole,
|
||||
customStrategy: Partial<CacheStrategy>
|
||||
): BaseOperationalMode {
|
||||
const mode = this.createMode(role)
|
||||
|
||||
// Apply custom strategy overrides
|
||||
mode.cacheStrategy = {
|
||||
...mode.cacheStrategy,
|
||||
...customStrategy
|
||||
}
|
||||
|
||||
return mode
|
||||
}
|
||||
}
|
||||
236
src/types/distributedTypes.ts
Normal file
236
src/types/distributedTypes.ts
Normal file
|
|
@ -0,0 +1,236 @@
|
|||
/**
|
||||
* Distributed types for Brainy
|
||||
* Defines types for distributed operations across multiple instances
|
||||
*/
|
||||
|
||||
export type InstanceRole = 'reader' | 'writer' | 'hybrid'
|
||||
|
||||
export type PartitionStrategy = 'hash' | 'semantic' | 'manual'
|
||||
|
||||
export interface DistributedConfig {
|
||||
/**
|
||||
* Enable distributed mode
|
||||
* Can be boolean for auto-detection or specific configuration
|
||||
*/
|
||||
enabled?: boolean | 'auto'
|
||||
|
||||
/**
|
||||
* Role of this instance in the distributed system
|
||||
* - reader: Read-only access, optimized for queries
|
||||
* - writer: Write-focused, handles data ingestion
|
||||
* - hybrid: Can both read and write (requires coordination)
|
||||
*/
|
||||
role?: InstanceRole
|
||||
|
||||
/**
|
||||
* Unique identifier for this instance
|
||||
* Auto-generated if not provided
|
||||
*/
|
||||
instanceId?: string
|
||||
|
||||
/**
|
||||
* Path to shared configuration file in S3
|
||||
* Default: '_brainy/config.json'
|
||||
*/
|
||||
configPath?: string
|
||||
|
||||
/**
|
||||
* Heartbeat interval in milliseconds
|
||||
* Default: 30000 (30 seconds)
|
||||
*/
|
||||
heartbeatInterval?: number
|
||||
|
||||
/**
|
||||
* Config check interval in milliseconds
|
||||
* Default: 10000 (10 seconds)
|
||||
*/
|
||||
configCheckInterval?: number
|
||||
|
||||
/**
|
||||
* Instance timeout in milliseconds
|
||||
* Instances not seen for this duration are considered dead
|
||||
* Default: 60000 (60 seconds)
|
||||
*/
|
||||
instanceTimeout?: number
|
||||
}
|
||||
|
||||
export interface SharedConfig {
|
||||
/**
|
||||
* Configuration version for compatibility checking
|
||||
*/
|
||||
version: number
|
||||
|
||||
/**
|
||||
* Last update timestamp
|
||||
*/
|
||||
updated: string
|
||||
|
||||
/**
|
||||
* Global settings that must be consistent across all instances
|
||||
*/
|
||||
settings: {
|
||||
/**
|
||||
* Partitioning strategy
|
||||
* - hash: Deterministic hash-based partitioning (recommended for multi-writer)
|
||||
* - semantic: Group similar vectors (single writer only)
|
||||
* - manual: Explicit partition assignment
|
||||
*/
|
||||
partitionStrategy: PartitionStrategy
|
||||
|
||||
/**
|
||||
* Number of partitions (for hash strategy)
|
||||
*/
|
||||
partitionCount: number
|
||||
|
||||
/**
|
||||
* Embedding model name (must be consistent)
|
||||
*/
|
||||
embeddingModel: string
|
||||
|
||||
/**
|
||||
* Vector dimensions
|
||||
*/
|
||||
dimensions: number
|
||||
|
||||
/**
|
||||
* Distance metric
|
||||
*/
|
||||
distanceMetric: 'cosine' | 'euclidean' | 'manhattan'
|
||||
|
||||
/**
|
||||
* HNSW parameters (must be consistent for index compatibility)
|
||||
*/
|
||||
hnswParams?: {
|
||||
M: number
|
||||
efConstruction: number
|
||||
maxElements?: number
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Active instances in the distributed system
|
||||
*/
|
||||
instances: {
|
||||
[instanceId: string]: InstanceInfo
|
||||
}
|
||||
|
||||
/**
|
||||
* Partition assignments (for manual strategy)
|
||||
*/
|
||||
partitionAssignments?: {
|
||||
[instanceId: string]: string[]
|
||||
}
|
||||
}
|
||||
|
||||
export interface InstanceInfo {
|
||||
/**
|
||||
* Instance role
|
||||
*/
|
||||
role: InstanceRole
|
||||
|
||||
/**
|
||||
* Instance status
|
||||
*/
|
||||
status: 'active' | 'inactive' | 'unhealthy'
|
||||
|
||||
/**
|
||||
* Last heartbeat timestamp
|
||||
*/
|
||||
lastHeartbeat: string
|
||||
|
||||
/**
|
||||
* Optional endpoint for health checks
|
||||
*/
|
||||
endpoint?: string
|
||||
|
||||
/**
|
||||
* Instance metrics
|
||||
*/
|
||||
metrics?: {
|
||||
vectorCount?: number
|
||||
cacheHitRate?: number
|
||||
memoryUsage?: number
|
||||
cpuUsage?: number
|
||||
}
|
||||
|
||||
/**
|
||||
* Assigned partitions (for manual assignment)
|
||||
*/
|
||||
assignedPartitions?: string[]
|
||||
|
||||
/**
|
||||
* Preferred partitions (for affinity)
|
||||
*/
|
||||
preferredPartitions?: number[]
|
||||
}
|
||||
|
||||
export interface DomainMetadata {
|
||||
/**
|
||||
* Domain identifier for logical data separation
|
||||
*/
|
||||
domain?: string
|
||||
|
||||
/**
|
||||
* Additional domain-specific metadata
|
||||
*/
|
||||
domainMetadata?: Record<string, any>
|
||||
}
|
||||
|
||||
export interface CacheStrategy {
|
||||
/**
|
||||
* Percentage of memory allocated to hot cache (0-1)
|
||||
*/
|
||||
hotCacheRatio: number
|
||||
|
||||
/**
|
||||
* Enable aggressive prefetching
|
||||
*/
|
||||
prefetchAggressive?: boolean
|
||||
|
||||
/**
|
||||
* Cache time-to-live in milliseconds
|
||||
*/
|
||||
ttl?: number
|
||||
|
||||
/**
|
||||
* Enable compression to trade CPU for memory
|
||||
*/
|
||||
compressionEnabled?: boolean
|
||||
|
||||
/**
|
||||
* Write buffer size for batching
|
||||
*/
|
||||
writeBufferSize?: number
|
||||
|
||||
/**
|
||||
* Enable write batching
|
||||
*/
|
||||
batchWrites?: boolean
|
||||
|
||||
/**
|
||||
* Adaptive caching based on workload
|
||||
*/
|
||||
adaptive?: boolean
|
||||
}
|
||||
|
||||
export interface OperationalMode {
|
||||
/**
|
||||
* Whether this mode can read
|
||||
*/
|
||||
canRead: boolean
|
||||
|
||||
/**
|
||||
* Whether this mode can write
|
||||
*/
|
||||
canWrite: boolean
|
||||
|
||||
/**
|
||||
* Whether this mode can delete
|
||||
*/
|
||||
canDelete: boolean
|
||||
|
||||
/**
|
||||
* Cache strategy for this mode
|
||||
*/
|
||||
cacheStrategy: CacheStrategy
|
||||
}
|
||||
47
src/utils/crypto.ts
Normal file
47
src/utils/crypto.ts
Normal file
|
|
@ -0,0 +1,47 @@
|
|||
/**
|
||||
* Cross-platform crypto utilities
|
||||
* Provides hashing functions that work in both Node.js and browser environments
|
||||
*/
|
||||
|
||||
/**
|
||||
* Simple string hash function that works in all environments
|
||||
* Uses djb2 algorithm - fast and good distribution
|
||||
* @param str - String to hash
|
||||
* @returns Positive integer hash
|
||||
*/
|
||||
export function hashString(str: string): number {
|
||||
let hash = 5381
|
||||
for (let i = 0; i < str.length; i++) {
|
||||
const char = str.charCodeAt(i)
|
||||
hash = ((hash << 5) + hash) + char // hash * 33 + char
|
||||
}
|
||||
// Ensure positive number
|
||||
return Math.abs(hash)
|
||||
}
|
||||
|
||||
/**
|
||||
* Alternative: FNV-1a hash algorithm
|
||||
* Good distribution and fast
|
||||
* @param str - String to hash
|
||||
* @returns Positive integer hash
|
||||
*/
|
||||
export function fnv1aHash(str: string): number {
|
||||
let hash = 2166136261
|
||||
for (let i = 0; i < str.length; i++) {
|
||||
hash ^= str.charCodeAt(i)
|
||||
hash = (hash * 16777619) >>> 0
|
||||
}
|
||||
return hash
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate a deterministic hash for partitioning
|
||||
* Uses the most appropriate algorithm for the environment
|
||||
* @param input - Input string to hash
|
||||
* @returns Positive integer hash suitable for modulo operations
|
||||
*/
|
||||
export function getPartitionHash(input: string): number {
|
||||
// Use djb2 by default as it's fast and has good distribution
|
||||
// This ensures consistent partitioning across all environments
|
||||
return hashString(input)
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue