Initial commit: Brainy - Multi-Dimensional AI Database
Open source vector database with HNSW indexing, graph relationships, and metadata facets. Features CLI with professional augmentation registry integration for discovering extensions and capabilities.
This commit is contained in:
commit
f8c45f2d8d
448 changed files with 103294 additions and 0 deletions
106
dist/distributed/configManager.d.ts
vendored
Normal file
106
dist/distributed/configManager.d.ts
vendored
Normal file
|
|
@ -0,0 +1,106 @@
|
|||
/**
|
||||
* Distributed Configuration Manager
|
||||
* Manages shared configuration in S3 for distributed Brainy instances
|
||||
*/
|
||||
import { DistributedConfig, SharedConfig, InstanceInfo, InstanceRole } from '../types/distributedTypes.js';
|
||||
import { StorageAdapter } from '../coreTypes.js';
|
||||
export declare class DistributedConfigManager {
|
||||
private config;
|
||||
private instanceId;
|
||||
private role;
|
||||
private configPath;
|
||||
private heartbeatInterval;
|
||||
private configCheckInterval;
|
||||
private instanceTimeout;
|
||||
private storage;
|
||||
private heartbeatTimer?;
|
||||
private configWatchTimer?;
|
||||
private lastConfigVersion;
|
||||
private onConfigUpdate?;
|
||||
private hasMigrated;
|
||||
constructor(storage: StorageAdapter, distributedConfig?: DistributedConfig, brainyMode?: {
|
||||
readOnly?: boolean;
|
||||
writeOnly?: boolean;
|
||||
});
|
||||
/**
|
||||
* Initialize the distributed configuration
|
||||
*/
|
||||
initialize(): Promise<SharedConfig>;
|
||||
/**
|
||||
* Load existing config or create new one
|
||||
*/
|
||||
private loadOrCreateConfig;
|
||||
/**
|
||||
* Determine role based on configuration
|
||||
* IMPORTANT: Role must be explicitly set - no automatic assignment based on order
|
||||
*/
|
||||
private determineRole;
|
||||
/**
|
||||
* Check if an instance is still alive
|
||||
*/
|
||||
private isInstanceAlive;
|
||||
/**
|
||||
* Register this instance in the shared config
|
||||
*/
|
||||
private registerInstance;
|
||||
/**
|
||||
* Migrate config from legacy location to new location
|
||||
*/
|
||||
private migrateConfigFromLegacyLocation;
|
||||
/**
|
||||
* Migrate config to new location in index folder
|
||||
*/
|
||||
private migrateConfig;
|
||||
/**
|
||||
* Save configuration with version increment
|
||||
*/
|
||||
private saveConfig;
|
||||
/**
|
||||
* Start heartbeat to keep instance alive in config
|
||||
*/
|
||||
private startHeartbeat;
|
||||
/**
|
||||
* Update heartbeat and clean stale instances
|
||||
*/
|
||||
private updateHeartbeat;
|
||||
/**
|
||||
* Start watching for config changes
|
||||
*/
|
||||
private startConfigWatch;
|
||||
/**
|
||||
* Check for configuration updates
|
||||
*/
|
||||
private checkForConfigUpdates;
|
||||
/**
|
||||
* Load configuration from storage
|
||||
*/
|
||||
private loadConfig;
|
||||
/**
|
||||
* Get current configuration
|
||||
*/
|
||||
getConfig(): SharedConfig | null;
|
||||
/**
|
||||
* Get instance role
|
||||
*/
|
||||
getRole(): InstanceRole;
|
||||
/**
|
||||
* Get instance ID
|
||||
*/
|
||||
getInstanceId(): string;
|
||||
/**
|
||||
* Set config update callback
|
||||
*/
|
||||
setOnConfigUpdate(callback: (config: SharedConfig) => void): void;
|
||||
/**
|
||||
* Get all active instances of a specific role
|
||||
*/
|
||||
getInstancesByRole(role: InstanceRole): InstanceInfo[];
|
||||
/**
|
||||
* Update instance metrics
|
||||
*/
|
||||
updateMetrics(metrics: Partial<InstanceInfo['metrics']>): Promise<void>;
|
||||
/**
|
||||
* Cleanup resources
|
||||
*/
|
||||
cleanup(): Promise<void>;
|
||||
}
|
||||
441
dist/distributed/configManager.js
vendored
Normal file
441
dist/distributed/configManager.js
vendored
Normal file
|
|
@ -0,0 +1,441 @@
|
|||
/**
|
||||
* Distributed Configuration Manager
|
||||
* Manages shared configuration in S3 for distributed Brainy instances
|
||||
*/
|
||||
import { v4 as uuidv4 } from '../universal/uuid.js';
|
||||
// Constants for config storage locations
|
||||
const DISTRIBUTED_CONFIG_KEY = 'distributed_config';
|
||||
const LEGACY_CONFIG_KEY = '_distributed_config';
|
||||
export class DistributedConfigManager {
|
||||
constructor(storage, distributedConfig, brainyMode) {
|
||||
this.config = null;
|
||||
this.lastConfigVersion = 0;
|
||||
this.hasMigrated = false;
|
||||
this.storage = storage;
|
||||
this.instanceId = distributedConfig?.instanceId || `instance-${uuidv4()}`;
|
||||
// Updated default path to use _system instead of _brainy
|
||||
this.configPath = distributedConfig?.configPath || '_system/distributed_config.json';
|
||||
this.heartbeatInterval = distributedConfig?.heartbeatInterval || 30000;
|
||||
this.configCheckInterval = distributedConfig?.configCheckInterval || 10000;
|
||||
this.instanceTimeout = distributedConfig?.instanceTimeout || 60000;
|
||||
// Set role from distributed config if provided
|
||||
if (distributedConfig?.role) {
|
||||
this.role = distributedConfig.role;
|
||||
}
|
||||
// Infer role from Brainy's read/write mode if not explicitly set
|
||||
else if (brainyMode) {
|
||||
if (brainyMode.writeOnly) {
|
||||
this.role = 'writer';
|
||||
}
|
||||
else if (brainyMode.readOnly) {
|
||||
this.role = 'reader';
|
||||
}
|
||||
// If neither readOnly nor writeOnly, role must be explicitly set
|
||||
}
|
||||
}
|
||||
/**
|
||||
* Initialize the distributed configuration
|
||||
*/
|
||||
async initialize() {
|
||||
// Load or create configuration
|
||||
this.config = await this.loadOrCreateConfig();
|
||||
// Determine role if not explicitly set
|
||||
if (!this.role) {
|
||||
this.role = await this.determineRole();
|
||||
}
|
||||
// Register this instance
|
||||
await this.registerInstance();
|
||||
// Start heartbeat and config watching
|
||||
this.startHeartbeat();
|
||||
this.startConfigWatch();
|
||||
return this.config;
|
||||
}
|
||||
/**
|
||||
* Load existing config or create new one
|
||||
*/
|
||||
async loadOrCreateConfig() {
|
||||
// First, try to load from the new location in index folder
|
||||
try {
|
||||
const configData = await this.storage.getStatistics();
|
||||
if (configData && configData.distributedConfig) {
|
||||
this.lastConfigVersion = configData.distributedConfig.version;
|
||||
return configData.distributedConfig;
|
||||
}
|
||||
}
|
||||
catch (error) {
|
||||
// Config doesn't exist in new location yet
|
||||
}
|
||||
// Check if we need to migrate from old location
|
||||
if (!this.hasMigrated) {
|
||||
const migrated = await this.migrateConfigFromLegacyLocation();
|
||||
if (migrated) {
|
||||
return migrated;
|
||||
}
|
||||
}
|
||||
// Legacy fallback - try old location
|
||||
try {
|
||||
const configData = await this.storage.getMetadata(LEGACY_CONFIG_KEY);
|
||||
if (configData) {
|
||||
// Migrate to new location
|
||||
await this.migrateConfig(configData);
|
||||
this.lastConfigVersion = configData.version;
|
||||
return configData;
|
||||
}
|
||||
}
|
||||
catch (error) {
|
||||
// Config doesn't exist yet
|
||||
}
|
||||
// Create default config
|
||||
const newConfig = {
|
||||
version: 1,
|
||||
updated: new Date().toISOString(),
|
||||
settings: {
|
||||
partitionStrategy: 'hash',
|
||||
partitionCount: 100,
|
||||
embeddingModel: 'text-embedding-ada-002',
|
||||
dimensions: 1536,
|
||||
distanceMetric: 'cosine',
|
||||
hnswParams: {
|
||||
M: 16,
|
||||
efConstruction: 200
|
||||
}
|
||||
},
|
||||
instances: {}
|
||||
};
|
||||
await this.saveConfig(newConfig);
|
||||
return newConfig;
|
||||
}
|
||||
/**
|
||||
* Determine role based on configuration
|
||||
* IMPORTANT: Role must be explicitly set - no automatic assignment based on order
|
||||
*/
|
||||
async determineRole() {
|
||||
// Check environment variable first
|
||||
if (process.env.BRAINY_ROLE) {
|
||||
const role = process.env.BRAINY_ROLE.toLowerCase();
|
||||
if (role === 'writer' || role === 'reader' || role === 'hybrid') {
|
||||
return role;
|
||||
}
|
||||
throw new Error(`Invalid BRAINY_ROLE: ${process.env.BRAINY_ROLE}. Must be 'writer', 'reader', or 'hybrid'`);
|
||||
}
|
||||
// Check if explicitly passed in distributed config
|
||||
if (this.role) {
|
||||
return this.role;
|
||||
}
|
||||
// DO NOT auto-assign roles based on deployment order or existing instances
|
||||
// This is dangerous and can lead to data corruption or loss
|
||||
throw new Error('Distributed mode requires explicit role configuration. ' +
|
||||
'Set BRAINY_ROLE environment variable or pass role in distributed config. ' +
|
||||
'Valid roles: "writer", "reader", "hybrid"');
|
||||
}
|
||||
/**
|
||||
* Check if an instance is still alive
|
||||
*/
|
||||
isInstanceAlive(instance) {
|
||||
const lastSeen = new Date(instance.lastHeartbeat).getTime();
|
||||
const now = Date.now();
|
||||
return (now - lastSeen) < this.instanceTimeout;
|
||||
}
|
||||
/**
|
||||
* Register this instance in the shared config
|
||||
*/
|
||||
async registerInstance() {
|
||||
if (!this.config)
|
||||
return;
|
||||
// Role must be set by this point
|
||||
if (!this.role) {
|
||||
throw new Error('Cannot register instance without a role');
|
||||
}
|
||||
const instanceInfo = {
|
||||
role: this.role,
|
||||
status: 'active',
|
||||
lastHeartbeat: new Date().toISOString(),
|
||||
metrics: {
|
||||
memoryUsage: process.memoryUsage().heapUsed
|
||||
}
|
||||
};
|
||||
// Add endpoint if available
|
||||
if (process.env.SERVICE_ENDPOINT) {
|
||||
instanceInfo.endpoint = process.env.SERVICE_ENDPOINT;
|
||||
}
|
||||
this.config.instances[this.instanceId] = instanceInfo;
|
||||
await this.saveConfig(this.config);
|
||||
}
|
||||
/**
|
||||
* Migrate config from legacy location to new location
|
||||
*/
|
||||
async migrateConfigFromLegacyLocation() {
|
||||
try {
|
||||
// Try to load from old location
|
||||
const legacyConfig = await this.storage.getMetadata(LEGACY_CONFIG_KEY);
|
||||
if (legacyConfig) {
|
||||
console.log('Migrating distributed config from legacy location to index folder...');
|
||||
// Save to new location
|
||||
await this.migrateConfig(legacyConfig);
|
||||
// Delete from old location (optional - we can keep it for rollback)
|
||||
// await this.storage.deleteMetadata(LEGACY_CONFIG_KEY)
|
||||
this.hasMigrated = true;
|
||||
this.lastConfigVersion = legacyConfig.version;
|
||||
return legacyConfig;
|
||||
}
|
||||
}
|
||||
catch (error) {
|
||||
console.error('Error during config migration:', error);
|
||||
}
|
||||
this.hasMigrated = true;
|
||||
return null;
|
||||
}
|
||||
/**
|
||||
* Migrate config to new location in index folder
|
||||
*/
|
||||
async migrateConfig(config) {
|
||||
// Get existing statistics or create new
|
||||
let stats = await this.storage.getStatistics();
|
||||
if (!stats) {
|
||||
stats = {
|
||||
nounCount: {},
|
||||
verbCount: {},
|
||||
metadataCount: {},
|
||||
hnswIndexSize: 0,
|
||||
lastUpdated: new Date().toISOString()
|
||||
};
|
||||
}
|
||||
// Add distributed config to statistics
|
||||
stats.distributedConfig = config;
|
||||
// Save updated statistics
|
||||
await this.storage.saveStatistics(stats);
|
||||
}
|
||||
/**
|
||||
* Save configuration with version increment
|
||||
*/
|
||||
async saveConfig(config) {
|
||||
config.version++;
|
||||
config.updated = new Date().toISOString();
|
||||
this.lastConfigVersion = config.version;
|
||||
// Save to new location in index folder along with statistics
|
||||
let stats = await this.storage.getStatistics();
|
||||
if (!stats) {
|
||||
stats = {
|
||||
nounCount: {},
|
||||
verbCount: {},
|
||||
metadataCount: {},
|
||||
hnswIndexSize: 0,
|
||||
lastUpdated: new Date().toISOString()
|
||||
};
|
||||
}
|
||||
// Update distributed config in statistics
|
||||
stats.distributedConfig = config;
|
||||
// Save updated statistics
|
||||
await this.storage.saveStatistics(stats);
|
||||
this.config = config;
|
||||
}
|
||||
/**
|
||||
* Start heartbeat to keep instance alive in config
|
||||
*/
|
||||
startHeartbeat() {
|
||||
this.heartbeatTimer = setInterval(async () => {
|
||||
await this.updateHeartbeat();
|
||||
}, this.heartbeatInterval);
|
||||
}
|
||||
/**
|
||||
* Update heartbeat and clean stale instances
|
||||
*/
|
||||
async updateHeartbeat() {
|
||||
if (!this.config)
|
||||
return;
|
||||
// Reload config to get latest state
|
||||
try {
|
||||
const latestConfig = await this.loadConfig();
|
||||
if (latestConfig) {
|
||||
this.config = latestConfig;
|
||||
}
|
||||
}
|
||||
catch (error) {
|
||||
console.error('Failed to reload config:', error);
|
||||
}
|
||||
// Update our heartbeat
|
||||
if (this.config.instances[this.instanceId]) {
|
||||
this.config.instances[this.instanceId].lastHeartbeat = new Date().toISOString();
|
||||
this.config.instances[this.instanceId].status = 'active';
|
||||
// Update metrics if available
|
||||
this.config.instances[this.instanceId].metrics = {
|
||||
memoryUsage: process.memoryUsage().heapUsed
|
||||
};
|
||||
}
|
||||
else {
|
||||
// Re-register if we were removed
|
||||
await this.registerInstance();
|
||||
return;
|
||||
}
|
||||
// Clean up stale instances
|
||||
const now = Date.now();
|
||||
let hasChanges = false;
|
||||
for (const [id, instance] of Object.entries(this.config.instances)) {
|
||||
if (id === this.instanceId)
|
||||
continue;
|
||||
const lastSeen = new Date(instance.lastHeartbeat).getTime();
|
||||
if (now - lastSeen > this.instanceTimeout) {
|
||||
delete this.config.instances[id];
|
||||
hasChanges = true;
|
||||
}
|
||||
}
|
||||
// Save if there were changes
|
||||
if (hasChanges) {
|
||||
await this.saveConfig(this.config);
|
||||
}
|
||||
else {
|
||||
// Just update our heartbeat without version increment
|
||||
// Get existing statistics
|
||||
let stats = await this.storage.getStatistics();
|
||||
if (!stats) {
|
||||
stats = {
|
||||
nounCount: {},
|
||||
verbCount: {},
|
||||
metadataCount: {},
|
||||
hnswIndexSize: 0,
|
||||
lastUpdated: new Date().toISOString()
|
||||
};
|
||||
}
|
||||
// Update distributed config in statistics without version increment
|
||||
stats.distributedConfig = this.config;
|
||||
// Save updated statistics
|
||||
await this.storage.saveStatistics(stats);
|
||||
}
|
||||
}
|
||||
/**
|
||||
* Start watching for config changes
|
||||
*/
|
||||
startConfigWatch() {
|
||||
this.configWatchTimer = setInterval(async () => {
|
||||
await this.checkForConfigUpdates();
|
||||
}, this.configCheckInterval);
|
||||
}
|
||||
/**
|
||||
* Check for configuration updates
|
||||
*/
|
||||
async checkForConfigUpdates() {
|
||||
try {
|
||||
const latestConfig = await this.loadConfig();
|
||||
if (!latestConfig)
|
||||
return;
|
||||
if (latestConfig.version > this.lastConfigVersion) {
|
||||
this.config = latestConfig;
|
||||
this.lastConfigVersion = latestConfig.version;
|
||||
// Notify listeners of config update
|
||||
if (this.onConfigUpdate) {
|
||||
this.onConfigUpdate(latestConfig);
|
||||
}
|
||||
}
|
||||
}
|
||||
catch (error) {
|
||||
console.error('Failed to check config updates:', error);
|
||||
}
|
||||
}
|
||||
/**
|
||||
* Load configuration from storage
|
||||
*/
|
||||
async loadConfig() {
|
||||
try {
|
||||
// Try new location first
|
||||
const stats = await this.storage.getStatistics();
|
||||
if (stats && stats.distributedConfig) {
|
||||
return stats.distributedConfig;
|
||||
}
|
||||
// Fallback to legacy location if not migrated yet
|
||||
if (!this.hasMigrated) {
|
||||
const configData = await this.storage.getMetadata(LEGACY_CONFIG_KEY);
|
||||
if (configData) {
|
||||
// Trigger migration on next save
|
||||
return configData;
|
||||
}
|
||||
}
|
||||
}
|
||||
catch (error) {
|
||||
console.error('Failed to load config:', error);
|
||||
}
|
||||
return null;
|
||||
}
|
||||
/**
|
||||
* Get current configuration
|
||||
*/
|
||||
getConfig() {
|
||||
return this.config;
|
||||
}
|
||||
/**
|
||||
* Get instance role
|
||||
*/
|
||||
getRole() {
|
||||
if (!this.role) {
|
||||
throw new Error('Role not initialized');
|
||||
}
|
||||
return this.role;
|
||||
}
|
||||
/**
|
||||
* Get instance ID
|
||||
*/
|
||||
getInstanceId() {
|
||||
return this.instanceId;
|
||||
}
|
||||
/**
|
||||
* Set config update callback
|
||||
*/
|
||||
setOnConfigUpdate(callback) {
|
||||
this.onConfigUpdate = callback;
|
||||
}
|
||||
/**
|
||||
* Get all active instances of a specific role
|
||||
*/
|
||||
getInstancesByRole(role) {
|
||||
if (!this.config)
|
||||
return [];
|
||||
return Object.entries(this.config.instances)
|
||||
.filter(([_, instance]) => instance.role === role &&
|
||||
this.isInstanceAlive(instance))
|
||||
.map(([_, instance]) => instance);
|
||||
}
|
||||
/**
|
||||
* Update instance metrics
|
||||
*/
|
||||
async updateMetrics(metrics) {
|
||||
if (!this.config || !this.config.instances[this.instanceId])
|
||||
return;
|
||||
this.config.instances[this.instanceId].metrics = {
|
||||
...this.config.instances[this.instanceId].metrics,
|
||||
...metrics
|
||||
};
|
||||
// Don't increment version for metric updates
|
||||
// Get existing statistics
|
||||
let stats = await this.storage.getStatistics();
|
||||
if (!stats) {
|
||||
stats = {
|
||||
nounCount: {},
|
||||
verbCount: {},
|
||||
metadataCount: {},
|
||||
hnswIndexSize: 0,
|
||||
lastUpdated: new Date().toISOString()
|
||||
};
|
||||
}
|
||||
// Update distributed config in statistics without version increment
|
||||
stats.distributedConfig = this.config;
|
||||
// Save updated statistics
|
||||
await this.storage.saveStatistics(stats);
|
||||
}
|
||||
/**
|
||||
* Cleanup resources
|
||||
*/
|
||||
async cleanup() {
|
||||
// Stop timers
|
||||
if (this.heartbeatTimer) {
|
||||
clearInterval(this.heartbeatTimer);
|
||||
}
|
||||
if (this.configWatchTimer) {
|
||||
clearInterval(this.configWatchTimer);
|
||||
}
|
||||
// Mark instance as inactive
|
||||
if (this.config && this.config.instances[this.instanceId]) {
|
||||
this.config.instances[this.instanceId].status = 'inactive';
|
||||
await this.saveConfig(this.config);
|
||||
}
|
||||
}
|
||||
}
|
||||
//# sourceMappingURL=configManager.js.map
|
||||
1
dist/distributed/configManager.js.map
vendored
Normal file
1
dist/distributed/configManager.js.map
vendored
Normal file
File diff suppressed because one or more lines are too long
77
dist/distributed/domainDetector.d.ts
vendored
Normal file
77
dist/distributed/domainDetector.d.ts
vendored
Normal file
|
|
@ -0,0 +1,77 @@
|
|||
/**
|
||||
* Domain Detector
|
||||
* Automatically detects and manages data domains for logical separation
|
||||
*/
|
||||
import { DomainMetadata } from '../types/distributedTypes.js';
|
||||
export interface DomainPattern {
|
||||
domain: string;
|
||||
patterns: {
|
||||
fields?: string[];
|
||||
keywords?: string[];
|
||||
regex?: RegExp;
|
||||
};
|
||||
priority?: number;
|
||||
}
|
||||
export declare class DomainDetector {
|
||||
private domainPatterns;
|
||||
private customPatterns;
|
||||
private domainStats;
|
||||
/**
|
||||
* Detect domain from data object
|
||||
* @param data - The data object to analyze
|
||||
* @returns The detected domain and metadata
|
||||
*/
|
||||
detectDomain(data: any): DomainMetadata;
|
||||
/**
|
||||
* Score a data object against a domain pattern
|
||||
*/
|
||||
private scorePattern;
|
||||
/**
|
||||
* Extract domain-specific metadata
|
||||
*/
|
||||
private extractDomainMetadata;
|
||||
/**
|
||||
* Calculate detection confidence
|
||||
*/
|
||||
private calculateConfidence;
|
||||
/**
|
||||
* Categorize price ranges
|
||||
*/
|
||||
private getPriceRange;
|
||||
/**
|
||||
* Categorize customer value
|
||||
*/
|
||||
private getValueCategory;
|
||||
/**
|
||||
* Categorize amount ranges
|
||||
*/
|
||||
private getAmountRange;
|
||||
/**
|
||||
* Add custom domain pattern
|
||||
* @param pattern - Custom domain pattern to add
|
||||
*/
|
||||
addCustomPattern(pattern: DomainPattern): void;
|
||||
/**
|
||||
* Remove custom domain pattern
|
||||
* @param domain - Domain to remove pattern for
|
||||
*/
|
||||
removeCustomPattern(domain: string): void;
|
||||
/**
|
||||
* Update domain statistics
|
||||
*/
|
||||
private updateStats;
|
||||
/**
|
||||
* Get domain statistics
|
||||
* @returns Map of domain to count
|
||||
*/
|
||||
getDomainStats(): Map<string, number>;
|
||||
/**
|
||||
* Clear domain statistics
|
||||
*/
|
||||
clearStats(): void;
|
||||
/**
|
||||
* Get all configured domains
|
||||
* @returns Array of domain names
|
||||
*/
|
||||
getConfiguredDomains(): string[];
|
||||
}
|
||||
307
dist/distributed/domainDetector.js
vendored
Normal file
307
dist/distributed/domainDetector.js
vendored
Normal file
|
|
@ -0,0 +1,307 @@
|
|||
/**
|
||||
* Domain Detector
|
||||
* Automatically detects and manages data domains for logical separation
|
||||
*/
|
||||
export class DomainDetector {
|
||||
constructor() {
|
||||
this.domainPatterns = [
|
||||
{
|
||||
domain: 'medical',
|
||||
patterns: {
|
||||
fields: ['symptoms', 'diagnosis', 'treatment', 'medication', 'patient'],
|
||||
keywords: ['medical', 'health', 'disease', 'symptom', 'treatment', 'doctor', 'patient']
|
||||
},
|
||||
priority: 1
|
||||
},
|
||||
{
|
||||
domain: 'legal',
|
||||
patterns: {
|
||||
fields: ['contract', 'clause', 'litigation', 'statute', 'jurisdiction'],
|
||||
keywords: ['legal', 'law', 'contract', 'court', 'attorney', 'litigation', 'statute']
|
||||
},
|
||||
priority: 1
|
||||
},
|
||||
{
|
||||
domain: 'product',
|
||||
patterns: {
|
||||
fields: ['price', 'sku', 'inventory', 'category', 'brand'],
|
||||
keywords: ['product', 'price', 'sale', 'inventory', 'catalog', 'item', 'sku']
|
||||
},
|
||||
priority: 1
|
||||
},
|
||||
{
|
||||
domain: 'customer',
|
||||
patterns: {
|
||||
fields: ['customerId', 'email', 'phone', 'address', 'orders'],
|
||||
keywords: ['customer', 'client', 'user', 'account', 'profile', 'contact']
|
||||
},
|
||||
priority: 1
|
||||
},
|
||||
{
|
||||
domain: 'financial',
|
||||
patterns: {
|
||||
fields: ['amount', 'currency', 'transaction', 'balance', 'account'],
|
||||
keywords: ['financial', 'money', 'payment', 'transaction', 'bank', 'credit', 'debit']
|
||||
},
|
||||
priority: 1
|
||||
},
|
||||
{
|
||||
domain: 'technical',
|
||||
patterns: {
|
||||
fields: ['code', 'function', 'error', 'stack', 'api'],
|
||||
keywords: ['code', 'software', 'api', 'error', 'debug', 'function', 'class', 'method']
|
||||
},
|
||||
priority: 2
|
||||
}
|
||||
];
|
||||
this.customPatterns = [];
|
||||
this.domainStats = new Map();
|
||||
}
|
||||
/**
|
||||
* Detect domain from data object
|
||||
* @param data - The data object to analyze
|
||||
* @returns The detected domain and metadata
|
||||
*/
|
||||
detectDomain(data) {
|
||||
if (!data || typeof data !== 'object') {
|
||||
return { domain: 'general' };
|
||||
}
|
||||
// Check for explicit domain field
|
||||
if (data.domain && typeof data.domain === 'string') {
|
||||
this.updateStats(data.domain);
|
||||
return {
|
||||
domain: data.domain,
|
||||
domainMetadata: this.extractDomainMetadata(data, data.domain)
|
||||
};
|
||||
}
|
||||
// Score each domain pattern
|
||||
const scores = new Map();
|
||||
// Check custom patterns first (higher priority)
|
||||
for (const pattern of this.customPatterns) {
|
||||
const score = this.scorePattern(data, pattern);
|
||||
if (score > 0) {
|
||||
scores.set(pattern.domain, score * (pattern.priority || 1));
|
||||
}
|
||||
}
|
||||
// Check default patterns
|
||||
for (const pattern of this.domainPatterns) {
|
||||
const score = this.scorePattern(data, pattern);
|
||||
if (score > 0) {
|
||||
const currentScore = scores.get(pattern.domain) || 0;
|
||||
scores.set(pattern.domain, currentScore + score * (pattern.priority || 1));
|
||||
}
|
||||
}
|
||||
// Find highest scoring domain
|
||||
let bestDomain = 'general';
|
||||
let bestScore = 0;
|
||||
for (const [domain, score] of scores.entries()) {
|
||||
if (score > bestScore) {
|
||||
bestDomain = domain;
|
||||
bestScore = score;
|
||||
}
|
||||
}
|
||||
this.updateStats(bestDomain);
|
||||
return {
|
||||
domain: bestDomain,
|
||||
domainMetadata: this.extractDomainMetadata(data, bestDomain)
|
||||
};
|
||||
}
|
||||
/**
|
||||
* Score a data object against a domain pattern
|
||||
*/
|
||||
scorePattern(data, pattern) {
|
||||
let score = 0;
|
||||
// Check field matches
|
||||
if (pattern.patterns.fields) {
|
||||
const dataKeys = Object.keys(data);
|
||||
for (const field of pattern.patterns.fields) {
|
||||
if (dataKeys.some(key => key.toLowerCase().includes(field.toLowerCase()))) {
|
||||
score += 2; // Field match is strong signal
|
||||
}
|
||||
}
|
||||
}
|
||||
// Check keyword matches in values
|
||||
if (pattern.patterns.keywords) {
|
||||
const dataStr = JSON.stringify(data).toLowerCase();
|
||||
for (const keyword of pattern.patterns.keywords) {
|
||||
if (dataStr.includes(keyword.toLowerCase())) {
|
||||
score += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
// Check regex patterns
|
||||
if (pattern.patterns.regex) {
|
||||
const dataStr = JSON.stringify(data);
|
||||
if (pattern.patterns.regex.test(dataStr)) {
|
||||
score += 3; // Regex match is very specific
|
||||
}
|
||||
}
|
||||
return score;
|
||||
}
|
||||
/**
|
||||
* Extract domain-specific metadata
|
||||
*/
|
||||
extractDomainMetadata(data, domain) {
|
||||
const metadata = {};
|
||||
switch (domain) {
|
||||
case 'medical':
|
||||
if (data.patientId)
|
||||
metadata.patientId = data.patientId;
|
||||
if (data.condition)
|
||||
metadata.condition = data.condition;
|
||||
if (data.severity)
|
||||
metadata.severity = data.severity;
|
||||
break;
|
||||
case 'legal':
|
||||
if (data.caseId)
|
||||
metadata.caseId = data.caseId;
|
||||
if (data.jurisdiction)
|
||||
metadata.jurisdiction = data.jurisdiction;
|
||||
if (data.documentType)
|
||||
metadata.documentType = data.documentType;
|
||||
break;
|
||||
case 'product':
|
||||
if (data.sku)
|
||||
metadata.sku = data.sku;
|
||||
if (data.category)
|
||||
metadata.category = data.category;
|
||||
if (data.brand)
|
||||
metadata.brand = data.brand;
|
||||
if (data.price)
|
||||
metadata.priceRange = this.getPriceRange(data.price);
|
||||
break;
|
||||
case 'customer':
|
||||
if (data.customerId)
|
||||
metadata.customerId = data.customerId;
|
||||
if (data.segment)
|
||||
metadata.segment = data.segment;
|
||||
if (data.lifetime_value)
|
||||
metadata.valueCategory = this.getValueCategory(data.lifetime_value);
|
||||
break;
|
||||
case 'financial':
|
||||
if (data.accountId)
|
||||
metadata.accountId = data.accountId;
|
||||
if (data.transactionType)
|
||||
metadata.transactionType = data.transactionType;
|
||||
if (data.amount)
|
||||
metadata.amountRange = this.getAmountRange(data.amount);
|
||||
break;
|
||||
case 'technical':
|
||||
if (data.service)
|
||||
metadata.service = data.service;
|
||||
if (data.environment)
|
||||
metadata.environment = data.environment;
|
||||
if (data.severity)
|
||||
metadata.severity = data.severity;
|
||||
break;
|
||||
}
|
||||
// Add detection confidence
|
||||
metadata.detectionConfidence = this.calculateConfidence(data, domain);
|
||||
return metadata;
|
||||
}
|
||||
/**
|
||||
* Calculate detection confidence
|
||||
*/
|
||||
calculateConfidence(data, domain) {
|
||||
// If domain was explicitly specified
|
||||
if (data.domain === domain)
|
||||
return 'high';
|
||||
// Check how many patterns matched
|
||||
const pattern = [...this.customPatterns, ...this.domainPatterns]
|
||||
.find(p => p.domain === domain);
|
||||
if (!pattern)
|
||||
return 'low';
|
||||
const score = this.scorePattern(data, pattern);
|
||||
if (score >= 5)
|
||||
return 'high';
|
||||
if (score >= 2)
|
||||
return 'medium';
|
||||
return 'low';
|
||||
}
|
||||
/**
|
||||
* Categorize price ranges
|
||||
*/
|
||||
getPriceRange(price) {
|
||||
if (price < 10)
|
||||
return 'low';
|
||||
if (price < 100)
|
||||
return 'medium';
|
||||
if (price < 1000)
|
||||
return 'high';
|
||||
return 'premium';
|
||||
}
|
||||
/**
|
||||
* Categorize customer value
|
||||
*/
|
||||
getValueCategory(value) {
|
||||
if (value < 100)
|
||||
return 'low';
|
||||
if (value < 1000)
|
||||
return 'medium';
|
||||
if (value < 10000)
|
||||
return 'high';
|
||||
return 'vip';
|
||||
}
|
||||
/**
|
||||
* Categorize amount ranges
|
||||
*/
|
||||
getAmountRange(amount) {
|
||||
if (amount < 100)
|
||||
return 'micro';
|
||||
if (amount < 1000)
|
||||
return 'small';
|
||||
if (amount < 10000)
|
||||
return 'medium';
|
||||
if (amount < 100000)
|
||||
return 'large';
|
||||
return 'enterprise';
|
||||
}
|
||||
/**
|
||||
* Add custom domain pattern
|
||||
* @param pattern - Custom domain pattern to add
|
||||
*/
|
||||
addCustomPattern(pattern) {
|
||||
// Remove existing pattern for same domain if exists
|
||||
this.customPatterns = this.customPatterns.filter(p => p.domain !== pattern.domain);
|
||||
this.customPatterns.push(pattern);
|
||||
}
|
||||
/**
|
||||
* Remove custom domain pattern
|
||||
* @param domain - Domain to remove pattern for
|
||||
*/
|
||||
removeCustomPattern(domain) {
|
||||
this.customPatterns = this.customPatterns.filter(p => p.domain !== domain);
|
||||
}
|
||||
/**
|
||||
* Update domain statistics
|
||||
*/
|
||||
updateStats(domain) {
|
||||
const count = this.domainStats.get(domain) || 0;
|
||||
this.domainStats.set(domain, count + 1);
|
||||
}
|
||||
/**
|
||||
* Get domain statistics
|
||||
* @returns Map of domain to count
|
||||
*/
|
||||
getDomainStats() {
|
||||
return new Map(this.domainStats);
|
||||
}
|
||||
/**
|
||||
* Clear domain statistics
|
||||
*/
|
||||
clearStats() {
|
||||
this.domainStats.clear();
|
||||
}
|
||||
/**
|
||||
* Get all configured domains
|
||||
* @returns Array of domain names
|
||||
*/
|
||||
getConfiguredDomains() {
|
||||
const domains = new Set();
|
||||
for (const pattern of [...this.domainPatterns, ...this.customPatterns]) {
|
||||
domains.add(pattern.domain);
|
||||
}
|
||||
return Array.from(domains).sort();
|
||||
}
|
||||
}
|
||||
//# sourceMappingURL=domainDetector.js.map
|
||||
1
dist/distributed/domainDetector.js.map
vendored
Normal file
1
dist/distributed/domainDetector.js.map
vendored
Normal file
File diff suppressed because one or more lines are too long
77
dist/distributed/hashPartitioner.d.ts
vendored
Normal file
77
dist/distributed/hashPartitioner.d.ts
vendored
Normal file
|
|
@ -0,0 +1,77 @@
|
|||
/**
|
||||
* Hash-based Partitioner
|
||||
* Provides deterministic partitioning for distributed writes
|
||||
*/
|
||||
import { SharedConfig } from '../types/distributedTypes.js';
|
||||
export declare class HashPartitioner {
|
||||
private partitionCount;
|
||||
private partitionPrefix;
|
||||
constructor(config: SharedConfig);
|
||||
/**
|
||||
* Get partition for a given vector ID using deterministic hashing
|
||||
* @param vectorId - The unique identifier of the vector
|
||||
* @returns The partition path
|
||||
*/
|
||||
getPartition(vectorId: string): string;
|
||||
/**
|
||||
* Get partition with domain metadata (domain stored as metadata, not in path)
|
||||
* @param vectorId - The unique identifier of the vector
|
||||
* @param domain - The domain identifier (for metadata only)
|
||||
* @returns The partition path
|
||||
*/
|
||||
getPartitionWithDomain(vectorId: string, domain?: string): string;
|
||||
/**
|
||||
* Get all partition paths
|
||||
* @returns Array of all partition paths
|
||||
*/
|
||||
getAllPartitions(): string[];
|
||||
/**
|
||||
* Get partition index from partition path
|
||||
* @param partitionPath - The partition path
|
||||
* @returns The partition index
|
||||
*/
|
||||
getPartitionIndex(partitionPath: string): number;
|
||||
/**
|
||||
* Hash a string to a number for consistent partitioning
|
||||
* @param str - The string to hash
|
||||
* @returns A positive integer hash
|
||||
*/
|
||||
private hashString;
|
||||
/**
|
||||
* Get partitions for batch operations
|
||||
* Groups vector IDs by their target partition
|
||||
* @param vectorIds - Array of vector IDs
|
||||
* @returns Map of partition to vector IDs
|
||||
*/
|
||||
getPartitionsForBatch(vectorIds: string[]): Map<string, string[]>;
|
||||
}
|
||||
/**
|
||||
* Affinity-based Partitioner
|
||||
* Extends HashPartitioner to prefer certain partitions for a writer
|
||||
* while maintaining correctness
|
||||
*/
|
||||
export declare class AffinityPartitioner extends HashPartitioner {
|
||||
private preferredPartitions;
|
||||
private instanceId;
|
||||
constructor(config: SharedConfig, instanceId: string);
|
||||
/**
|
||||
* Calculate preferred partitions for this instance
|
||||
*/
|
||||
private calculatePreferredPartitions;
|
||||
/**
|
||||
* Check if a partition is preferred for this instance
|
||||
* @param partitionPath - The partition path
|
||||
* @returns Whether this partition is preferred
|
||||
*/
|
||||
isPreferredPartition(partitionPath: string): boolean;
|
||||
/**
|
||||
* Get all preferred partitions for this instance
|
||||
* @returns Array of preferred partition paths
|
||||
*/
|
||||
getPreferredPartitions(): string[];
|
||||
/**
|
||||
* Update preferred partitions based on new config
|
||||
* @param config - The updated shared configuration
|
||||
*/
|
||||
updatePreferences(config: SharedConfig): void;
|
||||
}
|
||||
146
dist/distributed/hashPartitioner.js
vendored
Normal file
146
dist/distributed/hashPartitioner.js
vendored
Normal file
|
|
@ -0,0 +1,146 @@
|
|||
/**
|
||||
* Hash-based Partitioner
|
||||
* Provides deterministic partitioning for distributed writes
|
||||
*/
|
||||
import { getPartitionHash } from '../utils/crypto.js';
|
||||
export class HashPartitioner {
|
||||
constructor(config) {
|
||||
this.partitionPrefix = 'vectors/p';
|
||||
this.partitionCount = config.settings.partitionCount || 100;
|
||||
}
|
||||
/**
|
||||
* Get partition for a given vector ID using deterministic hashing
|
||||
* @param vectorId - The unique identifier of the vector
|
||||
* @returns The partition path
|
||||
*/
|
||||
getPartition(vectorId) {
|
||||
const hash = this.hashString(vectorId);
|
||||
const partitionIndex = hash % this.partitionCount;
|
||||
return `${this.partitionPrefix}${partitionIndex.toString().padStart(3, '0')}`;
|
||||
}
|
||||
/**
|
||||
* Get partition with domain metadata (domain stored as metadata, not in path)
|
||||
* @param vectorId - The unique identifier of the vector
|
||||
* @param domain - The domain identifier (for metadata only)
|
||||
* @returns The partition path
|
||||
*/
|
||||
getPartitionWithDomain(vectorId, domain) {
|
||||
// Domain doesn't affect partitioning - it's just metadata
|
||||
return this.getPartition(vectorId);
|
||||
}
|
||||
/**
|
||||
* Get all partition paths
|
||||
* @returns Array of all partition paths
|
||||
*/
|
||||
getAllPartitions() {
|
||||
const partitions = [];
|
||||
for (let i = 0; i < this.partitionCount; i++) {
|
||||
partitions.push(`${this.partitionPrefix}${i.toString().padStart(3, '0')}`);
|
||||
}
|
||||
return partitions;
|
||||
}
|
||||
/**
|
||||
* Get partition index from partition path
|
||||
* @param partitionPath - The partition path
|
||||
* @returns The partition index
|
||||
*/
|
||||
getPartitionIndex(partitionPath) {
|
||||
const match = partitionPath.match(/p(\d+)$/);
|
||||
if (match) {
|
||||
return parseInt(match[1], 10);
|
||||
}
|
||||
throw new Error(`Invalid partition path: ${partitionPath}`);
|
||||
}
|
||||
/**
|
||||
* Hash a string to a number for consistent partitioning
|
||||
* @param str - The string to hash
|
||||
* @returns A positive integer hash
|
||||
*/
|
||||
hashString(str) {
|
||||
// Use our cross-platform hash function
|
||||
return getPartitionHash(str);
|
||||
}
|
||||
/**
|
||||
* Get partitions for batch operations
|
||||
* Groups vector IDs by their target partition
|
||||
* @param vectorIds - Array of vector IDs
|
||||
* @returns Map of partition to vector IDs
|
||||
*/
|
||||
getPartitionsForBatch(vectorIds) {
|
||||
const partitionMap = new Map();
|
||||
for (const id of vectorIds) {
|
||||
const partition = this.getPartition(id);
|
||||
if (!partitionMap.has(partition)) {
|
||||
partitionMap.set(partition, []);
|
||||
}
|
||||
partitionMap.get(partition).push(id);
|
||||
}
|
||||
return partitionMap;
|
||||
}
|
||||
}
|
||||
/**
|
||||
* Affinity-based Partitioner
|
||||
* Extends HashPartitioner to prefer certain partitions for a writer
|
||||
* while maintaining correctness
|
||||
*/
|
||||
export class AffinityPartitioner extends HashPartitioner {
|
||||
constructor(config, instanceId) {
|
||||
super(config);
|
||||
this.instanceId = instanceId;
|
||||
this.preferredPartitions = this.calculatePreferredPartitions(config);
|
||||
}
|
||||
/**
|
||||
* Calculate preferred partitions for this instance
|
||||
*/
|
||||
calculatePreferredPartitions(config) {
|
||||
const partitionCount = config.settings.partitionCount || 100;
|
||||
const writers = Object.entries(config.instances)
|
||||
.filter(([_, inst]) => inst.role === 'writer')
|
||||
.map(([id, _]) => id)
|
||||
.sort(); // Ensure consistent ordering
|
||||
const writerIndex = writers.indexOf(this.instanceId);
|
||||
if (writerIndex === -1) {
|
||||
// Not a writer or not found, no preferences
|
||||
return new Set();
|
||||
}
|
||||
const writerCount = writers.length;
|
||||
const partitionsPerWriter = Math.ceil(partitionCount / writerCount);
|
||||
const preferred = new Set();
|
||||
const start = writerIndex * partitionsPerWriter;
|
||||
const end = Math.min(start + partitionsPerWriter, partitionCount);
|
||||
for (let i = start; i < end; i++) {
|
||||
preferred.add(i);
|
||||
}
|
||||
return preferred;
|
||||
}
|
||||
/**
|
||||
* Check if a partition is preferred for this instance
|
||||
* @param partitionPath - The partition path
|
||||
* @returns Whether this partition is preferred
|
||||
*/
|
||||
isPreferredPartition(partitionPath) {
|
||||
try {
|
||||
const index = this.getPartitionIndex(partitionPath);
|
||||
return this.preferredPartitions.has(index);
|
||||
}
|
||||
catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
/**
|
||||
* Get all preferred partitions for this instance
|
||||
* @returns Array of preferred partition paths
|
||||
*/
|
||||
getPreferredPartitions() {
|
||||
return Array.from(this.preferredPartitions)
|
||||
.map(index => `vectors/p${index.toString().padStart(3, '0')}`);
|
||||
}
|
||||
/**
|
||||
* Update preferred partitions based on new config
|
||||
* @param config - The updated shared configuration
|
||||
*/
|
||||
updatePreferences(config) {
|
||||
this.preferredPartitions = this.calculatePreferredPartitions(config);
|
||||
}
|
||||
}
|
||||
//# sourceMappingURL=hashPartitioner.js.map
|
||||
1
dist/distributed/hashPartitioner.js.map
vendored
Normal file
1
dist/distributed/hashPartitioner.js.map
vendored
Normal file
|
|
@ -0,0 +1 @@
|
|||
{"version":3,"file":"hashPartitioner.js","sourceRoot":"","sources":["../../src/distributed/hashPartitioner.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EAAE,gBAAgB,EAAE,MAAM,oBAAoB,CAAA;AAGrD,MAAM,OAAO,eAAe;IAI1B,YAAY,MAAoB;QAFxB,oBAAe,GAAW,WAAW,CAAA;QAG3C,IAAI,CAAC,cAAc,GAAG,MAAM,CAAC,QAAQ,CAAC,cAAc,IAAI,GAAG,CAAA;IAC7D,CAAC;IAED;;;;OAIG;IACH,YAAY,CAAC,QAAgB;QAC3B,MAAM,IAAI,GAAG,IAAI,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAA;QACtC,MAAM,cAAc,GAAG,IAAI,GAAG,IAAI,CAAC,cAAc,CAAA;QACjD,OAAO,GAAG,IAAI,CAAC,eAAe,GAAG,cAAc,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CAAA;IAC/E,CAAC;IAED;;;;;OAKG;IACH,sBAAsB,CAAC,QAAgB,EAAE,MAAe;QACtD,0DAA0D;QAC1D,OAAO,IAAI,CAAC,YAAY,CAAC,QAAQ,CAAC,CAAA;IACpC,CAAC;IAED;;;OAGG;IACH,gBAAgB;QACd,MAAM,UAAU,GAAa,EAAE,CAAA;QAC/B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,cAAc,EAAE,CAAC,EAAE,EAAE,CAAC;YAC7C,UAAU,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC,eAAe,GAAG,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC,CAAA;QAC5E,CAAC;QACD,OAAO,UAAU,CAAA;IACnB,CAAC;IAED;;;;OAIG;IACH,iBAAiB,CAAC,aAAqB;QACrC,MAAM,KAAK,GAAG,aAAa,CAAC,KAAK,CAAC,SAAS,CAAC,CAAA;QAC5C,IAAI,KAAK,EAAE,CAAC;YACV,OAAO,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAA;QAC/B,CAAC;QACD,MAAM,IAAI,KAAK,CAAC,2BAA2B,aAAa,EAAE,CAAC,CAAA;IAC7D,CAAC;IAED;;;;OAIG;IACK,UAAU,CAAC,GAAW;QAC5B,uCAAuC;QACvC,OAAO,gBAAgB,CAAC,GAAG,CAAC,CAAA;IAC9B,CAAC;IAED;;;;;OAKG;IACH,qBAAqB,CAAC,SAAmB;QACvC,MAAM,YAAY,GAAG,IAAI,GAAG,EAAoB,CAAA;QAEhD,KAAK,MAAM,EAAE,IAAI,SAAS,EAAE,CAAC;YAC3B,MAAM,SAAS,GAAG,IAAI,CAAC,YAAY,CAAC,EAAE,CAAC,CAAA;YACvC,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,SAAS,CAAC,EAAE,CAAC;gBACjC,YAAY,CAAC,GAAG,CAAC,SAAS,EAAE,EAAE,CAAC,CAAA;YACjC,CAAC;YACD,YAAY,CAAC,GAAG,CAAC,SAAS,CAAE,CAAC,IAAI,CAAC,EAAE,CAAC,CAAA;QACvC,CAAC;QAED,OAAO,YAAY,CAAA;IACrB,CAAC;CACF;AAED;;;;GAIG;AACH,MAAM,OAAO,mBAAoB,SAAQ,eAAe;IAItD,YAAY,MAAoB,EAAE,UAAkB;QAClD,KAAK,CAAC,MAAM,CAAC,CAAA;QACb,IAAI,CAAC,UAAU,GAAG,UAAU,CAAA;QAC5B,IAAI,CAAC,mBAAmB,GAAG,IAAI,CAAC,4BAA4B,CAAC,MAAM,CAAC,CAAA;IACtE,CAAC;IAED;;OAEG;IACK,4BAA4B,CAAC,MAAoB;QACvD,MAAM,cAAc,GAAG,MAAM,CAAC,QAAQ,CAAC,cAAc,IAAI,GAAG,CAAA;QAC5D,MAAM,OAAO,GAAG,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,SAAS,CAAC;aAC7C,MAAM,CAAC,CAAC,CAAC,CAAC,EAAE,IAAI,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,KAAK,QAAQ,CAAC;aAC7C,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,EAAE,CAAC;aACpB,IAAI,EAAE,CAAA,CAAC,6BAA6B;QAEvC,MAAM,WAAW,GAAG,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC,UAAU,CAAC,CAAA;QACpD,IAAI,WAAW,KAAK,CAAC,CAAC,EAAE,CAAC;YACvB,4CAA4C;YAC5C,OAAO,IAAI,GAAG,EAAE,CAAA;QAClB,CAAC;QAED,MAAM,WAAW,GAAG,OAAO,CAAC,MAAM,CAAA;QAClC,MAAM,mBAAmB,GAAG,IAAI,CAAC,IAAI,CAAC,cAAc,GAAG,WAAW,CAAC,CAAA;QAEnE,MAAM,SAAS,GAAG,IAAI,GAAG,EAAU,CAAA;QACnC,MAAM,KAAK,GAAG,WAAW,GAAG,mBAAmB,CAAA;QAC/C,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,KAAK,GAAG,mBAAmB,EAAE,cAAc,CAAC,CAAA;QAEjE,KAAK,IAAI,CAAC,GAAG,KAAK,EAAE,CAAC,GAAG,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC;YACjC,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,CAAA;QAClB,CAAC;QAED,OAAO,SAAS,CAAA;IAClB,CAAC;IAED;;;;OAIG;IACH,oBAAoB,CAAC,aAAqB;QACxC,IAAI,CAAC;YACH,MAAM,KAAK,GAAG,IAAI,CAAC,iBAAiB,CAAC,aAAa,CAAC,CAAA;YACnD,OAAO,IAAI,CAAC,mBAAmB,CAAC,GAAG,CAAC,KAAK,CAAC,CAAA;QAC5C,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,KAAK,CAAA;QACd,CAAC;IACH,CAAC;IAED;;;OAGG;IACH,sBAAsB;QACpB,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,mBAAmB,CAAC;aACxC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,YAAY,KAAK,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC,CAAA;IAClE,CAAC;IAED;;;OAGG;IACH,iBAAiB,CAAC,MAAoB;QACpC,IAAI,CAAC,mBAAmB,GAAG,IAAI,CAAC,4BAA4B,CAAC,MAAM,CAAC,CAAA;IACtE,CAAC;CACF"}
|
||||
110
dist/distributed/healthMonitor.d.ts
vendored
Normal file
110
dist/distributed/healthMonitor.d.ts
vendored
Normal file
|
|
@ -0,0 +1,110 @@
|
|||
/**
|
||||
* Health Monitor
|
||||
* Monitors and reports instance health in distributed deployments
|
||||
*/
|
||||
import { DistributedConfigManager } from './configManager.js';
|
||||
export interface HealthMetrics {
|
||||
vectorCount: number;
|
||||
cacheHitRate: number;
|
||||
memoryUsage: number;
|
||||
cpuUsage?: number;
|
||||
requestsPerSecond?: number;
|
||||
averageLatency?: number;
|
||||
errorRate?: number;
|
||||
}
|
||||
export interface HealthStatus {
|
||||
status: 'healthy' | 'degraded' | 'unhealthy';
|
||||
instanceId: string;
|
||||
role: string;
|
||||
uptime: number;
|
||||
lastCheck: string;
|
||||
metrics: HealthMetrics;
|
||||
warnings?: string[];
|
||||
errors?: string[];
|
||||
}
|
||||
export declare class HealthMonitor {
|
||||
private configManager;
|
||||
private startTime;
|
||||
private requestCount;
|
||||
private errorCount;
|
||||
private totalLatency;
|
||||
private cacheHits;
|
||||
private cacheMisses;
|
||||
private vectorCount;
|
||||
private checkInterval;
|
||||
private healthCheckTimer?;
|
||||
private metricsWindow;
|
||||
private latencyWindow;
|
||||
private windowSize;
|
||||
constructor(configManager: DistributedConfigManager);
|
||||
/**
|
||||
* Start health monitoring
|
||||
*/
|
||||
start(): void;
|
||||
/**
|
||||
* Stop health monitoring
|
||||
*/
|
||||
stop(): void;
|
||||
/**
|
||||
* Update health status and metrics
|
||||
*/
|
||||
private updateHealth;
|
||||
/**
|
||||
* Collect current metrics
|
||||
*/
|
||||
private collectMetrics;
|
||||
/**
|
||||
* Calculate cache hit rate
|
||||
*/
|
||||
private calculateCacheHitRate;
|
||||
/**
|
||||
* Calculate requests per second
|
||||
*/
|
||||
private calculateRPS;
|
||||
/**
|
||||
* Calculate average latency
|
||||
*/
|
||||
private calculateAverageLatency;
|
||||
/**
|
||||
* Calculate error rate
|
||||
*/
|
||||
private calculateErrorRate;
|
||||
/**
|
||||
* Get CPU usage (simplified)
|
||||
*/
|
||||
private getCPUUsage;
|
||||
/**
|
||||
* Clean old entries from sliding windows
|
||||
*/
|
||||
private cleanWindows;
|
||||
/**
|
||||
* Record a request
|
||||
* @param latency - Request latency in milliseconds
|
||||
* @param error - Whether the request resulted in an error
|
||||
*/
|
||||
recordRequest(latency: number, error?: boolean): void;
|
||||
/**
|
||||
* Record cache access
|
||||
* @param hit - Whether it was a cache hit
|
||||
*/
|
||||
recordCacheAccess(hit: boolean): void;
|
||||
/**
|
||||
* Update vector count
|
||||
* @param count - New vector count
|
||||
*/
|
||||
updateVectorCount(count: number): void;
|
||||
/**
|
||||
* Get current health status
|
||||
* @returns Health status object
|
||||
*/
|
||||
getHealthStatus(): HealthStatus;
|
||||
/**
|
||||
* Get health check endpoint data
|
||||
* @returns JSON-serializable health data
|
||||
*/
|
||||
getHealthEndpointData(): Record<string, any>;
|
||||
/**
|
||||
* Reset metrics (useful for testing)
|
||||
*/
|
||||
resetMetrics(): void;
|
||||
}
|
||||
244
dist/distributed/healthMonitor.js
vendored
Normal file
244
dist/distributed/healthMonitor.js
vendored
Normal file
|
|
@ -0,0 +1,244 @@
|
|||
/**
|
||||
* Health Monitor
|
||||
* Monitors and reports instance health in distributed deployments
|
||||
*/
|
||||
export class HealthMonitor {
|
||||
constructor(configManager) {
|
||||
this.requestCount = 0;
|
||||
this.errorCount = 0;
|
||||
this.totalLatency = 0;
|
||||
this.cacheHits = 0;
|
||||
this.cacheMisses = 0;
|
||||
this.vectorCount = 0;
|
||||
this.checkInterval = 30000; // 30 seconds
|
||||
this.metricsWindow = []; // Sliding window for RPS calculation
|
||||
this.latencyWindow = []; // Sliding window for latency
|
||||
this.windowSize = 60000; // 1 minute window
|
||||
this.configManager = configManager;
|
||||
this.startTime = Date.now();
|
||||
}
|
||||
/**
|
||||
* Start health monitoring
|
||||
*/
|
||||
start() {
|
||||
// Initial health update
|
||||
this.updateHealth();
|
||||
// Schedule periodic health checks
|
||||
this.healthCheckTimer = setInterval(() => {
|
||||
this.updateHealth();
|
||||
}, this.checkInterval);
|
||||
}
|
||||
/**
|
||||
* Stop health monitoring
|
||||
*/
|
||||
stop() {
|
||||
if (this.healthCheckTimer) {
|
||||
clearInterval(this.healthCheckTimer);
|
||||
this.healthCheckTimer = undefined;
|
||||
}
|
||||
}
|
||||
/**
|
||||
* Update health status and metrics
|
||||
*/
|
||||
async updateHealth() {
|
||||
const metrics = this.collectMetrics();
|
||||
// Update config with latest metrics
|
||||
await this.configManager.updateMetrics({
|
||||
vectorCount: metrics.vectorCount,
|
||||
cacheHitRate: metrics.cacheHitRate,
|
||||
memoryUsage: metrics.memoryUsage,
|
||||
cpuUsage: metrics.cpuUsage
|
||||
});
|
||||
// Clean sliding windows
|
||||
this.cleanWindows();
|
||||
}
|
||||
/**
|
||||
* Collect current metrics
|
||||
*/
|
||||
collectMetrics() {
|
||||
const memUsage = process.memoryUsage();
|
||||
return {
|
||||
vectorCount: this.vectorCount,
|
||||
cacheHitRate: this.calculateCacheHitRate(),
|
||||
memoryUsage: memUsage.heapUsed,
|
||||
cpuUsage: this.getCPUUsage(),
|
||||
requestsPerSecond: this.calculateRPS(),
|
||||
averageLatency: this.calculateAverageLatency(),
|
||||
errorRate: this.calculateErrorRate()
|
||||
};
|
||||
}
|
||||
/**
|
||||
* Calculate cache hit rate
|
||||
*/
|
||||
calculateCacheHitRate() {
|
||||
const total = this.cacheHits + this.cacheMisses;
|
||||
if (total === 0)
|
||||
return 0;
|
||||
return this.cacheHits / total;
|
||||
}
|
||||
/**
|
||||
* Calculate requests per second
|
||||
*/
|
||||
calculateRPS() {
|
||||
const now = Date.now();
|
||||
const recentRequests = this.metricsWindow.filter(timestamp => now - timestamp < this.windowSize);
|
||||
return recentRequests.length / (this.windowSize / 1000);
|
||||
}
|
||||
/**
|
||||
* Calculate average latency
|
||||
*/
|
||||
calculateAverageLatency() {
|
||||
if (this.latencyWindow.length === 0)
|
||||
return 0;
|
||||
const sum = this.latencyWindow.reduce((a, b) => a + b, 0);
|
||||
return sum / this.latencyWindow.length;
|
||||
}
|
||||
/**
|
||||
* Calculate error rate
|
||||
*/
|
||||
calculateErrorRate() {
|
||||
if (this.requestCount === 0)
|
||||
return 0;
|
||||
return this.errorCount / this.requestCount;
|
||||
}
|
||||
/**
|
||||
* Get CPU usage (simplified)
|
||||
*/
|
||||
getCPUUsage() {
|
||||
// Simplified CPU usage based on process time
|
||||
const usage = process.cpuUsage();
|
||||
const total = usage.user + usage.system;
|
||||
const seconds = (Date.now() - this.startTime) / 1000;
|
||||
return Math.min(100, (total / 1000000 / seconds) * 100);
|
||||
}
|
||||
/**
|
||||
* Clean old entries from sliding windows
|
||||
*/
|
||||
cleanWindows() {
|
||||
const now = Date.now();
|
||||
const cutoff = now - this.windowSize;
|
||||
this.metricsWindow = this.metricsWindow.filter(t => t > cutoff);
|
||||
// Keep only recent latency measurements
|
||||
if (this.latencyWindow.length > 100) {
|
||||
this.latencyWindow = this.latencyWindow.slice(-100);
|
||||
}
|
||||
}
|
||||
/**
|
||||
* Record a request
|
||||
* @param latency - Request latency in milliseconds
|
||||
* @param error - Whether the request resulted in an error
|
||||
*/
|
||||
recordRequest(latency, error = false) {
|
||||
this.requestCount++;
|
||||
this.metricsWindow.push(Date.now());
|
||||
this.latencyWindow.push(latency);
|
||||
if (error) {
|
||||
this.errorCount++;
|
||||
}
|
||||
}
|
||||
/**
|
||||
* Record cache access
|
||||
* @param hit - Whether it was a cache hit
|
||||
*/
|
||||
recordCacheAccess(hit) {
|
||||
if (hit) {
|
||||
this.cacheHits++;
|
||||
}
|
||||
else {
|
||||
this.cacheMisses++;
|
||||
}
|
||||
}
|
||||
/**
|
||||
* Update vector count
|
||||
* @param count - New vector count
|
||||
*/
|
||||
updateVectorCount(count) {
|
||||
this.vectorCount = count;
|
||||
}
|
||||
/**
|
||||
* Get current health status
|
||||
* @returns Health status object
|
||||
*/
|
||||
getHealthStatus() {
|
||||
const metrics = this.collectMetrics();
|
||||
const uptime = Date.now() - this.startTime;
|
||||
const warnings = [];
|
||||
const errors = [];
|
||||
// Check for warnings
|
||||
if (metrics.memoryUsage > 1024 * 1024 * 1024) { // > 1GB
|
||||
warnings.push('High memory usage detected');
|
||||
}
|
||||
if (metrics.cacheHitRate < 0.5) {
|
||||
warnings.push('Low cache hit rate');
|
||||
}
|
||||
if (metrics.errorRate && metrics.errorRate > 0.05) {
|
||||
warnings.push('High error rate detected');
|
||||
}
|
||||
if (metrics.averageLatency && metrics.averageLatency > 1000) {
|
||||
warnings.push('High latency detected');
|
||||
}
|
||||
// Check for errors
|
||||
if (metrics.memoryUsage > 2 * 1024 * 1024 * 1024) { // > 2GB
|
||||
errors.push('Critical memory usage');
|
||||
}
|
||||
if (metrics.errorRate && metrics.errorRate > 0.2) {
|
||||
errors.push('Critical error rate');
|
||||
}
|
||||
// Determine overall status
|
||||
let status = 'healthy';
|
||||
if (errors.length > 0) {
|
||||
status = 'unhealthy';
|
||||
}
|
||||
else if (warnings.length > 0) {
|
||||
status = 'degraded';
|
||||
}
|
||||
return {
|
||||
status,
|
||||
instanceId: this.configManager.getInstanceId(),
|
||||
role: this.configManager.getRole(),
|
||||
uptime,
|
||||
lastCheck: new Date().toISOString(),
|
||||
metrics,
|
||||
warnings: warnings.length > 0 ? warnings : undefined,
|
||||
errors: errors.length > 0 ? errors : undefined
|
||||
};
|
||||
}
|
||||
/**
|
||||
* Get health check endpoint data
|
||||
* @returns JSON-serializable health data
|
||||
*/
|
||||
getHealthEndpointData() {
|
||||
const status = this.getHealthStatus();
|
||||
return {
|
||||
status: status.status,
|
||||
instanceId: status.instanceId,
|
||||
role: status.role,
|
||||
uptime: Math.floor(status.uptime / 1000), // Convert to seconds
|
||||
lastCheck: status.lastCheck,
|
||||
metrics: {
|
||||
vectorCount: status.metrics.vectorCount,
|
||||
cacheHitRate: Math.round(status.metrics.cacheHitRate * 100) / 100,
|
||||
memoryUsageMB: Math.round(status.metrics.memoryUsage / 1024 / 1024),
|
||||
cpuUsagePercent: Math.round(status.metrics.cpuUsage || 0),
|
||||
requestsPerSecond: Math.round(status.metrics.requestsPerSecond || 0),
|
||||
averageLatencyMs: Math.round(status.metrics.averageLatency || 0),
|
||||
errorRate: Math.round((status.metrics.errorRate || 0) * 100) / 100
|
||||
},
|
||||
warnings: status.warnings,
|
||||
errors: status.errors
|
||||
};
|
||||
}
|
||||
/**
|
||||
* Reset metrics (useful for testing)
|
||||
*/
|
||||
resetMetrics() {
|
||||
this.requestCount = 0;
|
||||
this.errorCount = 0;
|
||||
this.totalLatency = 0;
|
||||
this.cacheHits = 0;
|
||||
this.cacheMisses = 0;
|
||||
this.metricsWindow = [];
|
||||
this.latencyWindow = [];
|
||||
}
|
||||
}
|
||||
//# sourceMappingURL=healthMonitor.js.map
|
||||
1
dist/distributed/healthMonitor.js.map
vendored
Normal file
1
dist/distributed/healthMonitor.js.map
vendored
Normal file
File diff suppressed because one or more lines are too long
10
dist/distributed/index.d.ts
vendored
Normal file
10
dist/distributed/index.d.ts
vendored
Normal file
|
|
@ -0,0 +1,10 @@
|
|||
/**
|
||||
* Distributed module exports
|
||||
*/
|
||||
export { DistributedConfigManager } from './configManager.js';
|
||||
export { HashPartitioner, AffinityPartitioner } from './hashPartitioner.js';
|
||||
export { BaseOperationalMode, ReaderMode, WriterMode, HybridMode, OperationalModeFactory } from './operationalModes.js';
|
||||
export { DomainDetector } from './domainDetector.js';
|
||||
export { HealthMonitor } from './healthMonitor.js';
|
||||
export type { HealthMetrics, HealthStatus } from './healthMonitor.js';
|
||||
export type { DomainPattern } from './domainDetector.js';
|
||||
9
dist/distributed/index.js
vendored
Normal file
9
dist/distributed/index.js
vendored
Normal file
|
|
@ -0,0 +1,9 @@
|
|||
/**
|
||||
* Distributed module exports
|
||||
*/
|
||||
export { DistributedConfigManager } from './configManager.js';
|
||||
export { HashPartitioner, AffinityPartitioner } from './hashPartitioner.js';
|
||||
export { BaseOperationalMode, ReaderMode, WriterMode, HybridMode, OperationalModeFactory } from './operationalModes.js';
|
||||
export { DomainDetector } from './domainDetector.js';
|
||||
export { HealthMonitor } from './healthMonitor.js';
|
||||
//# sourceMappingURL=index.js.map
|
||||
1
dist/distributed/index.js.map
vendored
Normal file
1
dist/distributed/index.js.map
vendored
Normal file
|
|
@ -0,0 +1 @@
|
|||
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/distributed/index.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,EAAE,wBAAwB,EAAE,MAAM,oBAAoB,CAAA;AAC7D,OAAO,EAAE,eAAe,EAAE,mBAAmB,EAAE,MAAM,sBAAsB,CAAA;AAC3E,OAAO,EACL,mBAAmB,EACnB,UAAU,EACV,UAAU,EACV,UAAU,EACV,sBAAsB,EACvB,MAAM,uBAAuB,CAAA;AAC9B,OAAO,EAAE,cAAc,EAAE,MAAM,qBAAqB,CAAA;AACpD,OAAO,EAAE,aAAa,EAAE,MAAM,oBAAoB,CAAA"}
|
||||
104
dist/distributed/operationalModes.d.ts
vendored
Normal file
104
dist/distributed/operationalModes.d.ts
vendored
Normal file
|
|
@ -0,0 +1,104 @@
|
|||
/**
|
||||
* Operational Modes for Distributed Brainy
|
||||
* Defines different modes with optimized caching strategies
|
||||
*/
|
||||
import { OperationalMode, CacheStrategy, InstanceRole } from '../types/distributedTypes.js';
|
||||
/**
|
||||
* Base operational mode
|
||||
*/
|
||||
export declare abstract class BaseOperationalMode implements OperationalMode {
|
||||
abstract canRead: boolean;
|
||||
abstract canWrite: boolean;
|
||||
abstract canDelete: boolean;
|
||||
abstract cacheStrategy: CacheStrategy;
|
||||
/**
|
||||
* Validate operation is allowed in this mode
|
||||
*/
|
||||
validateOperation(operation: 'read' | 'write' | 'delete'): void;
|
||||
}
|
||||
/**
|
||||
* Read-only mode optimized for query performance
|
||||
*/
|
||||
export declare class ReaderMode extends BaseOperationalMode {
|
||||
canRead: boolean;
|
||||
canWrite: boolean;
|
||||
canDelete: boolean;
|
||||
cacheStrategy: CacheStrategy;
|
||||
/**
|
||||
* Get optimized cache configuration for readers
|
||||
*/
|
||||
getCacheConfig(): {
|
||||
hotCacheMaxSize: number;
|
||||
hotCacheEvictionThreshold: number;
|
||||
warmCacheTTL: number;
|
||||
batchSize: number;
|
||||
autoTune: boolean;
|
||||
autoTuneInterval: number;
|
||||
readOnly: boolean;
|
||||
};
|
||||
}
|
||||
/**
|
||||
* Write-only mode optimized for ingestion
|
||||
*/
|
||||
export declare class WriterMode extends BaseOperationalMode {
|
||||
canRead: boolean;
|
||||
canWrite: boolean;
|
||||
canDelete: boolean;
|
||||
cacheStrategy: CacheStrategy;
|
||||
/**
|
||||
* Get optimized cache configuration for writers
|
||||
*/
|
||||
getCacheConfig(): {
|
||||
hotCacheMaxSize: number;
|
||||
hotCacheEvictionThreshold: number;
|
||||
warmCacheTTL: number;
|
||||
batchSize: number;
|
||||
autoTune: boolean;
|
||||
writeOnly: boolean;
|
||||
};
|
||||
}
|
||||
/**
|
||||
* Hybrid mode that can both read and write
|
||||
*/
|
||||
export declare class HybridMode extends BaseOperationalMode {
|
||||
canRead: boolean;
|
||||
canWrite: boolean;
|
||||
canDelete: boolean;
|
||||
cacheStrategy: CacheStrategy;
|
||||
private readWriteRatio;
|
||||
/**
|
||||
* Get balanced cache configuration
|
||||
*/
|
||||
getCacheConfig(): {
|
||||
hotCacheMaxSize: number;
|
||||
hotCacheEvictionThreshold: number;
|
||||
warmCacheTTL: number;
|
||||
batchSize: number;
|
||||
autoTune: boolean;
|
||||
autoTuneInterval: number;
|
||||
};
|
||||
/**
|
||||
* Update cache strategy based on workload
|
||||
* @param readCount - Number of recent reads
|
||||
* @param writeCount - Number of recent writes
|
||||
*/
|
||||
updateWorkloadBalance(readCount: number, writeCount: number): void;
|
||||
}
|
||||
/**
|
||||
* Factory for creating operational modes
|
||||
*/
|
||||
export declare class OperationalModeFactory {
|
||||
/**
|
||||
* Create operational mode based on role
|
||||
* @param role - The instance role
|
||||
* @returns The appropriate operational mode
|
||||
*/
|
||||
static createMode(role: InstanceRole): BaseOperationalMode;
|
||||
/**
|
||||
* Create mode with custom cache strategy
|
||||
* @param role - The instance role
|
||||
* @param customStrategy - Custom cache strategy overrides
|
||||
* @returns The operational mode with custom strategy
|
||||
*/
|
||||
static createModeWithStrategy(role: InstanceRole, customStrategy: Partial<CacheStrategy>): BaseOperationalMode;
|
||||
}
|
||||
201
dist/distributed/operationalModes.js
vendored
Normal file
201
dist/distributed/operationalModes.js
vendored
Normal file
|
|
@ -0,0 +1,201 @@
|
|||
/**
|
||||
* Operational Modes for Distributed Brainy
|
||||
* Defines different modes with optimized caching strategies
|
||||
*/
|
||||
/**
|
||||
* Base operational mode
|
||||
*/
|
||||
export class BaseOperationalMode {
|
||||
/**
|
||||
* Validate operation is allowed in this mode
|
||||
*/
|
||||
validateOperation(operation) {
|
||||
switch (operation) {
|
||||
case 'read':
|
||||
if (!this.canRead) {
|
||||
throw new Error('Read operations are not allowed in write-only mode');
|
||||
}
|
||||
break;
|
||||
case 'write':
|
||||
if (!this.canWrite) {
|
||||
throw new Error('Write operations are not allowed in read-only mode');
|
||||
}
|
||||
break;
|
||||
case 'delete':
|
||||
if (!this.canDelete) {
|
||||
throw new Error('Delete operations are not allowed in this mode');
|
||||
}
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
/**
|
||||
* Read-only mode optimized for query performance
|
||||
*/
|
||||
export class ReaderMode extends BaseOperationalMode {
|
||||
constructor() {
|
||||
super(...arguments);
|
||||
this.canRead = true;
|
||||
this.canWrite = false;
|
||||
this.canDelete = false;
|
||||
this.cacheStrategy = {
|
||||
hotCacheRatio: 0.8, // 80% of memory for read cache
|
||||
prefetchAggressive: true, // Aggressively prefetch related vectors
|
||||
ttl: 3600000, // 1 hour cache TTL
|
||||
compressionEnabled: true, // Trade CPU for more cache capacity
|
||||
writeBufferSize: 0, // No write buffer needed
|
||||
batchWrites: false, // No writes
|
||||
adaptive: true // Adapt to query patterns
|
||||
};
|
||||
}
|
||||
/**
|
||||
* Get optimized cache configuration for readers
|
||||
*/
|
||||
getCacheConfig() {
|
||||
return {
|
||||
hotCacheMaxSize: 1000000, // Large hot cache
|
||||
hotCacheEvictionThreshold: 0.9, // Keep cache full
|
||||
warmCacheTTL: 3600000, // 1 hour warm cache
|
||||
batchSize: 100, // Large batch reads
|
||||
autoTune: true, // Auto-tune for read patterns
|
||||
autoTuneInterval: 60000, // Tune every minute
|
||||
readOnly: true // Enable read-only optimizations
|
||||
};
|
||||
}
|
||||
}
|
||||
/**
|
||||
* Write-only mode optimized for ingestion
|
||||
*/
|
||||
export class WriterMode extends BaseOperationalMode {
|
||||
constructor() {
|
||||
super(...arguments);
|
||||
this.canRead = false;
|
||||
this.canWrite = true;
|
||||
this.canDelete = true;
|
||||
this.cacheStrategy = {
|
||||
hotCacheRatio: 0.2, // Only 20% for cache, rest for write buffer
|
||||
prefetchAggressive: false, // No prefetching needed
|
||||
ttl: 60000, // Short TTL (1 minute)
|
||||
compressionEnabled: false, // Speed over memory efficiency
|
||||
writeBufferSize: 10000, // Large write buffer for batching
|
||||
batchWrites: true, // Enable write batching
|
||||
adaptive: false // Fixed strategy for consistent writes
|
||||
};
|
||||
}
|
||||
/**
|
||||
* Get optimized cache configuration for writers
|
||||
*/
|
||||
getCacheConfig() {
|
||||
return {
|
||||
hotCacheMaxSize: 100000, // Small hot cache
|
||||
hotCacheEvictionThreshold: 0.5, // Aggressive eviction
|
||||
warmCacheTTL: 60000, // 1 minute warm cache
|
||||
batchSize: 1000, // Large batch writes
|
||||
autoTune: false, // Fixed configuration
|
||||
writeOnly: true // Enable write-only optimizations
|
||||
};
|
||||
}
|
||||
}
|
||||
/**
|
||||
* Hybrid mode that can both read and write
|
||||
*/
|
||||
export class HybridMode extends BaseOperationalMode {
|
||||
constructor() {
|
||||
super(...arguments);
|
||||
this.canRead = true;
|
||||
this.canWrite = true;
|
||||
this.canDelete = true;
|
||||
this.cacheStrategy = {
|
||||
hotCacheRatio: 0.5, // Balanced cache/buffer allocation
|
||||
prefetchAggressive: false, // Moderate prefetching
|
||||
ttl: 600000, // 10 minute TTL
|
||||
compressionEnabled: true, // Compress when beneficial
|
||||
writeBufferSize: 5000, // Moderate write buffer
|
||||
batchWrites: true, // Batch writes when possible
|
||||
adaptive: true // Adapt to workload mix
|
||||
};
|
||||
this.readWriteRatio = 0.5; // Track read/write ratio
|
||||
}
|
||||
/**
|
||||
* Get balanced cache configuration
|
||||
*/
|
||||
getCacheConfig() {
|
||||
return {
|
||||
hotCacheMaxSize: 500000, // Medium cache size
|
||||
hotCacheEvictionThreshold: 0.7, // Balanced eviction
|
||||
warmCacheTTL: 600000, // 10 minute warm cache
|
||||
batchSize: 500, // Medium batch size
|
||||
autoTune: true, // Auto-tune based on workload
|
||||
autoTuneInterval: 300000 // Tune every 5 minutes
|
||||
};
|
||||
}
|
||||
/**
|
||||
* Update cache strategy based on workload
|
||||
* @param readCount - Number of recent reads
|
||||
* @param writeCount - Number of recent writes
|
||||
*/
|
||||
updateWorkloadBalance(readCount, writeCount) {
|
||||
const total = readCount + writeCount;
|
||||
if (total === 0)
|
||||
return;
|
||||
this.readWriteRatio = readCount / total;
|
||||
// Adjust cache strategy based on workload
|
||||
if (this.readWriteRatio > 0.8) {
|
||||
// Read-heavy workload
|
||||
this.cacheStrategy.hotCacheRatio = 0.7;
|
||||
this.cacheStrategy.prefetchAggressive = true;
|
||||
this.cacheStrategy.writeBufferSize = 2000;
|
||||
}
|
||||
else if (this.readWriteRatio < 0.2) {
|
||||
// Write-heavy workload
|
||||
this.cacheStrategy.hotCacheRatio = 0.3;
|
||||
this.cacheStrategy.prefetchAggressive = false;
|
||||
this.cacheStrategy.writeBufferSize = 8000;
|
||||
}
|
||||
else {
|
||||
// Balanced workload
|
||||
this.cacheStrategy.hotCacheRatio = 0.5;
|
||||
this.cacheStrategy.prefetchAggressive = false;
|
||||
this.cacheStrategy.writeBufferSize = 5000;
|
||||
}
|
||||
}
|
||||
}
|
||||
/**
|
||||
* Factory for creating operational modes
|
||||
*/
|
||||
export class OperationalModeFactory {
|
||||
/**
|
||||
* Create operational mode based on role
|
||||
* @param role - The instance role
|
||||
* @returns The appropriate operational mode
|
||||
*/
|
||||
static createMode(role) {
|
||||
switch (role) {
|
||||
case 'reader':
|
||||
return new ReaderMode();
|
||||
case 'writer':
|
||||
return new WriterMode();
|
||||
case 'hybrid':
|
||||
return new HybridMode();
|
||||
default:
|
||||
// Default to reader for safety
|
||||
return new ReaderMode();
|
||||
}
|
||||
}
|
||||
/**
|
||||
* Create mode with custom cache strategy
|
||||
* @param role - The instance role
|
||||
* @param customStrategy - Custom cache strategy overrides
|
||||
* @returns The operational mode with custom strategy
|
||||
*/
|
||||
static createModeWithStrategy(role, customStrategy) {
|
||||
const mode = this.createMode(role);
|
||||
// Apply custom strategy overrides
|
||||
mode.cacheStrategy = {
|
||||
...mode.cacheStrategy,
|
||||
...customStrategy
|
||||
};
|
||||
return mode;
|
||||
}
|
||||
}
|
||||
//# sourceMappingURL=operationalModes.js.map
|
||||
1
dist/distributed/operationalModes.js.map
vendored
Normal file
1
dist/distributed/operationalModes.js.map
vendored
Normal file
|
|
@ -0,0 +1 @@
|
|||
{"version":3,"file":"operationalModes.js","sourceRoot":"","sources":["../../src/distributed/operationalModes.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAQH;;GAEG;AACH,MAAM,OAAgB,mBAAmB;IAMvC;;OAEG;IACH,iBAAiB,CAAC,SAAsC;QACtD,QAAQ,SAAS,EAAE,CAAC;YAClB,KAAK,MAAM;gBACT,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC;oBAClB,MAAM,IAAI,KAAK,CAAC,oDAAoD,CAAC,CAAA;gBACvE,CAAC;gBACD,MAAK;YACP,KAAK,OAAO;gBACV,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,CAAC;oBACnB,MAAM,IAAI,KAAK,CAAC,oDAAoD,CAAC,CAAA;gBACvE,CAAC;gBACD,MAAK;YACP,KAAK,QAAQ;gBACX,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,CAAC;oBACpB,MAAM,IAAI,KAAK,CAAC,gDAAgD,CAAC,CAAA;gBACnE,CAAC;gBACD,MAAK;QACT,CAAC;IACH,CAAC;CACF;AAED;;GAEG;AACH,MAAM,OAAO,UAAW,SAAQ,mBAAmB;IAAnD;;QACE,YAAO,GAAG,IAAI,CAAA;QACd,aAAQ,GAAG,KAAK,CAAA;QAChB,cAAS,GAAG,KAAK,CAAA;QAEjB,kBAAa,GAAkB;YAC7B,aAAa,EAAE,GAAG,EAAY,+BAA+B;YAC7D,kBAAkB,EAAE,IAAI,EAAO,wCAAwC;YACvE,GAAG,EAAE,OAAO,EAAkB,mBAAmB;YACjD,kBAAkB,EAAE,IAAI,EAAO,oCAAoC;YACnE,eAAe,EAAE,CAAC,EAAY,yBAAyB;YACvD,WAAW,EAAE,KAAK,EAAY,YAAY;YAC1C,QAAQ,EAAE,IAAI,CAAgB,0BAA0B;SACzD,CAAA;IAgBH,CAAC;IAdC;;OAEG;IACH,cAAc;QACZ,OAAO;YACL,eAAe,EAAE,OAAO,EAAO,kBAAkB;YACjD,yBAAyB,EAAE,GAAG,EAAE,kBAAkB;YAClD,YAAY,EAAE,OAAO,EAAU,oBAAoB;YACnD,SAAS,EAAE,GAAG,EAAiB,oBAAoB;YACnD,QAAQ,EAAE,IAAI,EAAiB,8BAA8B;YAC7D,gBAAgB,EAAE,KAAK,EAAQ,oBAAoB;YACnD,QAAQ,EAAE,IAAI,CAAiB,iCAAiC;SACjE,CAAA;IACH,CAAC;CACF;AAED;;GAEG;AACH,MAAM,OAAO,UAAW,SAAQ,mBAAmB;IAAnD;;QACE,YAAO,GAAG,KAAK,CAAA;QACf,aAAQ,GAAG,IAAI,CAAA;QACf,cAAS,GAAG,IAAI,CAAA;QAEhB,kBAAa,GAAkB;YAC7B,aAAa,EAAE,GAAG,EAAY,4CAA4C;YAC1E,kBAAkB,EAAE,KAAK,EAAM,wBAAwB;YACvD,GAAG,EAAE,KAAK,EAAoB,uBAAuB;YACrD,kBAAkB,EAAE,KAAK,EAAM,+BAA+B;YAC9D,eAAe,EAAE,KAAK,EAAQ,kCAAkC;YAChE,WAAW,EAAE,IAAI,EAAa,wBAAwB;YACtD,QAAQ,EAAE,KAAK,CAAe,uCAAuC;SACtE,CAAA;IAeH,CAAC;IAbC;;OAEG;IACH,cAAc;QACZ,OAAO;YACL,eAAe,EAAE,MAAM,EAAS,kBAAkB;YAClD,yBAAyB,EAAE,GAAG,EAAE,sBAAsB;YACtD,YAAY,EAAE,KAAK,EAAY,sBAAsB;YACrD,SAAS,EAAE,IAAI,EAAgB,qBAAqB;YACpD,QAAQ,EAAE,KAAK,EAAgB,sBAAsB;YACrD,SAAS,EAAE,IAAI,CAAgB,kCAAkC;SAClE,CAAA;IACH,CAAC;CACF;AAED;;GAEG;AACH,MAAM,OAAO,UAAW,SAAQ,mBAAmB;IAAnD;;QACE,YAAO,GAAG,IAAI,CAAA;QACd,aAAQ,GAAG,IAAI,CAAA;QACf,cAAS,GAAG,IAAI,CAAA;QAEhB,kBAAa,GAAkB;YAC7B,aAAa,EAAE,GAAG,EAAY,mCAAmC;YACjE,kBAAkB,EAAE,KAAK,EAAM,uBAAuB;YACtD,GAAG,EAAE,MAAM,EAAmB,gBAAgB;YAC9C,kBAAkB,EAAE,IAAI,EAAO,2BAA2B;YAC1D,eAAe,EAAE,IAAI,EAAS,wBAAwB;YACtD,WAAW,EAAE,IAAI,EAAa,6BAA6B;YAC3D,QAAQ,EAAE,IAAI,CAAgB,wBAAwB;SACvD,CAAA;QAEO,mBAAc,GAAW,GAAG,CAAA,CAAC,yBAAyB;IA6ChE,CAAC;IA3CC;;OAEG;IACH,cAAc;QACZ,OAAO;YACL,eAAe,EAAE,MAAM,EAAS,oBAAoB;YACpD,yBAAyB,EAAE,GAAG,EAAE,oBAAoB;YACpD,YAAY,EAAE,MAAM,EAAW,uBAAuB;YACtD,SAAS,EAAE,GAAG,EAAiB,oBAAoB;YACnD,QAAQ,EAAE,IAAI,EAAiB,8BAA8B;YAC7D,gBAAgB,EAAE,MAAM,CAAO,uBAAuB;SACvD,CAAA;IACH,CAAC;IAED;;;;OAIG;IACH,qBAAqB,CAAC,SAAiB,EAAE,UAAkB;QACzD,MAAM,KAAK,GAAG,SAAS,GAAG,UAAU,CAAA;QACpC,IAAI,KAAK,KAAK,CAAC;YAAE,OAAM;QAEvB,IAAI,CAAC,cAAc,GAAG,SAAS,GAAG,KAAK,CAAA;QAEvC,0CAA0C;QAC1C,IAAI,IAAI,CAAC,cAAc,GAAG,GAAG,EAAE,CAAC;YAC9B,sBAAsB;YACtB,IAAI,CAAC,aAAa,CAAC,aAAa,GAAG,GAAG,CAAA;YACtC,IAAI,CAAC,aAAa,CAAC,kBAAkB,GAAG,IAAI,CAAA;YAC5C,IAAI,CAAC,aAAa,CAAC,eAAe,GAAG,IAAI,CAAA;QAC3C,CAAC;aAAM,IAAI,IAAI,CAAC,cAAc,GAAG,GAAG,EAAE,CAAC;YACrC,uBAAuB;YACvB,IAAI,CAAC,aAAa,CAAC,aAAa,GAAG,GAAG,CAAA;YACtC,IAAI,CAAC,aAAa,CAAC,kBAAkB,GAAG,KAAK,CAAA;YAC7C,IAAI,CAAC,aAAa,CAAC,eAAe,GAAG,IAAI,CAAA;QAC3C,CAAC;aAAM,CAAC;YACN,oBAAoB;YACpB,IAAI,CAAC,aAAa,CAAC,aAAa,GAAG,GAAG,CAAA;YACtC,IAAI,CAAC,aAAa,CAAC,kBAAkB,GAAG,KAAK,CAAA;YAC7C,IAAI,CAAC,aAAa,CAAC,eAAe,GAAG,IAAI,CAAA;QAC3C,CAAC;IACH,CAAC;CACF;AAED;;GAEG;AACH,MAAM,OAAO,sBAAsB;IACjC;;;;OAIG;IACH,MAAM,CAAC,UAAU,CAAC,IAAkB;QAClC,QAAQ,IAAI,EAAE,CAAC;YACb,KAAK,QAAQ;gBACX,OAAO,IAAI,UAAU,EAAE,CAAA;YACzB,KAAK,QAAQ;gBACX,OAAO,IAAI,UAAU,EAAE,CAAA;YACzB,KAAK,QAAQ;gBACX,OAAO,IAAI,UAAU,EAAE,CAAA;YACzB;gBACE,+BAA+B;gBAC/B,OAAO,IAAI,UAAU,EAAE,CAAA;QAC3B,CAAC;IACH,CAAC;IAED;;;;;OAKG;IACH,MAAM,CAAC,sBAAsB,CAC3B,IAAkB,EAClB,cAAsC;QAEtC,MAAM,IAAI,GAAG,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,CAAA;QAElC,kCAAkC;QAClC,IAAI,CAAC,aAAa,GAAG;YACnB,GAAG,IAAI,CAAC,aAAa;YACrB,GAAG,cAAc;SAClB,CAAA;QAED,OAAO,IAAI,CAAA;IACb,CAAC;CACF"}
|
||||
Loading…
Add table
Add a link
Reference in a new issue