soulcraft/brainy
2
0
Fork 0
Code Issues 1 Pull requests Projects Releases 335 Packages Wiki Activity Actions

Initial commit: Brainy - Multi-Dimensional AI Database

Browse source 
Open source vector database with HNSW indexing, graph relationships,
and metadata facets. Features CLI with professional augmentation registry
integration for discovering extensions and capabilities.
This commit is contained in:
David Snelling 2025-08-18 17:35:06 -07:00
commit f8c45f2d8d
448 changed files with 103294 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

103
dist/utils/adaptiveBackpressure.d.ts vendored Normal file
View file

@ -0,0 +1,103 @@
/**
* Adaptive Backpressure System
* Automatically manages request flow and prevents system overload
* Self-healing with pattern learning for optimal throughput
*/
interface BackpressureMetrics {
queueDepth: number;
processingRate: number;
errorRate: number;
latency: number;
throughput: number;
}
interface BackpressureConfig {
maxQueueDepth: number;
targetLatency: number;
minThroughput: number;
adaptationRate: number;
}
/**
* Self-healing backpressure manager that learns from load patterns
*/
export declare class AdaptiveBackpressure {
private logger;
private queue;
private activeOperations;
private maxConcurrent;
private metrics;
private config;
private patterns;
private circuitState;
private circuitOpenTime;
private circuitFailures;
private circuitThreshold;
private circuitTimeout;
private operationTimes;
private completedOps;
private errorOps;
private lastAdaptation;
/**
* Request permission to proceed with an operation
*/
requestPermission(operationId: string, priority?: number): Promise<void>;
/**
* Release permission after operation completes
*/
releasePermission(operationId: string, success?: boolean): void;
/**
* Check if circuit breaker is open
*/
private isCircuitOpen;
/**
* Open the circuit breaker
*/
private openCircuit;
/**
* Close the circuit breaker
*/
private closeCircuit;
/**
* Adapt configuration based on metrics
*/
private adaptIfNeeded;
/**
* Update current metrics
*/
private updateMetrics;
/**
* Learn from current load patterns
*/
private learnPattern;
/**
* Calculate optimal concurrency based on Little's Law
*/
private calculateOptimalConcurrency;
/**
* Adapt configuration based on metrics and patterns
*/
private adaptConfiguration;
/**
* Predict future load based on patterns
*/
predictLoad(futureSeconds?: number): number;
/**
* Get current configuration and metrics
*/
getStatus(): {
config: BackpressureConfig;
metrics: BackpressureMetrics;
circuit: string;
maxConcurrent: number;
activeOps: number;
queueLength: number;
};
/**
* Reset to default state
*/
reset(): void;
}
/**
* Get the global backpressure instance
*/
export declare function getGlobalBackpressure(): AdaptiveBackpressure;
export {};

342
dist/utils/adaptiveBackpressure.js vendored Normal file
View file

@ -0,0 +1,342 @@
/**
* Adaptive Backpressure System
* Automatically manages request flow and prevents system overload
* Self-healing with pattern learning for optimal throughput
*/
import { createModuleLogger } from './logger.js';
/**
* Self-healing backpressure manager that learns from load patterns
*/
export class AdaptiveBackpressure {
constructor() {
this.logger = createModuleLogger('AdaptiveBackpressure');
// Queue management
this.queue = [];
// Active operations tracking
this.activeOperations = new Set();
this.maxConcurrent = 100;
// Metrics tracking
this.metrics = {
queueDepth: 0,
processingRate: 0,
errorRate: 0,
latency: 0,
throughput: 0
};
// Configuration that adapts over time
this.config = {
maxQueueDepth: 1000,
targetLatency: 1000, // 1 second target
minThroughput: 10, // Minimum 10 ops/sec
adaptationRate: 0.1 // How quickly to adapt
};
// Historical patterns for learning
this.patterns = [];
// Circuit breaker state
this.circuitState = 'closed';
this.circuitOpenTime = 0;
this.circuitFailures = 0;
this.circuitThreshold = 5;
this.circuitTimeout = 30000; // 30 seconds
// Performance tracking
this.operationTimes = new Map();
this.completedOps = [];
this.errorOps = 0;
this.lastAdaptation = Date.now();
}
/**
* Request permission to proceed with an operation
*/
async requestPermission(operationId, priority = 1) {
// Check circuit breaker
if (this.isCircuitOpen()) {
throw new Error('Circuit breaker is open - system is recovering');
}
// Fast path for low load
if (this.activeOperations.size < this.maxConcurrent * 0.5 && this.queue.length === 0) {
this.activeOperations.add(operationId);
this.operationTimes.set(operationId, Date.now());
return;
}
// Check if we need to queue
if (this.activeOperations.size >= this.maxConcurrent) {
// Check queue depth
if (this.queue.length >= this.config.maxQueueDepth) {
throw new Error('Backpressure queue is full - try again later');
}
// Add to queue and wait
return new Promise((resolve) => {
this.queue.push({
id: operationId,
priority,
timestamp: Date.now(),
resolve
});
// Sort queue by priority (higher priority first)
this.queue.sort((a, b) => b.priority - a.priority);
// Update metrics
this.metrics.queueDepth = this.queue.length;
});
}
// Add to active operations
this.activeOperations.add(operationId);
this.operationTimes.set(operationId, Date.now());
}
/**
* Release permission after operation completes
*/
releasePermission(operationId, success = true) {
// Remove from active operations
this.activeOperations.delete(operationId);
// Track completion time
const startTime = this.operationTimes.get(operationId);
if (startTime) {
const duration = Date.now() - startTime;
this.completedOps.push(duration);
this.operationTimes.delete(operationId);
// Keep array bounded
if (this.completedOps.length > 1000) {
this.completedOps = this.completedOps.slice(-500);
}
}
// Track errors for circuit breaker
if (!success) {
this.errorOps++;
this.circuitFailures++;
// Check if we should open circuit
if (this.circuitFailures >= this.circuitThreshold) {
this.openCircuit();
}
}
else {
// Reset circuit failures on success
if (this.circuitState === 'half-open') {
this.closeCircuit();
}
}
// Process queue if there are waiting operations
if (this.queue.length > 0 && this.activeOperations.size < this.maxConcurrent) {
const next = this.queue.shift();
if (next) {
this.activeOperations.add(next.id);
this.operationTimes.set(next.id, Date.now());
next.resolve();
// Update metrics
this.metrics.queueDepth = this.queue.length;
}
}
// Adapt configuration periodically
this.adaptIfNeeded();
}
/**
* Check if circuit breaker is open
*/
isCircuitOpen() {
if (this.circuitState === 'open') {
// Check if timeout has passed
if (Date.now() - this.circuitOpenTime > this.circuitTimeout) {
this.circuitState = 'half-open';
this.logger.info('Circuit breaker entering half-open state');
return false;
}
return true;
}
return false;
}
/**
* Open the circuit breaker
*/
openCircuit() {
if (this.circuitState !== 'open') {
this.circuitState = 'open';
this.circuitOpenTime = Date.now();
this.logger.warn('Circuit breaker opened due to high error rate');
// Reduce load immediately
this.maxConcurrent = Math.max(10, Math.floor(this.maxConcurrent * 0.3));
}
}
/**
* Close the circuit breaker
*/
closeCircuit() {
this.circuitState = 'closed';
this.circuitFailures = 0;
this.logger.info('Circuit breaker closed - system recovered');
// Gradually increase capacity
this.maxConcurrent = Math.min(500, Math.floor(this.maxConcurrent * 1.5));
}
/**
* Adapt configuration based on metrics
*/
adaptIfNeeded() {
const now = Date.now();
if (now - this.lastAdaptation < 5000) { // Adapt every 5 seconds
return;
}
this.lastAdaptation = now;
this.updateMetrics();
// Learn from current patterns
this.learnPattern();
// Adapt based on metrics
this.adaptConfiguration();
}
/**
* Update current metrics
*/
updateMetrics() {
// Calculate processing rate
this.metrics.processingRate = this.completedOps.length > 0
? 1000 / (this.completedOps.reduce((a, b) => a + b, 0) / this.completedOps.length)
: 0;
// Calculate error rate
const totalOps = this.completedOps.length + this.errorOps;
this.metrics.errorRate = totalOps > 0 ? this.errorOps / totalOps : 0;
// Calculate average latency
this.metrics.latency = this.completedOps.length > 0
? this.completedOps.reduce((a, b) => a + b, 0) / this.completedOps.length
: 0;
// Calculate throughput
this.metrics.throughput = this.activeOperations.size + this.metrics.processingRate;
// Reset error counter periodically
if (this.completedOps.length > 100) {
this.errorOps = Math.floor(this.errorOps * 0.9); // Decay error count
}
}
/**
* Learn from current load patterns
*/
learnPattern() {
const currentLoad = this.activeOperations.size + this.queue.length;
const optimalConcurrency = this.calculateOptimalConcurrency();
this.patterns.push({
timestamp: Date.now(),
load: currentLoad,
optimal: optimalConcurrency
});
// Keep patterns bounded
if (this.patterns.length > 1000) {
this.patterns = this.patterns.slice(-500);
}
}
/**
* Calculate optimal concurrency based on Little's Law
*/
calculateOptimalConcurrency() {
// Little's Law: L = λ * W
// L = number of requests in system
// λ = arrival rate
// W = average time in system
if (this.metrics.latency === 0 || this.metrics.processingRate === 0) {
return this.maxConcurrent; // Keep current if no data
}
// Target: Keep latency under target while maximizing throughput
const targetConcurrency = Math.ceil(this.metrics.processingRate * (this.config.targetLatency / 1000));
// Adjust based on error rate
const errorAdjustment = 1 - (this.metrics.errorRate * 2); // Reduce by up to 50% for errors
// Apply adjustment
const adjusted = Math.floor(targetConcurrency * errorAdjustment);
// Apply bounds
return Math.max(10, Math.min(500, adjusted));
}
/**
* Adapt configuration based on metrics and patterns
*/
adaptConfiguration() {
const optimal = this.calculateOptimalConcurrency();
const current = this.maxConcurrent;
// Smooth adaptation using exponential moving average
const newConcurrency = Math.floor(current * (1 - this.config.adaptationRate) +
optimal * this.config.adaptationRate);
// Check if adaptation is needed
if (Math.abs(newConcurrency - current) > current * 0.1) { // 10% threshold
const oldValue = this.maxConcurrent;
this.maxConcurrent = newConcurrency;
this.logger.debug('Adapted concurrency', {
from: oldValue,
to: newConcurrency,
metrics: this.metrics
});
}
// Adapt queue depth based on throughput
if (this.metrics.throughput > 0) {
// Allow queue depth to be 10 seconds worth of throughput
this.config.maxQueueDepth = Math.max(100, Math.min(10000, Math.floor(this.metrics.throughput * 10)));
}
// Adapt circuit breaker threshold based on error patterns
if (this.metrics.errorRate < 0.01 && this.circuitThreshold > 5) {
this.circuitThreshold = Math.max(5, this.circuitThreshold - 1);
}
else if (this.metrics.errorRate > 0.05 && this.circuitThreshold < 20) {
this.circuitThreshold = Math.min(20, this.circuitThreshold + 1);
}
}
/**
* Predict future load based on patterns
*/
predictLoad(futureSeconds = 60) {
if (this.patterns.length < 10) {
return this.maxConcurrent; // Not enough data
}
// Simple linear regression on recent patterns
const recentPatterns = this.patterns.slice(-50);
const n = recentPatterns.length;
// Calculate averages
let sumX = 0, sumY = 0, sumXY = 0, sumX2 = 0;
const startTime = recentPatterns[0].timestamp;
recentPatterns.forEach(p => {
const x = (p.timestamp - startTime) / 1000; // Time in seconds
const y = p.load;
sumX += x;
sumY += y;
sumXY += x * y;
sumX2 += x * x;
});
// Calculate slope and intercept
const slope = (n * sumXY - sumX * sumY) / (n * sumX2 - sumX * sumX);
const intercept = (sumY - slope * sumX) / n;
// Predict future load
const currentTime = (Date.now() - startTime) / 1000;
const predictedLoad = intercept + slope * (currentTime + futureSeconds);
return Math.max(0, Math.min(this.config.maxQueueDepth, Math.floor(predictedLoad)));
}
/**
* Get current configuration and metrics
*/
getStatus() {
return {
config: { ...this.config },
metrics: { ...this.metrics },
circuit: this.circuitState,
maxConcurrent: this.maxConcurrent,
activeOps: this.activeOperations.size,
queueLength: this.queue.length
};
}
/**
* Reset to default state
*/
reset() {
this.queue = [];
this.activeOperations.clear();
this.operationTimes.clear();
this.completedOps = [];
this.errorOps = 0;
this.patterns = [];
this.circuitState = 'closed';
this.circuitFailures = 0;
this.maxConcurrent = 100;
this.logger.info('Backpressure system reset to defaults');
}
}
// Global singleton instance
let globalBackpressure = null;
/**
* Get the global backpressure instance
*/
export function getGlobalBackpressure() {
if (!globalBackpressure) {
globalBackpressure = new AdaptiveBackpressure();
}
return globalBackpressure;
}
//# sourceMappingURL=adaptiveBackpressure.js.map

1
dist/utils/adaptiveBackpressure.js.map vendored Normal file
View file

File diff suppressed because one or more lines are too long

117
dist/utils/adaptiveSocketManager.d.ts vendored Normal file
View file

@ -0,0 +1,117 @@
/**
* Adaptive Socket Manager
* Automatically manages socket pools and connection settings based on load patterns
* Zero-configuration approach that learns and adapts to workload characteristics
*/
import { NodeHttpHandler } from '@smithy/node-http-handler';
interface LoadMetrics {
requestsPerSecond: number;
pendingRequests: number;
socketUtilization: number;
errorRate: number;
latencyP50: number;
latencyP95: number;
memoryUsage: number;
}
interface AdaptiveConfig {
maxSockets: number;
maxFreeSockets: number;
keepAliveTimeout: number;
connectionTimeout: number;
socketTimeout: number;
batchSize: number;
}
/**
* Adaptive Socket Manager that automatically scales based on load patterns
*/
export declare class AdaptiveSocketManager {
private logger;
private config;
private metrics;
private history;
private maxHistorySize;
private lastAdaptationTime;
private adaptationInterval;
private consecutiveHighLoad;
private consecutiveLowLoad;
private requestStartTimes;
private requestLatencies;
private errorCount;
private successCount;
private lastMetricReset;
private currentAgent;
private currentHandler;
/**
* Get or create an optimized HTTP handler
*/
getHttpHandler(): NodeHttpHandler;
/**
* Get current batch size recommendation
*/
getBatchSize(): number;
/**
* Track request start
*/
trackRequestStart(requestId: string): void;
/**
* Track request completion
*/
trackRequestComplete(requestId: string, success: boolean): void;
/**
* Check if we should adapt configuration
*/
private adaptIfNeeded;
/**
* Update current metrics
*/
private updateMetrics;
/**
* Analyze metrics and adapt configuration
*/
private analyzeAndAdapt;
/**
* Detect high load conditions
*/
private detectHighLoad;
/**
* Detect low load conditions
*/
private detectLowLoad;
/**
* Scale up resources for high load
*/
private scaleUp;
/**
* Scale down resources for low load
*/
private scaleDown;
/**
* Handle error conditions by adjusting configuration
*/
private handleErrors;
/**
* Check if we should recreate the handler
*/
private shouldRecreateHandler;
/**
* Get current configuration (for monitoring)
*/
getConfig(): Readonly<AdaptiveConfig>;
/**
* Get current metrics (for monitoring)
*/
getMetrics(): Readonly<LoadMetrics>;
/**
* Predict optimal configuration based on historical data
*/
predictOptimalConfig(): AdaptiveConfig;
/**
* Reset to default configuration
*/
reset(): void;
}
/**
* Get the global socket manager instance
*/
export declare function getGlobalSocketManager(): AdaptiveSocketManager;
export {};

378
dist/utils/adaptiveSocketManager.js vendored Normal file
View file

@ -0,0 +1,378 @@
/**
* Adaptive Socket Manager
* Automatically manages socket pools and connection settings based on load patterns
* Zero-configuration approach that learns and adapts to workload characteristics
*/
import { Agent as HttpsAgent } from 'https';
import { NodeHttpHandler } from '@smithy/node-http-handler';
import { createModuleLogger } from './logger.js';
/**
* Adaptive Socket Manager that automatically scales based on load patterns
*/
export class AdaptiveSocketManager {
constructor() {
this.logger = createModuleLogger('AdaptiveSocketManager');
// Current configuration
this.config = {
maxSockets: 100, // Start conservative
maxFreeSockets: 20,
keepAliveTimeout: 60000,
connectionTimeout: 10000,
socketTimeout: 60000,
batchSize: 10
};
// Performance tracking
this.metrics = {
requestsPerSecond: 0,
pendingRequests: 0,
socketUtilization: 0,
errorRate: 0,
latencyP50: 0,
latencyP95: 0,
memoryUsage: 0
};
// Historical data for learning
this.history = [];
this.maxHistorySize = 100;
// Adaptation state
this.lastAdaptationTime = 0;
this.adaptationInterval = 5000; // Check every 5 seconds
this.consecutiveHighLoad = 0;
this.consecutiveLowLoad = 0;
// Request tracking
this.requestStartTimes = new Map();
this.requestLatencies = [];
this.errorCount = 0;
this.successCount = 0;
this.lastMetricReset = Date.now();
// Socket pool instances
this.currentAgent = null;
this.currentHandler = null;
}
/**
* Get or create an optimized HTTP handler
*/
getHttpHandler() {
// Adapt configuration if needed
this.adaptIfNeeded();
// Create new handler if configuration changed
if (!this.currentHandler || this.shouldRecreateHandler()) {
this.currentAgent = new HttpsAgent({
keepAlive: true,
maxSockets: this.config.maxSockets,
maxFreeSockets: this.config.maxFreeSockets,
timeout: this.config.keepAliveTimeout,
scheduling: 'fifo' // Fair scheduling for high-volume scenarios
});
this.currentHandler = new NodeHttpHandler({
httpsAgent: this.currentAgent,
connectionTimeout: this.config.connectionTimeout,
socketTimeout: this.config.socketTimeout
});
this.logger.debug('Created new HTTP handler with config:', this.config);
}
return this.currentHandler;
}
/**
* Get current batch size recommendation
*/
getBatchSize() {
this.adaptIfNeeded();
return this.config.batchSize;
}
/**
* Track request start
*/
trackRequestStart(requestId) {
this.requestStartTimes.set(requestId, Date.now());
this.metrics.pendingRequests++;
}
/**
* Track request completion
*/
trackRequestComplete(requestId, success) {
const startTime = this.requestStartTimes.get(requestId);
if (startTime) {
const latency = Date.now() - startTime;
this.requestLatencies.push(latency);
this.requestStartTimes.delete(requestId);
// Keep latency array bounded
if (this.requestLatencies.length > 1000) {
this.requestLatencies = this.requestLatencies.slice(-500);
}
}
if (success) {
this.successCount++;
}
else {
this.errorCount++;
}
this.metrics.pendingRequests = Math.max(0, this.metrics.pendingRequests - 1);
}
/**
* Check if we should adapt configuration
*/
adaptIfNeeded() {
const now = Date.now();
if (now - this.lastAdaptationTime < this.adaptationInterval) {
return;
}
this.lastAdaptationTime = now;
this.updateMetrics();
this.analyzeAndAdapt();
}
/**
* Update current metrics
*/
updateMetrics() {
const now = Date.now();
const timeSinceReset = (now - this.lastMetricReset) / 1000;
// Calculate requests per second
const totalRequests = this.successCount + this.errorCount;
this.metrics.requestsPerSecond = timeSinceReset > 0
? totalRequests / timeSinceReset
: 0;
// Calculate error rate
this.metrics.errorRate = totalRequests > 0
? this.errorCount / totalRequests
: 0;
// Calculate latency percentiles
if (this.requestLatencies.length > 0) {
const sorted = [...this.requestLatencies].sort((a, b) => a - b);
const p50Index = Math.floor(sorted.length * 0.5);
const p95Index = Math.floor(sorted.length * 0.95);
this.metrics.latencyP50 = sorted[p50Index] || 0;
this.metrics.latencyP95 = sorted[p95Index] || 0;
}
// Calculate socket utilization
this.metrics.socketUtilization = this.metrics.pendingRequests / this.config.maxSockets;
// Memory usage
if (typeof process !== 'undefined' && process.memoryUsage) {
const memUsage = process.memoryUsage();
this.metrics.memoryUsage = memUsage.heapUsed / memUsage.heapTotal;
}
// Add to history
this.history.push({ ...this.metrics });
if (this.history.length > this.maxHistorySize) {
this.history.shift();
}
// Reset counters periodically
if (timeSinceReset > 60) {
this.lastMetricReset = now;
this.successCount = 0;
this.errorCount = 0;
}
}
/**
* Analyze metrics and adapt configuration
*/
analyzeAndAdapt() {
const wasConfig = { ...this.config };
// Detect high load conditions
const isHighLoad = this.detectHighLoad();
const isLowLoad = this.detectLowLoad();
const hasErrors = this.metrics.errorRate > 0.01; // More than 1% errors
if (isHighLoad) {
this.consecutiveHighLoad++;
this.consecutiveLowLoad = 0;
if (this.consecutiveHighLoad >= 2) { // Wait for 2 consecutive high load readings
this.scaleUp();
}
}
else if (isLowLoad) {
this.consecutiveLowLoad++;
this.consecutiveHighLoad = 0;
if (this.consecutiveLowLoad >= 6) { // Wait longer before scaling down
this.scaleDown();
}
}
else {
// Reset counters if load is normal
this.consecutiveHighLoad = Math.max(0, this.consecutiveHighLoad - 1);
this.consecutiveLowLoad = Math.max(0, this.consecutiveLowLoad - 1);
}
// Handle error conditions
if (hasErrors) {
this.handleErrors();
}
// Log significant changes
if (JSON.stringify(wasConfig) !== JSON.stringify(this.config)) {
this.logger.info('Adapted configuration', {
from: wasConfig,
to: this.config,
metrics: this.metrics
});
}
}
/**
* Detect high load conditions
*/
detectHighLoad() {
return (this.metrics.socketUtilization > 0.7 || // Sockets heavily used
this.metrics.pendingRequests > this.config.maxSockets * 0.8 || // Many pending requests
this.metrics.latencyP95 > 5000 || // High latency
this.metrics.requestsPerSecond > 100 // High request rate
);
}
/**
* Detect low load conditions
*/
detectLowLoad() {
return (this.metrics.socketUtilization < 0.2 && // Sockets barely used
this.metrics.pendingRequests < 5 && // Few pending requests
this.metrics.latencyP95 < 1000 && // Low latency
this.metrics.requestsPerSecond < 10 && // Low request rate
this.metrics.memoryUsage < 0.5 // Low memory usage
);
}
/**
* Scale up resources for high load
*/
scaleUp() {
// Increase socket limits progressively
const scaleFactor = this.metrics.errorRate > 0.05 ? 1.5 : 2.0; // Scale more aggressively if no errors
this.config.maxSockets = Math.min(2000, // Hard limit to prevent resource exhaustion
Math.ceil(this.config.maxSockets * scaleFactor));
this.config.maxFreeSockets = Math.min(200, Math.ceil(this.config.maxSockets * 0.1) // Keep 10% as free sockets
);
// Increase batch size for better throughput
this.config.batchSize = Math.min(100, Math.ceil(this.config.batchSize * 1.5));
// Adjust timeouts for high load
this.config.keepAliveTimeout = 120000; // Keep connections alive longer
this.config.connectionTimeout = 15000; // Allow more time for connections
this.config.socketTimeout = 90000; // Allow more time for responses
this.logger.debug('Scaled up for high load', {
sockets: this.config.maxSockets,
batchSize: this.config.batchSize
});
}
/**
* Scale down resources for low load
*/
scaleDown() {
// Only scale down if memory pressure is low
if (this.metrics.memoryUsage > 0.7) {
return;
}
// Decrease socket limits conservatively
this.config.maxSockets = Math.max(50, // Minimum sockets
Math.floor(this.config.maxSockets * 0.7));
this.config.maxFreeSockets = Math.max(10, Math.floor(this.config.maxSockets * 0.2) // Keep 20% as free sockets
);
// Decrease batch size
this.config.batchSize = Math.max(5, Math.floor(this.config.batchSize * 0.7));
// Adjust timeouts for low load
this.config.keepAliveTimeout = 60000;
this.config.connectionTimeout = 10000;
this.config.socketTimeout = 60000;
this.logger.debug('Scaled down for low load', {
sockets: this.config.maxSockets,
batchSize: this.config.batchSize
});
}
/**
* Handle error conditions by adjusting configuration
*/
handleErrors() {
const errorRate = this.metrics.errorRate;
if (errorRate > 0.1) { // More than 10% errors
// Severe errors - back off aggressively
this.config.maxSockets = Math.max(50, Math.floor(this.config.maxSockets * 0.5));
this.config.batchSize = Math.max(1, Math.floor(this.config.batchSize * 0.3));
this.config.connectionTimeout = Math.min(30000, this.config.connectionTimeout * 2);
this.config.socketTimeout = Math.min(120000, this.config.socketTimeout * 2);
this.logger.warn('High error rate detected, backing off', {
errorRate,
newConfig: this.config
});
}
else if (errorRate > 0.05) { // More than 5% errors
// Moderate errors - reduce load slightly
this.config.batchSize = Math.max(1, Math.floor(this.config.batchSize * 0.7));
this.config.connectionTimeout = Math.min(20000, this.config.connectionTimeout * 1.2);
}
}
/**
* Check if we should recreate the handler
*/
shouldRecreateHandler() {
if (!this.currentAgent)
return true;
// Recreate if socket configuration changed significantly
const currentMaxSockets = this.currentAgent.maxSockets;
const socketsDiff = Math.abs(currentMaxSockets - this.config.maxSockets);
return socketsDiff > currentMaxSockets * 0.5; // 50% change threshold
}
/**
* Get current configuration (for monitoring)
*/
getConfig() {
return { ...this.config };
}
/**
* Get current metrics (for monitoring)
*/
getMetrics() {
return { ...this.metrics };
}
/**
* Predict optimal configuration based on historical data
*/
predictOptimalConfig() {
if (this.history.length < 10) {
return this.config; // Not enough data to predict
}
// Analyze recent history
const recentHistory = this.history.slice(-20);
const avgRPS = recentHistory.reduce((sum, m) => sum + m.requestsPerSecond, 0) / recentHistory.length;
const maxRPS = Math.max(...recentHistory.map(m => m.requestsPerSecond));
const avgLatency = recentHistory.reduce((sum, m) => sum + m.latencyP95, 0) / recentHistory.length;
// Predict optimal socket count based on request patterns
const optimalSockets = Math.min(2000, Math.max(50, Math.ceil(maxRPS * 2)));
// Predict optimal batch size based on latency
const optimalBatchSize = avgLatency < 1000 ? 50 : avgLatency < 3000 ? 20 : 10;
return {
maxSockets: optimalSockets,
maxFreeSockets: Math.ceil(optimalSockets * 0.15),
keepAliveTimeout: avgRPS > 50 ? 120000 : 60000,
connectionTimeout: avgLatency > 3000 ? 20000 : 10000,
socketTimeout: avgLatency > 3000 ? 90000 : 60000,
batchSize: optimalBatchSize
};
}
/**
* Reset to default configuration
*/
reset() {
this.config = {
maxSockets: 100,
maxFreeSockets: 20,
keepAliveTimeout: 60000,
connectionTimeout: 10000,
socketTimeout: 60000,
batchSize: 10
};
this.consecutiveHighLoad = 0;
this.consecutiveLowLoad = 0;
this.history = [];
this.requestLatencies = [];
this.errorCount = 0;
this.successCount = 0;
// Force recreation of handler
this.currentAgent = null;
this.currentHandler = null;
this.logger.info('Reset to default configuration');
}
}
// Global singleton instance
let globalSocketManager = null;
/**
* Get the global socket manager instance
*/
export function getGlobalSocketManager() {
if (!globalSocketManager) {
globalSocketManager = new AdaptiveSocketManager();
}
return globalSocketManager;
}
//# sourceMappingURL=adaptiveSocketManager.js.map

1
dist/utils/adaptiveSocketManager.js.map vendored Normal file
View file

File diff suppressed because one or more lines are too long

125
dist/utils/autoConfiguration.d.ts vendored Normal file
View file

@ -0,0 +1,125 @@
/**
* Automatic Configuration System for Brainy Vector Database
* Detects environment, resources, and data patterns to provide optimal settings
*/
export interface AutoConfigResult {
environment: 'browser' | 'nodejs' | 'serverless' | 'unknown';
availableMemory: number;
cpuCores: number;
threadingAvailable: boolean;
persistentStorageAvailable: boolean;
s3StorageDetected: boolean;
recommendedConfig: {
expectedDatasetSize: number;
maxMemoryUsage: number;
targetSearchLatency: number;
enablePartitioning: boolean;
enableCompression: boolean;
enableDistributedSearch: boolean;
enablePredictiveCaching: boolean;
partitionStrategy: 'semantic' | 'hash';
maxNodesPerPartition: number;
semanticClusters: number;
};
optimizationFlags: {
useMemoryMapping: boolean;
aggressiveCaching: boolean;
backgroundOptimization: boolean;
compressionLevel: 'none' | 'light' | 'aggressive';
};
}
export interface DatasetAnalysis {
estimatedSize: number;
vectorDimension?: number;
growthRate?: number;
accessPatterns?: 'read-heavy' | 'write-heavy' | 'balanced';
}
/**
* Automatic configuration system that detects environment and optimizes settings
*/
export declare class AutoConfiguration {
private static instance;
private cachedConfig;
private datasetStats;
private constructor();
static getInstance(): AutoConfiguration;
/**
* Detect environment and generate optimal configuration
*/
detectAndConfigure(hints?: {
expectedDataSize?: number;
s3Available?: boolean;
memoryBudget?: number;
}): Promise<AutoConfigResult>;
/**
* Update configuration based on runtime dataset analysis
*/
adaptToDataset(analysis: DatasetAnalysis): Promise<AutoConfigResult>;
/**
* Learn from performance metrics and adjust configuration
*/
learnFromPerformance(metrics: {
averageSearchTime: number;
memoryUsage: number;
cacheHitRate: number;
errorRate: number;
}): Promise<Partial<AutoConfigResult['recommendedConfig']>>;
/**
* Get minimal configuration for quick setup
*/
getQuickSetupConfig(scenario: 'small' | 'medium' | 'large' | 'enterprise'): Promise<{
expectedDatasetSize: number;
maxMemoryUsage: number;
targetSearchLatency: number;
s3Required: boolean;
}>;
/**
* Detect the current runtime environment
*/
private detectEnvironment;
/**
* Detect available system resources
*/
private detectResources;
/**
* Detect available storage capabilities
*/
private detectStorageCapabilities;
/**
* Generate recommended configuration based on detected environment and resources
*/
private generateRecommendedConfig;
/**
* Generate optimization flags based on environment and resources
*/
private generateOptimizationFlags;
/**
* Adapt configuration based on actual dataset analysis
*/
private adaptConfigurationToData;
/**
* Estimate dataset size if not provided
*/
private estimateDatasetSize;
/**
* Reset cached configuration (for testing or manual refresh)
*/
resetCache(): void;
}
/**
* Convenience function for quick auto-configuration
*/
export declare function autoConfigureBrainy(hints?: {
expectedDataSize?: number;
s3Available?: boolean;
memoryBudget?: number;
}): Promise<AutoConfigResult>;
/**
* Get quick setup configuration for common scenarios
*/
export declare function getQuickSetup(scenario: 'small' | 'medium' | 'large' | 'enterprise'): Promise<{
expectedDatasetSize: number;
maxMemoryUsage: number;
targetSearchLatency: number;
s3Required: boolean;
}>;

341
dist/utils/autoConfiguration.js vendored Normal file
View file

@ -0,0 +1,341 @@
/**
* Automatic Configuration System for Brainy Vector Database
* Detects environment, resources, and data patterns to provide optimal settings
*/
import { isBrowser, isNode, isThreadingAvailable } from './environment.js';
/**
* Automatic configuration system that detects environment and optimizes settings
*/
export class AutoConfiguration {
constructor() {
this.cachedConfig = null;
this.datasetStats = { estimatedSize: 0 };
}
static getInstance() {
if (!AutoConfiguration.instance) {
AutoConfiguration.instance = new AutoConfiguration();
}
return AutoConfiguration.instance;
}
/**
* Detect environment and generate optimal configuration
*/
async detectAndConfigure(hints) {
if (this.cachedConfig && !hints) {
return this.cachedConfig;
}
const environment = this.detectEnvironment();
const resources = await this.detectResources();
const storage = await this.detectStorageCapabilities(hints?.s3Available);
const config = {
environment,
...resources,
...storage,
recommendedConfig: this.generateRecommendedConfig(environment, resources, hints),
optimizationFlags: this.generateOptimizationFlags(environment, resources)
};
this.cachedConfig = config;
return config;
}
/**
* Update configuration based on runtime dataset analysis
*/
async adaptToDataset(analysis) {
this.datasetStats = analysis;
// Regenerate configuration with dataset insights
const currentConfig = await this.detectAndConfigure();
const adaptedConfig = this.adaptConfigurationToData(currentConfig, analysis);
this.cachedConfig = adaptedConfig;
return adaptedConfig;
}
/**
* Learn from performance metrics and adjust configuration
*/
async learnFromPerformance(metrics) {
const adjustments = {};
// Learn from search performance
if (metrics.averageSearchTime > 200) {
// Too slow - optimize for speed
adjustments.enableDistributedSearch = true;
adjustments.maxNodesPerPartition = Math.max(10000, (this.cachedConfig?.recommendedConfig.maxNodesPerPartition || 50000) * 0.8);
}
else if (metrics.averageSearchTime < 50) {
// Very fast - can optimize for quality
adjustments.maxNodesPerPartition = Math.min(100000, (this.cachedConfig?.recommendedConfig.maxNodesPerPartition || 50000) * 1.2);
}
// Learn from memory usage
if (metrics.memoryUsage > (this.cachedConfig?.recommendedConfig.maxMemoryUsage || 0) * 0.9) {
// High memory usage - enable compression
adjustments.enableCompression = true;
}
// Learn from cache performance
if (metrics.cacheHitRate < 0.7) {
// Poor cache performance - enable predictive caching
adjustments.enablePredictiveCaching = true;
}
// Update cached config with learned adjustments
if (this.cachedConfig) {
this.cachedConfig.recommendedConfig = {
...this.cachedConfig.recommendedConfig,
...adjustments
};
}
return adjustments;
}
/**
* Get minimal configuration for quick setup
*/
async getQuickSetupConfig(scenario) {
const environment = this.detectEnvironment();
const resources = await this.detectResources();
switch (scenario) {
case 'small':
return {
expectedDatasetSize: 10000,
maxMemoryUsage: Math.min(resources.availableMemory * 0.3, 1024 * 1024 * 1024), // 1GB max
targetSearchLatency: 100,
s3Required: false
};
case 'medium':
return {
expectedDatasetSize: 100000,
maxMemoryUsage: Math.min(resources.availableMemory * 0.5, 4 * 1024 * 1024 * 1024), // 4GB max
targetSearchLatency: 150,
s3Required: environment === 'serverless'
};
case 'large':
return {
expectedDatasetSize: 1000000,
maxMemoryUsage: Math.min(resources.availableMemory * 0.7, 8 * 1024 * 1024 * 1024), // 8GB max
targetSearchLatency: 200,
s3Required: true
};
case 'enterprise':
return {
expectedDatasetSize: 10000000,
maxMemoryUsage: Math.min(resources.availableMemory * 0.8, 32 * 1024 * 1024 * 1024), // 32GB max
targetSearchLatency: 300,
s3Required: true
};
}
}
/**
* Detect the current runtime environment
*/
detectEnvironment() {
if (isBrowser()) {
return 'browser';
}
if (isNode()) {
// Check for serverless environment indicators
if (process.env.AWS_LAMBDA_FUNCTION_NAME ||
process.env.VERCEL ||
process.env.NETLIFY ||
process.env.CLOUDFLARE_WORKERS) {
return 'serverless';
}
return 'nodejs';
}
return 'unknown';
}
/**
* Detect available system resources
*/
async detectResources() {
let availableMemory = 2 * 1024 * 1024 * 1024; // Default 2GB
let cpuCores = 4; // Default 4 cores
// Browser memory detection
if (isBrowser()) {
// @ts-ignore - navigator.deviceMemory is experimental
if (navigator.deviceMemory) {
// @ts-ignore
availableMemory = navigator.deviceMemory * 1024 * 1024 * 1024 * 0.3; // Use 30% of device memory
}
else {
availableMemory = 512 * 1024 * 1024; // Conservative 512MB for browsers
}
cpuCores = navigator.hardwareConcurrency || 4;
}
// Node.js memory detection
if (isNode()) {
try {
const os = await import('os');
availableMemory = os.totalmem() * 0.7; // Use 70% of total memory
cpuCores = os.cpus().length;
}
catch (error) {
// Fallback to defaults
}
}
return {
availableMemory,
cpuCores,
threadingAvailable: isThreadingAvailable()
};
}
/**
* Detect available storage capabilities
*/
async detectStorageCapabilities(s3Hint) {
let persistentStorageAvailable = false;
let s3StorageDetected = s3Hint || false;
if (isBrowser()) {
// Check for OPFS support
persistentStorageAvailable = 'navigator' in globalThis &&
'storage' in navigator &&
'getDirectory' in navigator.storage;
}
if (isNode()) {
persistentStorageAvailable = true; // Always available in Node.js
// Check for AWS SDK or S3 environment variables
s3StorageDetected = s3Hint ||
!!(process.env.AWS_ACCESS_KEY_ID && process.env.AWS_SECRET_ACCESS_KEY) ||
!!(process.env.S3_BUCKET_NAME);
}
return {
persistentStorageAvailable,
s3StorageDetected
};
}
/**
* Generate recommended configuration based on detected environment and resources
*/
generateRecommendedConfig(environment, resources, hints) {
const datasetSize = hints?.expectedDataSize || this.estimateDatasetSize();
const memoryBudget = hints?.memoryBudget || Math.floor(resources.availableMemory * 0.6);
// Base configuration
let config = {
expectedDatasetSize: datasetSize,
maxMemoryUsage: memoryBudget,
targetSearchLatency: 150,
enablePartitioning: datasetSize > 25000,
enableCompression: environment === 'browser' || memoryBudget < 2 * 1024 * 1024 * 1024,
enableDistributedSearch: resources.cpuCores > 2 && datasetSize > 50000,
enablePredictiveCaching: true,
partitionStrategy: 'semantic',
maxNodesPerPartition: 50000,
semanticClusters: 8
};
// Environment-specific adjustments
switch (environment) {
case 'browser':
config = {
...config,
maxMemoryUsage: Math.min(memoryBudget, 1024 * 1024 * 1024), // Cap at 1GB
targetSearchLatency: 200, // More lenient for browsers
enableCompression: true, // Always enable for browsers
maxNodesPerPartition: 25000, // Smaller partitions
semanticClusters: 4 // Fewer clusters to save memory
};
break;
case 'serverless':
config = {
...config,
targetSearchLatency: 500, // Account for cold starts
enablePredictiveCaching: false, // Avoid background processes
maxNodesPerPartition: 30000 // Moderate partition size
};
break;
case 'nodejs':
config = {
...config,
targetSearchLatency: 100, // Aggressive for Node.js
maxNodesPerPartition: Math.min(100000, Math.floor(datasetSize / 10)), // Larger partitions
semanticClusters: Math.min(16, Math.max(4, Math.floor(datasetSize / 50000))) // Scale clusters with data
};
break;
}
// Dataset size adjustments
if (datasetSize > 1000000) {
config.semanticClusters = Math.min(32, Math.floor(datasetSize / 100000));
config.maxNodesPerPartition = 100000;
}
else if (datasetSize < 10000) {
config.enablePartitioning = false;
config.enableDistributedSearch = false;
config.partitionStrategy = 'semantic'; // Keep semantic but disable partitioning
}
return config;
}
/**
* Generate optimization flags based on environment and resources
*/
generateOptimizationFlags(environment, resources) {
return {
useMemoryMapping: environment === 'nodejs' && resources.availableMemory > 4 * 1024 * 1024 * 1024,
aggressiveCaching: resources.availableMemory > 2 * 1024 * 1024 * 1024,
backgroundOptimization: environment !== 'serverless' && resources.cpuCores > 2,
compressionLevel: resources.availableMemory < 1024 * 1024 * 1024 ? 'aggressive' :
resources.availableMemory < 4 * 1024 * 1024 * 1024 ? 'light' : 'none'
};
}
/**
* Adapt configuration based on actual dataset analysis
*/
adaptConfigurationToData(baseConfig, analysis) {
const updatedConfig = { ...baseConfig };
// Adjust based on actual dataset size
if (analysis.estimatedSize !== baseConfig.recommendedConfig.expectedDatasetSize) {
const sizeRatio = analysis.estimatedSize / baseConfig.recommendedConfig.expectedDatasetSize;
updatedConfig.recommendedConfig.expectedDatasetSize = analysis.estimatedSize;
// Scale partition size with dataset
if (sizeRatio > 2) {
updatedConfig.recommendedConfig.maxNodesPerPartition = Math.min(100000, Math.floor(updatedConfig.recommendedConfig.maxNodesPerPartition * 1.5));
updatedConfig.recommendedConfig.semanticClusters = Math.min(32, Math.floor(updatedConfig.recommendedConfig.semanticClusters * 1.5));
}
}
// Adjust based on vector dimension
if (analysis.vectorDimension) {
if (analysis.vectorDimension > 1024) {
// High-dimensional vectors - optimize for compression
updatedConfig.recommendedConfig.enableCompression = true;
updatedConfig.optimizationFlags.compressionLevel = 'aggressive';
}
}
// Adjust based on access patterns
if (analysis.accessPatterns === 'read-heavy') {
updatedConfig.recommendedConfig.enablePredictiveCaching = true;
updatedConfig.optimizationFlags.aggressiveCaching = true;
}
else if (analysis.accessPatterns === 'write-heavy') {
updatedConfig.recommendedConfig.enablePredictiveCaching = false;
updatedConfig.optimizationFlags.backgroundOptimization = false;
}
return updatedConfig;
}
/**
* Estimate dataset size if not provided
*/
estimateDatasetSize() {
// Start with conservative estimate
const environment = this.detectEnvironment();
switch (environment) {
case 'browser': return 10000;
case 'serverless': return 50000;
case 'nodejs': return 100000;
default: return 25000;
}
}
/**
* Reset cached configuration (for testing or manual refresh)
*/
resetCache() {
this.cachedConfig = null;
this.datasetStats = { estimatedSize: 0 };
}
}
/**
* Convenience function for quick auto-configuration
*/
export async function autoConfigureBrainy(hints) {
const autoConfig = AutoConfiguration.getInstance();
return autoConfig.detectAndConfigure(hints);
}
/**
* Get quick setup configuration for common scenarios
*/
export async function getQuickSetup(scenario) {
const autoConfig = AutoConfiguration.getInstance();
return autoConfig.getQuickSetupConfig(scenario);
}
//# sourceMappingURL=autoConfiguration.js.map

1
dist/utils/autoConfiguration.js.map vendored Normal file
View file

File diff suppressed because one or more lines are too long

63
dist/utils/cacheAutoConfig.d.ts vendored Normal file
View file

@ -0,0 +1,63 @@
/**
* Intelligent cache auto-configuration system
* Adapts cache settings based on environment, usage patterns, and storage type
*/
import { SearchCacheConfig } from './searchCache.js';
import { BrainyDataConfig } from '../brainyData.js';
export interface CacheUsageStats {
totalQueries: number;
repeatQueries: number;
avgQueryTime: number;
memoryPressure: number;
storageType: 'memory' | 'opfs' | 's3' | 'filesystem';
isDistributed: boolean;
changeFrequency: number;
readWriteRatio: number;
}
export interface AutoConfigResult {
cacheConfig: SearchCacheConfig;
realtimeConfig: NonNullable<BrainyDataConfig['realtimeUpdates']>;
reasoning: string[];
}
export declare class CacheAutoConfigurator {
private stats;
private configHistory;
private lastOptimization;
/**
* Auto-detect optimal cache configuration based on current conditions
*/
autoDetectOptimalConfig(storageConfig?: BrainyDataConfig['storage'], currentStats?: Partial<CacheUsageStats>): AutoConfigResult;
/**
* Dynamically adjust configuration based on runtime performance
*/
adaptConfiguration(currentConfig: SearchCacheConfig, performanceMetrics: {
hitRate: number;
avgResponseTime: number;
memoryUsage: number;
externalChangesDetected: number;
timeSinceLastChange: number;
}): AutoConfigResult | null;
/**
* Get recommended configuration for specific use case
*/
getRecommendedConfig(useCase: 'high-consistency' | 'balanced' | 'performance-first'): AutoConfigResult;
/**
* Learn from usage patterns and improve recommendations
*/
learnFromUsage(usageData: {
queryPatterns: string[];
responseTime: number;
cacheHits: number;
totalQueries: number;
dataChanges: number;
timeWindow: number;
}): void;
private detectEnvironment;
private generateOptimalConfig;
private calculateRealtimeConfig;
private detectMemoryConstraints;
/**
* Get human-readable explanation of current configuration
*/
getConfigExplanation(config: AutoConfigResult): string;
}

261
dist/utils/cacheAutoConfig.js vendored Normal file
View file

@ -0,0 +1,261 @@
/**
* Intelligent cache auto-configuration system
* Adapts cache settings based on environment, usage patterns, and storage type
*/
export class CacheAutoConfigurator {
constructor() {
this.stats = {
totalQueries: 0,
repeatQueries: 0,
avgQueryTime: 50,
memoryPressure: 0,
storageType: 'memory',
isDistributed: false,
changeFrequency: 0,
readWriteRatio: 10,
};
this.configHistory = [];
this.lastOptimization = 0;
}
/**
* Auto-detect optimal cache configuration based on current conditions
*/
autoDetectOptimalConfig(storageConfig, currentStats) {
// Update stats with current information
if (currentStats) {
this.stats = { ...this.stats, ...currentStats };
}
// Detect environment characteristics
this.detectEnvironment(storageConfig);
// Generate optimal configuration
const result = this.generateOptimalConfig();
// Store for learning
this.configHistory.push(result);
this.lastOptimization = Date.now();
return result;
}
/**
* Dynamically adjust configuration based on runtime performance
*/
adaptConfiguration(currentConfig, performanceMetrics) {
const reasoning = [];
let needsUpdate = false;
// Check if we should update (don't over-optimize)
if (Date.now() - this.lastOptimization < 60000) {
return null; // Wait at least 1 minute between optimizations
}
// Analyze performance patterns
const adaptations = {};
// Low hit rate → adjust cache size or TTL
if (performanceMetrics.hitRate < 0.3) {
if (performanceMetrics.externalChangesDetected > 5) {
// Too many external changes → shorter TTL
adaptations.maxAge = Math.max(60000, currentConfig.maxAge * 0.7);
reasoning.push('Reduced cache TTL due to frequent external changes');
needsUpdate = true;
}
else {
// Expand cache size for better hit rate
adaptations.maxSize = Math.min(500, (currentConfig.maxSize || 100) * 1.5);
reasoning.push('Increased cache size due to low hit rate');
needsUpdate = true;
}
}
// High hit rate but slow responses → might need cache warming
if (performanceMetrics.hitRate > 0.8 && performanceMetrics.avgResponseTime > 100) {
reasoning.push('High hit rate but slow responses - consider cache warming');
}
// Memory pressure → reduce cache size
if (performanceMetrics.memoryUsage > 100 * 1024 * 1024) { // 100MB
adaptations.maxSize = Math.max(20, (currentConfig.maxSize || 100) * 0.7);
reasoning.push('Reduced cache size due to memory pressure');
needsUpdate = true;
}
// Recent external changes → adaptive TTL
if (performanceMetrics.timeSinceLastChange < 30000) { // 30 seconds
adaptations.maxAge = Math.max(30000, currentConfig.maxAge * 0.8);
reasoning.push('Shortened TTL due to recent external changes');
needsUpdate = true;
}
if (!needsUpdate) {
return null;
}
const newCacheConfig = {
...currentConfig,
...adaptations
};
const newRealtimeConfig = this.calculateRealtimeConfig();
return {
cacheConfig: newCacheConfig,
realtimeConfig: newRealtimeConfig,
reasoning
};
}
/**
* Get recommended configuration for specific use case
*/
getRecommendedConfig(useCase) {
const configs = {
'high-consistency': {
cache: { maxAge: 120000, maxSize: 50 },
realtime: { interval: 15000, enabled: true },
reasoning: ['Optimized for data consistency and real-time updates']
},
'balanced': {
cache: { maxAge: 300000, maxSize: 100 },
realtime: { interval: 30000, enabled: true },
reasoning: ['Balanced performance and consistency']
},
'performance-first': {
cache: { maxAge: 600000, maxSize: 200 },
realtime: { interval: 60000, enabled: true },
reasoning: ['Optimized for maximum cache performance']
}
};
const config = configs[useCase];
return {
cacheConfig: {
enabled: true,
...config.cache
},
realtimeConfig: {
updateIndex: true,
updateStatistics: true,
...config.realtime
},
reasoning: config.reasoning
};
}
/**
* Learn from usage patterns and improve recommendations
*/
learnFromUsage(usageData) {
// Update internal stats for better future recommendations
this.stats.totalQueries += usageData.totalQueries;
this.stats.repeatQueries += usageData.cacheHits;
this.stats.avgQueryTime = (this.stats.avgQueryTime + usageData.responseTime) / 2;
this.stats.changeFrequency = usageData.dataChanges / (usageData.timeWindow / 60000);
// Calculate read/write ratio
const writes = usageData.dataChanges;
const reads = usageData.totalQueries;
this.stats.readWriteRatio = reads > 0 ? reads / Math.max(writes, 1) : 10;
}
detectEnvironment(storageConfig) {
// Detect storage type
if (storageConfig?.s3Storage || storageConfig?.customS3Storage) {
this.stats.storageType = 's3';
this.stats.isDistributed = true;
}
else if (storageConfig?.forceFileSystemStorage) {
this.stats.storageType = 'filesystem';
}
else if (storageConfig?.forceMemoryStorage) {
this.stats.storageType = 'memory';
}
else {
// Auto-detect browser vs Node.js
this.stats.storageType = typeof window !== 'undefined' ? 'opfs' : 'filesystem';
}
// Detect distributed mode indicators
this.stats.isDistributed = this.stats.isDistributed ||
Boolean(storageConfig?.s3Storage || storageConfig?.customS3Storage);
}
generateOptimalConfig() {
const reasoning = [];
// Base configuration
let cacheConfig = {
enabled: true,
maxSize: 100,
maxAge: 300000, // 5 minutes
hitCountWeight: 0.3
};
let realtimeConfig = {
enabled: false,
interval: 60000,
updateIndex: true,
updateStatistics: true
};
// Adjust for storage type
if (this.stats.storageType === 's3' || this.stats.isDistributed) {
cacheConfig.maxAge = 180000; // 3 minutes for distributed
realtimeConfig.enabled = true;
realtimeConfig.interval = 30000; // 30 seconds
reasoning.push('Distributed storage detected - enabled real-time updates');
reasoning.push('Reduced cache TTL for distributed consistency');
}
// Adjust for read/write patterns
if (this.stats.readWriteRatio > 20) {
// Read-heavy workload
cacheConfig.maxSize = Math.min(300, (cacheConfig.maxSize || 100) * 2);
cacheConfig.maxAge = Math.min(900000, (cacheConfig.maxAge || 300000) * 1.5); // Up to 15 minutes
reasoning.push('Read-heavy workload detected - increased cache size and TTL');
}
else if (this.stats.readWriteRatio < 5) {
// Write-heavy workload
cacheConfig.maxSize = Math.max(50, (cacheConfig.maxSize || 100) * 0.7);
cacheConfig.maxAge = Math.max(60000, (cacheConfig.maxAge || 300000) * 0.6);
reasoning.push('Write-heavy workload detected - reduced cache size and TTL');
}
// Adjust for change frequency
if (this.stats.changeFrequency > 10) { // More than 10 changes per minute
realtimeConfig.interval = Math.max(10000, realtimeConfig.interval * 0.5);
cacheConfig.maxAge = Math.max(30000, (cacheConfig.maxAge || 300000) * 0.5);
reasoning.push('High change frequency detected - increased update frequency');
}
// Memory constraints
if (this.detectMemoryConstraints()) {
cacheConfig.maxSize = Math.max(20, (cacheConfig.maxSize || 100) * 0.6);
reasoning.push('Memory constraints detected - reduced cache size');
}
// Performance optimization
if (this.stats.avgQueryTime > 200) {
cacheConfig.maxSize = Math.min(500, (cacheConfig.maxSize || 100) * 1.5);
reasoning.push('Slow queries detected - increased cache size');
}
return {
cacheConfig,
realtimeConfig,
reasoning
};
}
calculateRealtimeConfig() {
return {
enabled: this.stats.isDistributed || this.stats.changeFrequency > 1,
interval: this.stats.isDistributed ? 30000 : 60000,
updateIndex: true,
updateStatistics: true
};
}
detectMemoryConstraints() {
// Simple heuristic for memory constraints
try {
if (typeof performance !== 'undefined' && 'memory' in performance) {
const memInfo = performance.memory;
return memInfo.usedJSHeapSize > memInfo.jsHeapSizeLimit * 0.8;
}
}
catch (e) {
// Ignore errors
}
// Default assumption for constrained environments
return false;
}
/**
* Get human-readable explanation of current configuration
*/
getConfigExplanation(config) {
const lines = [
'🤖 Brainy Auto-Configuration:',
'',
`📊 Cache: ${config.cacheConfig.maxSize} queries, ${config.cacheConfig.maxAge / 1000}s TTL`,
`🔄 Updates: ${config.realtimeConfig.enabled ? `Every ${(config.realtimeConfig.interval || 30000) / 1000}s` : 'Disabled'}`,
'',
'🎯 Optimizations applied:'
];
config.reasoning.forEach(reason => {
lines.push(` • ${reason}`);
});
return lines.join('\n');
}
}
//# sourceMappingURL=cacheAutoConfig.js.map

1
dist/utils/cacheAutoConfig.js.map vendored Normal file
View file

File diff suppressed because one or more lines are too long

25
dist/utils/crypto.d.ts vendored Normal file
View file

@ -0,0 +1,25 @@
/**
* Cross-platform crypto utilities
* Provides hashing functions that work in both Node.js and browser environments
*/
/**
* Simple string hash function that works in all environments
* Uses djb2 algorithm - fast and good distribution
* @param str - String to hash
* @returns Positive integer hash
*/
export declare function hashString(str: string): number;
/**
* Alternative: FNV-1a hash algorithm
* Good distribution and fast
* @param str - String to hash
* @returns Positive integer hash
*/
export declare function fnv1aHash(str: string): number;
/**
* Generate a deterministic hash for partitioning
* Uses the most appropriate algorithm for the environment
* @param input - Input string to hash
* @returns Positive integer hash suitable for modulo operations
*/
export declare function getPartitionHash(input: string): number;

45
dist/utils/crypto.js vendored Normal file
View file

@ -0,0 +1,45 @@
/**
* Cross-platform crypto utilities
* Provides hashing functions that work in both Node.js and browser environments
*/
/**
* Simple string hash function that works in all environments
* Uses djb2 algorithm - fast and good distribution
* @param str - String to hash
* @returns Positive integer hash
*/
export function hashString(str) {
let hash = 5381;
for (let i = 0; i < str.length; i++) {
const char = str.charCodeAt(i);
hash = ((hash << 5) + hash) + char; // hash * 33 + char
}
// Ensure positive number
return Math.abs(hash);
}
/**
* Alternative: FNV-1a hash algorithm
* Good distribution and fast
* @param str - String to hash
* @returns Positive integer hash
*/
export function fnv1aHash(str) {
let hash = 2166136261;
for (let i = 0; i < str.length; i++) {
hash ^= str.charCodeAt(i);
hash = (hash * 16777619) >>> 0;
}
return hash;
}
/**
* Generate a deterministic hash for partitioning
* Uses the most appropriate algorithm for the environment
* @param input - Input string to hash
* @returns Positive integer hash suitable for modulo operations
*/
export function getPartitionHash(input) {
// Use djb2 by default as it's fast and has good distribution
// This ensures consistent partitioning across all environments
return hashString(input);
}
//# sourceMappingURL=crypto.js.map

1
dist/utils/crypto.js.map vendored Normal file
View file

@ -0,0 +1 @@
{"version":3,"file":"crypto.js","sourceRoot":"","sources":["../../src/utils/crypto.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH;;;;;GAKG;AACH,MAAM,UAAU,UAAU,CAAC,GAAW;IACpC,IAAI,IAAI,GAAG,IAAI,CAAA;IACf,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,GAAG,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACpC,MAAM,IAAI,GAAG,GAAG,CAAC,UAAU,CAAC,CAAC,CAAC,CAAA;QAC9B,IAAI,GAAG,CAAC,CAAC,IAAI,IAAI,CAAC,CAAC,GAAG,IAAI,CAAC,GAAG,IAAI,CAAA,CAAC,mBAAmB;IACxD,CAAC;IACD,yBAAyB;IACzB,OAAO,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,CAAA;AACvB,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,SAAS,CAAC,GAAW;IACnC,IAAI,IAAI,GAAG,UAAU,CAAA;IACrB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,GAAG,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACpC,IAAI,IAAI,GAAG,CAAC,UAAU,CAAC,CAAC,CAAC,CAAA;QACzB,IAAI,GAAG,CAAC,IAAI,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAA;IAChC,CAAC;IACD,OAAO,IAAI,CAAA;AACb,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,gBAAgB,CAAC,KAAa;IAC5C,6DAA6D;IAC7D,+DAA+D;IAC/D,OAAO,UAAU,CAAC,KAAK,CAAC,CAAA;AAC1B,CAAC"}

42
dist/utils/distance.d.ts vendored Normal file
View file

@ -0,0 +1,42 @@
/**
* Distance functions for vector similarity calculations
* Optimized pure JavaScript implementations using enhanced array methods
* Faster than GPU for small vectors (384 dims) due to no transfer overhead
*/
import { DistanceFunction, Vector } from '../coreTypes.js';
/**
* Calculates the Euclidean distance between two vectors
* Lower values indicate higher similarity
* Optimized using array methods for Node.js 23.11+
*/
export declare const euclideanDistance: DistanceFunction;
/**
* Calculates the cosine distance between two vectors
* Lower values indicate higher similarity
* Range: 0 (identical) to 2 (opposite)
* Optimized using array methods for Node.js 23.11+
*/
export declare const cosineDistance: DistanceFunction;
/**
* Calculates the Manhattan (L1) distance between two vectors
* Lower values indicate higher similarity
* Optimized using array methods for Node.js 23.11+
*/
export declare const manhattanDistance: DistanceFunction;
/**
* Calculates the dot product similarity between two vectors
* Higher values indicate higher similarity
* Converted to a distance metric (lower is better)
* Optimized using array methods for Node.js 23.11+
*/
export declare const dotProductDistance: DistanceFunction;
/**
* Batch distance calculation using optimized JavaScript
* More efficient than GPU for small vectors due to no memory transfer overhead
*
* @param queryVector The query vector to compare against all vectors
* @param vectors Array of vectors to compare against
* @param distanceFunction The distance function to use
* @returns Promise resolving to array of distances
*/
export declare function calculateDistancesBatch(queryVector: Vector, vectors: Vector[], distanceFunction?: DistanceFunction): Promise<number[]>;

166
dist/utils/distance.js vendored Normal file
View file

@ -0,0 +1,166 @@
/**
* Distance functions for vector similarity calculations
* Optimized pure JavaScript implementations using enhanced array methods
* Faster than GPU for small vectors (384 dims) due to no transfer overhead
*/
/**
* Calculates the Euclidean distance between two vectors
* Lower values indicate higher similarity
* Optimized using array methods for Node.js 23.11+
*/
export const euclideanDistance = (a, b) => {
if (a.length !== b.length) {
throw new Error('Vectors must have the same dimensions');
}
// Use array.reduce for better performance in Node.js 23.11+
const sum = a.reduce((acc, val, i) => {
const diff = val - b[i];
return acc + diff * diff;
}, 0);
return Math.sqrt(sum);
};
/**
* Calculates the cosine distance between two vectors
* Lower values indicate higher similarity
* Range: 0 (identical) to 2 (opposite)
* Optimized using array methods for Node.js 23.11+
*/
export const cosineDistance = (a, b) => {
if (a.length !== b.length) {
throw new Error('Vectors must have the same dimensions');
}
// Use array.reduce to calculate all values in a single pass
const { dotProduct, normA, normB } = a.reduce((acc, val, i) => {
return {
dotProduct: acc.dotProduct + val * b[i],
normA: acc.normA + val * val,
normB: acc.normB + b[i] * b[i]
};
}, { dotProduct: 0, normA: 0, normB: 0 });
if (normA === 0 || normB === 0) {
return 2; // Maximum distance for zero vectors
}
const similarity = dotProduct / (Math.sqrt(normA) * Math.sqrt(normB));
// Convert cosine similarity (-1 to 1) to distance (0 to 2)
return 1 - similarity;
};
/**
* Calculates the Manhattan (L1) distance between two vectors
* Lower values indicate higher similarity
* Optimized using array methods for Node.js 23.11+
*/
export const manhattanDistance = (a, b) => {
if (a.length !== b.length) {
throw new Error('Vectors must have the same dimensions');
}
// Use array.reduce for better performance in Node.js 23.11+
return a.reduce((sum, val, i) => sum + Math.abs(val - b[i]), 0);
};
/**
* Calculates the dot product similarity between two vectors
* Higher values indicate higher similarity
* Converted to a distance metric (lower is better)
* Optimized using array methods for Node.js 23.11+
*/
export const dotProductDistance = (a, b) => {
if (a.length !== b.length) {
throw new Error('Vectors must have the same dimensions');
}
// Use array.reduce for better performance in Node.js 23.11+
const dotProduct = a.reduce((sum, val, i) => sum + val * b[i], 0);
// Convert to a distance metric (lower is better)
return -dotProduct;
};
/**
* Batch distance calculation using optimized JavaScript
* More efficient than GPU for small vectors due to no memory transfer overhead
*
* @param queryVector The query vector to compare against all vectors
* @param vectors Array of vectors to compare against
* @param distanceFunction The distance function to use
* @returns Promise resolving to array of distances
*/
export async function calculateDistancesBatch(queryVector, vectors, distanceFunction = euclideanDistance) {
// For small batches, use the standard distance function
if (vectors.length < 10) {
return vectors.map((vector) => distanceFunction(queryVector, vector));
}
try {
// Function for optimized batch distance calculation
const distanceCalculator = (args) => {
const { queryVector, vectors, distanceFnString } = args;
// Optimized JavaScript implementations for different distance functions
let distances;
if (distanceFnString.includes('euclideanDistance')) {
// Euclidean distance: sqrt(sum((a - b)^2))
distances = vectors.map((vector) => {
let sum = 0;
for (let i = 0; i < queryVector.length; i++) {
const diff = queryVector[i] - vector[i];
sum += diff * diff;
}
return Math.sqrt(sum);
});
}
else if (distanceFnString.includes('cosineDistance')) {
// Cosine distance: 1 - (a·b / (||a|| * ||b||))
distances = vectors.map((vector) => {
let dotProduct = 0;
let queryNorm = 0;
let vectorNorm = 0;
for (let i = 0; i < queryVector.length; i++) {
dotProduct += queryVector[i] * vector[i];
queryNorm += queryVector[i] * queryVector[i];
vectorNorm += vector[i] * vector[i];
}
queryNorm = Math.sqrt(queryNorm);
vectorNorm = Math.sqrt(vectorNorm);
if (queryNorm === 0 || vectorNorm === 0) {
return 1; // Maximum distance for zero vectors
}
const cosineSimilarity = dotProduct / (queryNorm * vectorNorm);
return 1 - cosineSimilarity;
});
}
else if (distanceFnString.includes('manhattanDistance')) {
// Manhattan distance: sum(|a - b|)
distances = vectors.map((vector) => {
let sum = 0;
for (let i = 0; i < queryVector.length; i++) {
sum += Math.abs(queryVector[i] - vector[i]);
}
return sum;
});
}
else if (distanceFnString.includes('dotProductDistance')) {
// Dot product distance: -sum(a * b)
distances = vectors.map((vector) => {
let dotProduct = 0;
for (let i = 0; i < queryVector.length; i++) {
dotProduct += queryVector[i] * vector[i];
}
return -dotProduct;
});
}
else {
// For unknown distance functions, use the provided function
const distanceFunction = new Function('return ' + distanceFnString)();
distances = vectors.map((vector) => distanceFunction(queryVector, vector));
}
return { distances };
};
// Use the optimized distance calculator
const result = distanceCalculator({
queryVector,
vectors,
distanceFnString: distanceFunction.toString()
});
return result.distances;
}
catch (error) {
// If anything fails, fall back to the standard distance function
console.error('Batch distance calculation failed:', error);
return vectors.map((vector) => distanceFunction(queryVector, vector));
}
}
//# sourceMappingURL=distance.js.map

1
dist/utils/distance.js.map vendored Normal file
View file

File diff suppressed because one or more lines are too long

102
dist/utils/embedding.d.ts vendored Normal file
View file

@ -0,0 +1,102 @@
/**
* Embedding functions for converting data to vectors using Transformers.js
* Complete rewrite to eliminate TensorFlow.js and use ONNX-based models
*/
import { EmbeddingFunction, EmbeddingModel, Vector } from '../coreTypes.js';
/**
* Detect the best available GPU device for the current environment
*/
export declare function detectBestDevice(): Promise<'cpu' | 'webgpu' | 'cuda'>;
/**
* Resolve device string to actual device configuration
*/
export declare function resolveDevice(device?: string): Promise<string>;
/**
* Transformers.js Sentence Encoder embedding model
* Uses ONNX Runtime for fast, offline embeddings with smaller models
* Default model: all-MiniLM-L6-v2 (384 dimensions, ~90MB)
*/
export interface TransformerEmbeddingOptions {
/** Model name/path to use - defaults to all-MiniLM-L6-v2 */
model?: string;
/** Whether to enable verbose logging */
verbose?: boolean;
/** Custom cache directory for models */
cacheDir?: string;
/** Force local files only (no downloads) */
localFilesOnly?: boolean;
/** Quantization setting (fp32, fp16, q8, q4) */
dtype?: 'fp32' | 'fp16' | 'q8' | 'q4';
/** Device to run inference on - 'auto' detects best available */
device?: 'auto' | 'cpu' | 'webgpu' | 'cuda' | 'gpu';
}
export declare class TransformerEmbedding implements EmbeddingModel {
private extractor;
private initialized;
private verbose;
private options;
/**
* Create a new TransformerEmbedding instance
*/
constructor(options?: TransformerEmbeddingOptions);
/**
* Get the default cache directory for models
*/
private getDefaultCacheDir;
/**
* Check if we're running in a test environment
*/
private isTestEnvironment;
/**
* Log message only if verbose mode is enabled
*/
private logger;
/**
* Initialize the embedding model
*/
init(): Promise<void>;
/**
* Generate embeddings for text data
*/
embed(data: string | string[]): Promise<Vector>;
/**
* Dispose of the model and free resources
*/
dispose(): Promise<void>;
/**
* Get the dimension of embeddings produced by this model
*/
getDimension(): number;
/**
* Check if the model is initialized
*/
isInitialized(): boolean;
}
export declare const UniversalSentenceEncoder: typeof TransformerEmbedding;
/**
* Create a new embedding model instance
*/
export declare function createEmbeddingModel(options?: TransformerEmbeddingOptions): EmbeddingModel;
/**
* Default embedding function using the lightweight transformer model
*/
export declare const defaultEmbeddingFunction: EmbeddingFunction;
/**
* Create an embedding function with custom options
*/
export declare function createEmbeddingFunction(options?: TransformerEmbeddingOptions): EmbeddingFunction;
/**
* Batch embedding function for processing multiple texts efficiently
*/
export declare function batchEmbed(texts: string[], options?: TransformerEmbeddingOptions): Promise<Vector[]>;
/**
* Embedding functions for specific model types
*/
export declare const embeddingFunctions: {
/** Default lightweight model (all-MiniLM-L6-v2, 384 dimensions) */
default: EmbeddingFunction;
/** Create custom embedding function */
create: typeof createEmbeddingFunction;
/** Batch processing */
batch: typeof batchEmbed;
};

409
dist/utils/embedding.js vendored Normal file
View file

@ -0,0 +1,409 @@
/**
* Embedding functions for converting data to vectors using Transformers.js
* Complete rewrite to eliminate TensorFlow.js and use ONNX-based models
*/
import { isBrowser } from './environment.js';
// @ts-ignore - Transformers.js is now the primary embedding library
import { pipeline, env } from '@huggingface/transformers';
/**
* Detect the best available GPU device for the current environment
*/
export async function detectBestDevice() {
// Browser environment - check for WebGPU support
if (isBrowser()) {
if (typeof navigator !== 'undefined' && 'gpu' in navigator) {
try {
const adapter = await navigator.gpu?.requestAdapter();
if (adapter) {
return 'webgpu';
}
}
catch (error) {
// WebGPU not available or failed to initialize
}
}
return 'cpu';
}
// Node.js environment - check for CUDA support
try {
// Check if ONNX Runtime GPU packages are available
// This is a simple heuristic - in production you might want more sophisticated detection
const hasGpu = process.env.CUDA_VISIBLE_DEVICES !== undefined ||
process.env.ONNXRUNTIME_GPU_ENABLED === 'true';
return hasGpu ? 'cuda' : 'cpu';
}
catch (error) {
return 'cpu';
}
}
/**
* Resolve device string to actual device configuration
*/
export async function resolveDevice(device = 'auto') {
if (device === 'auto') {
return await detectBestDevice();
}
// Map 'gpu' to appropriate GPU type for current environment
if (device === 'gpu') {
const detected = await detectBestDevice();
return detected === 'cpu' ? 'cpu' : detected;
}
return device;
}
export class TransformerEmbedding {
/**
* Create a new TransformerEmbedding instance
*/
constructor(options = {}) {
this.extractor = null;
this.initialized = false;
this.verbose = true;
this.verbose = options.verbose !== undefined ? options.verbose : true;
// PRODUCTION-READY MODEL CONFIGURATION
// Priority order: explicit option > environment variable > smart default
let localFilesOnly;
if (options.localFilesOnly !== undefined) {
// 1. Explicit option takes highest priority
localFilesOnly = options.localFilesOnly;
}
else if (process.env.BRAINY_ALLOW_REMOTE_MODELS !== undefined) {
// 2. Environment variable override
localFilesOnly = process.env.BRAINY_ALLOW_REMOTE_MODELS !== 'true';
}
else if (process.env.NODE_ENV === 'development') {
// 3. Development mode allows remote models
localFilesOnly = false;
}
else if (isBrowser()) {
// 4. Browser defaults to allowing remote models
localFilesOnly = false;
}
else {
// 5. Node.js production: try local first, but allow remote as fallback
// This is the NEW production-friendly default
localFilesOnly = false;
}
this.options = {
model: options.model || 'Xenova/all-MiniLM-L6-v2',
verbose: this.verbose,
cacheDir: options.cacheDir || './models',
localFilesOnly: localFilesOnly,
dtype: options.dtype || 'fp32',
device: options.device || 'auto'
};
if (this.verbose) {
this.logger('log', `Embedding config: localFilesOnly=${localFilesOnly}, model=${this.options.model}, cacheDir=${this.options.cacheDir}`);
}
// Configure transformers.js environment
if (!isBrowser()) {
// Set cache directory for Node.js
env.cacheDir = this.options.cacheDir;
// Prioritize local models for offline operation
env.allowRemoteModels = !this.options.localFilesOnly;
env.allowLocalModels = true;
}
else {
// Browser configuration
// Allow both local and remote models, but prefer local if available
env.allowLocalModels = true;
env.allowRemoteModels = true;
// Force the configuration to ensure it's applied
if (this.verbose) {
this.logger('log', `Browser env config - allowLocalModels: ${env.allowLocalModels}, allowRemoteModels: ${env.allowRemoteModels}, localFilesOnly: ${this.options.localFilesOnly}`);
}
}
}
/**
* Get the default cache directory for models
*/
async getDefaultCacheDir() {
if (isBrowser()) {
return './models'; // Browser default
}
// Check for bundled models in the package
const possiblePaths = [
// In the installed package
'./node_modules/@soulcraft/brainy/models',
// In development/source
'./models',
'./dist/../models',
// Alternative locations
'../models',
'../../models'
];
// Check if we're in Node.js and try to find the bundled models
if (typeof process !== 'undefined' && process.versions?.node) {
try {
// Use dynamic import instead of require for ES modules compatibility
const { createRequire } = await import('module');
const require = createRequire(import.meta.url);
const path = require('path');
const fs = require('fs');
// Try to resolve the package location
try {
const brainyPackagePath = require.resolve('@soulcraft/brainy/package.json');
const brainyPackageDir = path.dirname(brainyPackagePath);
const bundledModelsPath = path.join(brainyPackageDir, 'models');
if (fs.existsSync(bundledModelsPath)) {
this.logger('log', `Using bundled models from package: ${bundledModelsPath}`);
return bundledModelsPath;
}
}
catch (e) {
// Not installed as package, continue
}
// Try relative paths from current location
for (const relativePath of possiblePaths) {
const fullPath = path.resolve(relativePath);
if (fs.existsSync(fullPath)) {
this.logger('log', `Using bundled models from: ${fullPath}`);
return fullPath;
}
}
}
catch (error) {
// Silently fall back to default path if module detection fails
}
}
// Fallback to default cache directory
return './models';
}
/**
* Check if we're running in a test environment
*/
isTestEnvironment() {
// Always use real implementation - no more mocking
return false;
}
/**
* Log message only if verbose mode is enabled
*/
logger(level, message, ...args) {
if (level === 'error' || this.verbose) {
console[level](`[TransformerEmbedding] ${message}`, ...args);
}
}
/**
* Initialize the embedding model
*/
async init() {
if (this.initialized) {
return;
}
// Always use real implementation - no mocking
try {
// Resolve device configuration and cache directory
const device = await resolveDevice(this.options.device);
const cacheDir = this.options.cacheDir === './models'
? await this.getDefaultCacheDir()
: this.options.cacheDir;
this.logger('log', `Loading Transformer model: ${this.options.model} on device: ${device}`);
const startTime = Date.now();
// Load the feature extraction pipeline with GPU support
const pipelineOptions = {
cache_dir: cacheDir,
local_files_only: isBrowser() ? false : this.options.localFilesOnly,
dtype: this.options.dtype
};
// Add device configuration for GPU acceleration
if (device !== 'cpu') {
pipelineOptions.device = device;
this.logger('log', `🚀 GPU acceleration enabled: ${device}`);
}
if (this.verbose) {
this.logger('log', `Pipeline options: ${JSON.stringify(pipelineOptions)}`);
}
try {
this.extractor = await pipeline('feature-extraction', this.options.model, pipelineOptions);
}
catch (gpuError) {
// Fallback to CPU if GPU initialization fails
if (device !== 'cpu') {
this.logger('warn', `GPU initialization failed, falling back to CPU: ${gpuError?.message || gpuError}`);
const cpuOptions = { ...pipelineOptions };
delete cpuOptions.device;
this.extractor = await pipeline('feature-extraction', this.options.model, cpuOptions);
}
else {
// PRODUCTION-READY ERROR HANDLING
// If local_files_only is true and models are missing, try enabling remote downloads
if (pipelineOptions.local_files_only && gpuError?.message?.includes('local_files_only')) {
this.logger('warn', 'Local models not found, attempting remote download as fallback...');
try {
const remoteOptions = { ...pipelineOptions, local_files_only: false };
this.extractor = await pipeline('feature-extraction', this.options.model, remoteOptions);
this.logger('log', '✅ Successfully downloaded and loaded model from remote');
// Update the configuration to reflect what actually worked
this.options.localFilesOnly = false;
}
catch (remoteError) {
// Both local and remote failed - throw comprehensive error
const errorMsg = `Failed to load embedding model "${this.options.model}". ` +
`Local models not found and remote download failed. ` +
`To fix: 1) Set BRAINY_ALLOW_REMOTE_MODELS=true, ` +
`2) Run "npm run download-models", or ` +
`3) Use a custom embedding function.`;
throw new Error(errorMsg);
}
}
else {
throw gpuError;
}
}
}
const loadTime = Date.now() - startTime;
this.logger('log', `✅ Model loaded successfully in ${loadTime}ms`);
this.initialized = true;
}
catch (error) {
this.logger('error', 'Failed to initialize Transformer embedding model:', error);
throw new Error(`Transformer embedding initialization failed: ${error}`);
}
}
/**
* Generate embeddings for text data
*/
async embed(data) {
if (!this.initialized) {
await this.init();
}
try {
// Handle different input types
let textToEmbed;
if (typeof data === 'string') {
// Handle empty string case
if (data.trim() === '') {
// Return a zero vector of 384 dimensions (all-MiniLM-L6-v2 standard)
return new Array(384).fill(0);
}
textToEmbed = [data];
}
else if (Array.isArray(data) && data.every((item) => typeof item === 'string')) {
// Handle empty array or array with empty strings
if (data.length === 0 || data.every((item) => item.trim() === '')) {
return new Array(384).fill(0);
}
// Filter out empty strings
textToEmbed = data.filter((item) => item.trim() !== '');
if (textToEmbed.length === 0) {
return new Array(384).fill(0);
}
}
else {
throw new Error('TransformerEmbedding only supports string or string[] data');
}
// Ensure the extractor is available
if (!this.extractor) {
throw new Error('Transformer embedding model is not available');
}
// Generate embeddings with mean pooling and normalization
const result = await this.extractor(textToEmbed, {
pooling: 'mean',
normalize: true
});
// Extract the embedding data
let embedding;
if (textToEmbed.length === 1) {
// Single text input - return first embedding
embedding = Array.from(result.data.slice(0, 384));
}
else {
// Multiple texts - return first embedding (maintain compatibility)
embedding = Array.from(result.data.slice(0, 384));
}
// Validate embedding dimensions
if (embedding.length !== 384) {
this.logger('warn', `Unexpected embedding dimension: ${embedding.length}, expected 384`);
// Pad or truncate to 384 dimensions
if (embedding.length < 384) {
embedding = [...embedding, ...new Array(384 - embedding.length).fill(0)];
}
else {
embedding = embedding.slice(0, 384);
}
}
return embedding;
}
catch (error) {
this.logger('error', 'Error generating embeddings:', error);
throw new Error(`Failed to generate embeddings: ${error}`);
}
}
/**
* Dispose of the model and free resources
*/
async dispose() {
if (this.extractor && typeof this.extractor.dispose === 'function') {
await this.extractor.dispose();
}
this.extractor = null;
this.initialized = false;
}
/**
* Get the dimension of embeddings produced by this model
*/
getDimension() {
return 384;
}
/**
* Check if the model is initialized
*/
isInitialized() {
return this.initialized;
}
}
// Legacy alias for backward compatibility
export const UniversalSentenceEncoder = TransformerEmbedding;
/**
* Create a new embedding model instance
*/
export function createEmbeddingModel(options) {
return new TransformerEmbedding(options);
}
/**
* Default embedding function using the lightweight transformer model
*/
export const defaultEmbeddingFunction = async (data) => {
const embedder = new TransformerEmbedding({ verbose: false });
return await embedder.embed(data);
};
/**
* Create an embedding function with custom options
*/
export function createEmbeddingFunction(options = {}) {
const embedder = new TransformerEmbedding(options);
return async (data) => {
return await embedder.embed(data);
};
}
/**
* Batch embedding function for processing multiple texts efficiently
*/
export async function batchEmbed(texts, options = {}) {
const embedder = new TransformerEmbedding(options);
await embedder.init();
const embeddings = [];
// Process in batches for memory efficiency
const batchSize = 32;
for (let i = 0; i < texts.length; i += batchSize) {
const batch = texts.slice(i, i + batchSize);
for (const text of batch) {
const embedding = await embedder.embed(text);
embeddings.push(embedding);
}
}
await embedder.dispose();
return embeddings;
}
/**
* Embedding functions for specific model types
*/
export const embeddingFunctions = {
/** Default lightweight model (all-MiniLM-L6-v2, 384 dimensions) */
default: defaultEmbeddingFunction,
/** Create custom embedding function */
create: createEmbeddingFunction,
/** Batch processing */
batch: batchEmbed
};
//# sourceMappingURL=embedding.js.map

1
dist/utils/embedding.js.map vendored Normal file
View file

File diff suppressed because one or more lines are too long

49
dist/utils/environment.d.ts vendored Normal file
View file

@ -0,0 +1,49 @@
/**
* Utility functions for environment detection
*/
/**
* Check if code is running in a browser environment
*/
export declare function isBrowser(): boolean;
/**
* Check if code is running in a Node.js environment
*/
export declare function isNode(): boolean;
/**
* Check if code is running in a Web Worker environment
*/
export declare function isWebWorker(): boolean;
/**
* Check if Web Workers are available in the current environment
*/
export declare function areWebWorkersAvailable(): boolean;
/**
* Check if Worker Threads are available in the current environment (Node.js)
*/
export declare function areWorkerThreadsAvailable(): Promise<boolean>;
/**
* Synchronous version that doesn't actually try to load the module
* This is safer in ES module environments
*/
export declare function areWorkerThreadsAvailableSync(): boolean;
/**
* Determine if threading is available in the current environment
* Returns true if either Web Workers (browser) or Worker Threads (Node.js) are available
*/
export declare function isThreadingAvailable(): boolean;
/**
* Async version of isThreadingAvailable
*/
export declare function isThreadingAvailableAsync(): Promise<boolean>;
/**
* Auto-detect production environment to minimize logging costs
*/
export declare function isProductionEnvironment(): boolean;
/**
* Get appropriate log level based on environment
*/
export declare function getLogLevel(): 'silent' | 'error' | 'warn' | 'info' | 'verbose';
/**
* Check if logging should be enabled for a given level
*/
export declare function shouldLog(level: 'error' | 'warn' | 'info' | 'verbose'): boolean;

165
dist/utils/environment.js vendored Normal file
View file

@ -0,0 +1,165 @@
/**
* Utility functions for environment detection
*/
/**
* Check if code is running in a browser environment
*/
export function isBrowser() {
return typeof window !== 'undefined' && typeof document !== 'undefined';
}
/**
* Check if code is running in a Node.js environment
*/
export function isNode() {
// If browser environment is detected, prioritize it over Node.js
// This handles cases like jsdom where both window and process exist
if (isBrowser()) {
return false;
}
return (typeof process !== 'undefined' &&
process.versions != null &&
process.versions.node != null);
}
/**
* Check if code is running in a Web Worker environment
*/
export function isWebWorker() {
return (typeof self === 'object' &&
self.constructor &&
self.constructor.name === 'DedicatedWorkerGlobalScope');
}
/**
* Check if Web Workers are available in the current environment
*/
export function areWebWorkersAvailable() {
return isBrowser() && typeof Worker !== 'undefined';
}
/**
* Check if Worker Threads are available in the current environment (Node.js)
*/
export async function areWorkerThreadsAvailable() {
if (!isNode())
return false;
try {
// Use dynamic import to avoid errors in browser environments
await import('worker_threads');
return true;
}
catch (e) {
return false;
}
}
/**
* Synchronous version that doesn't actually try to load the module
* This is safer in ES module environments
*/
export function areWorkerThreadsAvailableSync() {
if (!isNode())
return false;
// In Node.js 24.4.0+, worker_threads is always available
return parseInt(process.versions.node.split('.')[0]) >= 24;
}
/**
* Determine if threading is available in the current environment
* Returns true if either Web Workers (browser) or Worker Threads (Node.js) are available
*/
export function isThreadingAvailable() {
return areWebWorkersAvailable() || areWorkerThreadsAvailableSync();
}
/**
* Async version of isThreadingAvailable
*/
export async function isThreadingAvailableAsync() {
return areWebWorkersAvailable() || (await areWorkerThreadsAvailable());
}
/**
* Auto-detect production environment to minimize logging costs
*/
export function isProductionEnvironment() {
// Node.js environment detection
if (isNode()) {
// Check common production environment indicators
const nodeEnv = process.env.NODE_ENV?.toLowerCase();
if (nodeEnv === 'production' || nodeEnv === 'prod')
return true;
// Google Cloud Run detection
if (process.env.K_SERVICE || process.env.GOOGLE_CLOUD_PROJECT)
return true;
// AWS Lambda detection
if (process.env.AWS_LAMBDA_FUNCTION_NAME || process.env.AWS_EXECUTION_ENV)
return true;
// Azure Functions detection
if (process.env.AZURE_FUNCTIONS_ENVIRONMENT || process.env.WEBSITE_SITE_NAME)
return true;
// Vercel detection
if (process.env.VERCEL || process.env.VERCEL_ENV === 'production')
return true;
// Netlify detection
if (process.env.NETLIFY && process.env.CONTEXT === 'production')
return true;
// Heroku detection
if (process.env.DYNO && process.env.NODE_ENV !== 'development')
return true;
// Railway detection
if (process.env.RAILWAY_ENVIRONMENT === 'production')
return true;
// Fly.io detection
if (process.env.FLY_APP_NAME && process.env.FLY_REGION)
return true;
// Docker in production (common patterns)
if (process.env.DOCKER_ENV === 'production' || process.env.ENVIRONMENT === 'production')
return true;
// Generic production indicators
if (process.env.PROD === 'true' || process.env.PRODUCTION === 'true')
return true;
}
// Browser environment - assume development unless explicitly production
if (isBrowser()) {
// Check for production domain patterns
const hostname = window?.location?.hostname;
if (hostname) {
// Avoid logging on production domains
if (hostname.includes('.com') || hostname.includes('.org') || hostname.includes('.net')) {
return !hostname.includes('localhost') && !hostname.includes('127.0.0.1') && !hostname.includes('dev');
}
}
}
return false;
}
/**
* Get appropriate log level based on environment
*/
export function getLogLevel() {
// Explicit log level override
const explicitLevel = process.env.BRAINY_LOG_LEVEL?.toLowerCase();
if (explicitLevel && ['silent', 'error', 'warn', 'info', 'verbose'].includes(explicitLevel)) {
return explicitLevel;
}
// Auto-detect based on environment
if (isProductionEnvironment()) {
return 'error'; // Only log errors in production to minimize costs
}
// Development environments get more verbose logging
if (process.env.NODE_ENV === 'development' || process.env.NODE_ENV === 'dev') {
return 'verbose';
}
// Test environments should be quieter
if (process.env.NODE_ENV === 'test') {
return 'warn';
}
// Default to info level
return 'info';
}
/**
* Check if logging should be enabled for a given level
*/
export function shouldLog(level) {
const currentLevel = getLogLevel();
if (currentLevel === 'silent')
return false;
const levels = ['error', 'warn', 'info', 'verbose'];
const currentIndex = levels.indexOf(currentLevel);
const messageIndex = levels.indexOf(level);
return messageIndex <= currentIndex;
}
//# sourceMappingURL=environment.js.map

1
dist/utils/environment.js.map vendored Normal file
View file

@ -0,0 +1 @@
{"version":3,"file":"environment.js","sourceRoot":"","sources":["../../src/utils/environment.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH;;GAEG;AACH,MAAM,UAAU,SAAS;IACvB,OAAO,OAAO,MAAM,KAAK,WAAW,IAAI,OAAO,QAAQ,KAAK,WAAW,CAAA;AACzE,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,MAAM;IACpB,iEAAiE;IACjE,oEAAoE;IACpE,IAAI,SAAS,EAAE,EAAE,CAAC;QAChB,OAAO,KAAK,CAAA;IACd,CAAC;IAED,OAAO,CACL,OAAO,OAAO,KAAK,WAAW;QAC9B,OAAO,CAAC,QAAQ,IAAI,IAAI;QACxB,OAAO,CAAC,QAAQ,CAAC,IAAI,IAAI,IAAI,CAC9B,CAAA;AACH,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,WAAW;IACzB,OAAO,CACL,OAAO,IAAI,KAAK,QAAQ;QACxB,IAAI,CAAC,WAAW;QAChB,IAAI,CAAC,WAAW,CAAC,IAAI,KAAK,4BAA4B,CACvD,CAAA;AACH,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,sBAAsB;IACpC,OAAO,SAAS,EAAE,IAAI,OAAO,MAAM,KAAK,WAAW,CAAA;AACrD,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,yBAAyB;IAC7C,IAAI,CAAC,MAAM,EAAE;QAAE,OAAO,KAAK,CAAA;IAE3B,IAAI,CAAC;QACH,6DAA6D;QAC7D,MAAM,MAAM,CAAC,gBAAgB,CAAC,CAAA;QAC9B,OAAO,IAAI,CAAA;IACb,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACX,OAAO,KAAK,CAAA;IACd,CAAC;AACH,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,6BAA6B;IAC3C,IAAI,CAAC,MAAM,EAAE;QAAE,OAAO,KAAK,CAAA;IAE3B,yDAAyD;IACzD,OAAO,QAAQ,CAAC,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAA;AAC5D,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,oBAAoB;IAClC,OAAO,sBAAsB,EAAE,IAAI,6BAA6B,EAAE,CAAA;AACpE,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,yBAAyB;IAC7C,OAAO,sBAAsB,EAAE,IAAI,CAAC,MAAM,yBAAyB,EAAE,CAAC,CAAA;AACxE,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,uBAAuB;IACrC,gCAAgC;IAChC,IAAI,MAAM,EAAE,EAAE,CAAC;QACb,iDAAiD;QACjD,MAAM,OAAO,GAAG,OAAO,CAAC,GAAG,CAAC,QAAQ,EAAE,WAAW,EAAE,CAAA;QACnD,IAAI,OAAO,KAAK,YAAY,IAAI,OAAO,KAAK,MAAM;YAAE,OAAO,IAAI,CAAA;QAE/D,6BAA6B;QAC7B,IAAI,OAAO,CAAC,GAAG,CAAC,SAAS,IAAI,OAAO,CAAC,GAAG,CAAC,oBAAoB;YAAE,OAAO,IAAI,CAAA;QAE1E,yBAAyB;QACzB,IAAI,OAAO,CAAC,GAAG,CAAC,wBAAwB,IAAI,OAAO,CAAC,GAAG,CAAC,iBAAiB;YAAE,OAAO,IAAI,CAAA;QAEtF,4BAA4B;QAC5B,IAAI,OAAO,CAAC,GAAG,CAAC,2BAA2B,IAAI,OAAO,CAAC,GAAG,CAAC,iBAAiB;YAAE,OAAO,IAAI,CAAA;QAEzF,mBAAmB;QACnB,IAAI,OAAO,CAAC,GAAG,CAAC,MAAM,IAAI,OAAO,CAAC,GAAG,CAAC,UAAU,KAAK,YAAY;YAAE,OAAO,IAAI,CAAA;QAE9E,oBAAoB;QACpB,IAAI,OAAO,CAAC,GAAG,CAAC,OAAO,IAAI,OAAO,CAAC,GAAG,CAAC,OAAO,KAAK,YAAY;YAAE,OAAO,IAAI,CAAA;QAE5E,mBAAmB;QACnB,IAAI,OAAO,CAAC,GAAG,CAAC,IAAI,IAAI,OAAO,CAAC,GAAG,CAAC,QAAQ,KAAK,aAAa;YAAE,OAAO,IAAI,CAAA;QAE3E,oBAAoB;QACpB,IAAI,OAAO,CAAC,GAAG,CAAC,mBAAmB,KAAK,YAAY;YAAE,OAAO,IAAI,CAAA;QAEjE,mBAAmB;QACnB,IAAI,OAAO,CAAC,GAAG,CAAC,YAAY,IAAI,OAAO,CAAC,GAAG,CAAC,UAAU;YAAE,OAAO,IAAI,CAAA;QAEnE,yCAAyC;QACzC,IAAI,OAAO,CAAC,GAAG,CAAC,UAAU,KAAK,YAAY,IAAI,OAAO,CAAC,GAAG,CAAC,WAAW,KAAK,YAAY;YAAE,OAAO,IAAI,CAAA;QAEpG,gCAAgC;QAChC,IAAI,OAAO,CAAC,GAAG,CAAC,IAAI,KAAK,MAAM,IAAI,OAAO,CAAC,GAAG,CAAC,UAAU,KAAK,MAAM;YAAE,OAAO,IAAI,CAAA;IACnF,CAAC;IAED,wEAAwE;IACxE,IAAI,SAAS,EAAE,EAAE,CAAC;QAChB,uCAAuC;QACvC,MAAM,QAAQ,GAAG,MAAM,EAAE,QAAQ,EAAE,QAAQ,CAAA;QAC3C,IAAI,QAAQ,EAAE,CAAC;YACb,sCAAsC;YACtC,IAAI,QAAQ,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,QAAQ,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,QAAQ,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC;gBACxF,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC,WAAW,CAAC,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,WAAW,CAAC,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAA;YACxG,CAAC;QACH,CAAC;IACH,CAAC;IAED,OAAO,KAAK,CAAA;AACd,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,WAAW;IACzB,8BAA8B;IAC9B,MAAM,aAAa,GAAG,OAAO,CAAC,GAAG,CAAC,gBAAgB,EAAE,WAAW,EAAE,CAAA;IACjE,IAAI,aAAa,IAAI,CAAC,QAAQ,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,SAAS,CAAC,CAAC,QAAQ,CAAC,aAAa,CAAC,EAAE,CAAC;QAC5F,OAAO,aAAiE,CAAA;IAC1E,CAAC;IAED,mCAAmC;IACnC,IAAI,uBAAuB,EAAE,EAAE,CAAC;QAC9B,OAAO,OAAO,CAAA,CAAC,kDAAkD;IACnE,CAAC;IAED,oDAAoD;IACpD,IAAI,OAAO,CAAC,GAAG,CAAC,QAAQ,KAAK,aAAa,IAAI,OAAO,CAAC,GAAG,CAAC,QAAQ,KAAK,KAAK,EAAE,CAAC;QAC7E,OAAO,SAAS,CAAA;IAClB,CAAC;IAED,sCAAsC;IACtC,IAAI,OAAO,CAAC,GAAG,CAAC,QAAQ,KAAK,MAAM,EAAE,CAAC;QACpC,OAAO,MAAM,CAAA;IACf,CAAC;IAED,wBAAwB;IACxB,OAAO,MAAM,CAAA;AACf,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,SAAS,CAAC,KAA4C;IACpE,MAAM,YAAY,GAAG,WAAW,EAAE,CAAA;IAElC,IAAI,YAAY,KAAK,QAAQ;QAAE,OAAO,KAAK,CAAA;IAE3C,MAAM,MAAM,GAAG,CAAC,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,SAAS,CAAC,CAAA;IACnD,MAAM,YAAY,GAAG,MAAM,CAAC,OAAO,CAAC,YAAY,CAAC,CAAA;IACjD,MAAM,YAAY,GAAG,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,CAAA;IAE1C,OAAO,YAAY,IAAI,YAAY,CAAA;AACrC,CAAC"}

21
dist/utils/fieldNameTracking.d.ts vendored Normal file
View file

@ -0,0 +1,21 @@
/**
* Utility functions for tracking and managing field names in JSON documents
*/
/**
* Extracts field names from a JSON document
* @param jsonObject The JSON object to extract field names from
* @param options Configuration options
* @returns An array of field paths (e.g., "user.name", "addresses[0].city")
*/
export declare function extractFieldNamesFromJson(jsonObject: any, options?: {
maxDepth?: number;
currentDepth?: number;
currentPath?: string;
fieldNames?: Set<string>;
}): string[];
/**
* Maps field names to standard field names based on common patterns
* @param fieldName The field name to map
* @returns The standard field name if a match is found, or null if no match
*/
export declare function mapToStandardField(fieldName: string): string | null;

90
dist/utils/fieldNameTracking.js vendored Normal file
View file

@ -0,0 +1,90 @@
/**
* Utility functions for tracking and managing field names in JSON documents
*/
/**
* Extracts field names from a JSON document
* @param jsonObject The JSON object to extract field names from
* @param options Configuration options
* @returns An array of field paths (e.g., "user.name", "addresses[0].city")
*/
export function extractFieldNamesFromJson(jsonObject, options = {}) {
const { maxDepth = 5, currentDepth = 0, currentPath = '', fieldNames = new Set() } = options;
if (jsonObject === null ||
jsonObject === undefined ||
typeof jsonObject !== 'object' ||
currentDepth >= maxDepth) {
return Array.from(fieldNames);
}
if (Array.isArray(jsonObject)) {
// For arrays, we'll just check the first item to avoid explosion of paths
if (jsonObject.length > 0) {
const arrayPath = currentPath ? `${currentPath}[0]` : '[0]';
extractFieldNamesFromJson(jsonObject[0], {
maxDepth,
currentDepth: currentDepth + 1,
currentPath: arrayPath,
fieldNames
});
}
}
else {
// For objects, process each property
for (const key of Object.keys(jsonObject)) {
const value = jsonObject[key];
const fieldPath = currentPath ? `${currentPath}.${key}` : key;
// Add this field path
fieldNames.add(fieldPath);
// Recursively process nested objects
if (typeof value === 'object' && value !== null) {
extractFieldNamesFromJson(value, {
maxDepth,
currentDepth: currentDepth + 1,
currentPath: fieldPath,
fieldNames
});
}
}
}
return Array.from(fieldNames);
}
/**
* Maps field names to standard field names based on common patterns
* @param fieldName The field name to map
* @returns The standard field name if a match is found, or null if no match
*/
export function mapToStandardField(fieldName) {
// Standard field mappings
const standardMappings = {
'title': ['title', 'name', 'headline', 'subject'],
'description': ['description', 'summary', 'content', 'text', 'body'],
'author': ['author', 'creator', 'user', 'owner', 'by'],
'date': ['date', 'created', 'createdAt', 'timestamp', 'published'],
'url': ['url', 'link', 'href', 'source'],
'image': ['image', 'thumbnail', 'photo', 'picture'],
'tags': ['tags', 'categories', 'keywords', 'topics']
};
// Check for matches
for (const [standardField, possibleMatches] of Object.entries(standardMappings)) {
// Exact match
if (possibleMatches.includes(fieldName)) {
return standardField;
}
// Path match (e.g., "user.name" matches "name")
const parts = fieldName.split('.');
const lastPart = parts[parts.length - 1];
if (possibleMatches.includes(lastPart)) {
return standardField;
}
// Array match (e.g., "items[0].name" matches "name")
if (fieldName.includes('[')) {
for (const part of parts) {
const cleanPart = part.split('[')[0];
if (possibleMatches.includes(cleanPart)) {
return standardField;
}
}
}
}
return null;
}
//# sourceMappingURL=fieldNameTracking.js.map

1
dist/utils/fieldNameTracking.js.map vendored Normal file
View file

@ -0,0 +1 @@
{"version":3,"file":"fieldNameTracking.js","sourceRoot":"","sources":["../../src/utils/fieldNameTracking.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH;;;;;GAKG;AACH,MAAM,UAAU,yBAAyB,CACvC,UAAe,EACf,UAKI,EAAE;IAEN,MAAM,EACJ,QAAQ,GAAG,CAAC,EACZ,YAAY,GAAG,CAAC,EAChB,WAAW,GAAG,EAAE,EAChB,UAAU,GAAG,IAAI,GAAG,EAAU,EAC/B,GAAG,OAAO,CAAA;IAEX,IACE,UAAU,KAAK,IAAI;QACnB,UAAU,KAAK,SAAS;QACxB,OAAO,UAAU,KAAK,QAAQ;QAC9B,YAAY,IAAI,QAAQ,EACxB,CAAC;QACD,OAAO,KAAK,CAAC,IAAI,CAAC,UAAU,CAAC,CAAA;IAC/B,CAAC;IAED,IAAI,KAAK,CAAC,OAAO,CAAC,UAAU,CAAC,EAAE,CAAC;QAC9B,0EAA0E;QAC1E,IAAI,UAAU,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC1B,MAAM,SAAS,GAAG,WAAW,CAAC,CAAC,CAAC,GAAG,WAAW,KAAK,CAAC,CAAC,CAAC,KAAK,CAAA;YAC3D,yBAAyB,CAAC,UAAU,CAAC,CAAC,CAAC,EAAE;gBACvC,QAAQ;gBACR,YAAY,EAAE,YAAY,GAAG,CAAC;gBAC9B,WAAW,EAAE,SAAS;gBACtB,UAAU;aACX,CAAC,CAAA;QACJ,CAAC;IACH,CAAC;SAAM,CAAC;QACN,qCAAqC;QACrC,KAAK,MAAM,GAAG,IAAI,MAAM,CAAC,IAAI,CAAC,UAAU,CAAC,EAAE,CAAC;YAC1C,MAAM,KAAK,GAAG,UAAU,CAAC,GAAG,CAAC,CAAA;YAC7B,MAAM,SAAS,GAAG,WAAW,CAAC,CAAC,CAAC,GAAG,WAAW,IAAI,GAAG,EAAE,CAAC,CAAC,CAAC,GAAG,CAAA;YAE7D,sBAAsB;YACtB,UAAU,CAAC,GAAG,CAAC,SAAS,CAAC,CAAA;YAEzB,qCAAqC;YACrC,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,KAAK,IAAI,EAAE,CAAC;gBAChD,yBAAyB,CAAC,KAAK,EAAE;oBAC/B,QAAQ;oBACR,YAAY,EAAE,YAAY,GAAG,CAAC;oBAC9B,WAAW,EAAE,SAAS;oBACtB,UAAU;iBACX,CAAC,CAAA;YACJ,CAAC;QACH,CAAC;IACH,CAAC;IAED,OAAO,KAAK,CAAC,IAAI,CAAC,UAAU,CAAC,CAAA;AAC/B,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,kBAAkB,CAAC,SAAiB;IAClD,0BAA0B;IAC1B,MAAM,gBAAgB,GAA6B;QACjD,OAAO,EAAE,CAAC,OAAO,EAAE,MAAM,EAAE,UAAU,EAAE,SAAS,CAAC;QACjD,aAAa,EAAE,CAAC,aAAa,EAAE,SAAS,EAAE,SAAS,EAAE,MAAM,EAAE,MAAM,CAAC;QACpE,QAAQ,EAAE,CAAC,QAAQ,EAAE,SAAS,EAAE,MAAM,EAAE,OAAO,EAAE,IAAI,CAAC;QACtD,MAAM,EAAE,CAAC,MAAM,EAAE,SAAS,EAAE,WAAW,EAAE,WAAW,EAAE,WAAW,CAAC;QAClE,KAAK,EAAE,CAAC,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,QAAQ,CAAC;QACxC,OAAO,EAAE,CAAC,OAAO,EAAE,WAAW,EAAE,OAAO,EAAE,SAAS,CAAC;QACnD,MAAM,EAAE,CAAC,MAAM,EAAE,YAAY,EAAE,UAAU,EAAE,QAAQ,CAAC;KACrD,CAAA;IAED,oBAAoB;IACpB,KAAK,MAAM,CAAC,aAAa,EAAE,eAAe,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,gBAAgB,CAAC,EAAE,CAAC;QAChF,cAAc;QACd,IAAI,eAAe,CAAC,QAAQ,CAAC,SAAS,CAAC,EAAE,CAAC;YACxC,OAAO,aAAa,CAAA;QACtB,CAAC;QAED,gDAAgD;QAChD,MAAM,KAAK,GAAG,SAAS,CAAC,KAAK,CAAC,GAAG,CAAC,CAAA;QAClC,MAAM,QAAQ,GAAG,KAAK,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAA;QACxC,IAAI,eAAe,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAAE,CAAC;YACvC,OAAO,aAAa,CAAA;QACtB,CAAC;QAED,qDAAqD;QACrD,IAAI,SAAS,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;YAC5B,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;gBACzB,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAA;gBACpC,IAAI,eAAe,CAAC,QAAQ,CAAC,SAAS,CAAC,EAAE,CAAC;oBACxC,OAAO,aAAa,CAAA;gBACtB,CAAC;YACH,CAAC;QACH,CAAC;IACH,CAAC;IAED,OAAO,IAAI,CAAA;AACb,CAAC"}

7
dist/utils/index.d.ts vendored Normal file
View file

@ -0,0 +1,7 @@
export * from './distance.js';
export * from './embedding.js';
export * from './workerUtils.js';
export * from './statistics.js';
export * from './jsonProcessing.js';
export * from './fieldNameTracking.js';
export * from './version.js';

8
dist/utils/index.js vendored Normal file
View file

@ -0,0 +1,8 @@
export * from './distance.js';
export * from './embedding.js';
export * from './workerUtils.js';
export * from './statistics.js';
export * from './jsonProcessing.js';
export * from './fieldNameTracking.js';
export * from './version.js';
//# sourceMappingURL=index.js.map

1
dist/utils/index.js.map vendored Normal file
View file

@ -0,0 +1 @@
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/utils/index.ts"],"names":[],"mappings":"AAAA,cAAc,eAAe,CAAA;AAC7B,cAAc,gBAAgB,CAAA;AAC9B,cAAc,kBAAkB,CAAA;AAChC,cAAc,iBAAiB,CAAA;AAC/B,cAAc,qBAAqB,CAAA;AACnC,cAAc,wBAAwB,CAAA;AACtC,cAAc,cAAc,CAAA"}

43
dist/utils/jsonProcessing.d.ts vendored Normal file
View file

@ -0,0 +1,43 @@
/**
* Utility functions for processing JSON documents for vectorization and search
*/
/**
* Extracts text from a JSON object for vectorization
* This function recursively processes the JSON object and extracts text from all fields
* It can also prioritize specific fields if provided
*
* @param jsonObject The JSON object to extract text from
* @param options Configuration options for text extraction
* @returns A string containing the extracted text
*/
export declare function extractTextFromJson(jsonObject: any, options?: {
priorityFields?: string[];
excludeFields?: string[];
includeFieldNames?: boolean;
maxDepth?: number;
currentDepth?: number;
fieldPath?: string[];
}): string;
/**
* Prepares a JSON document for vectorization
* This function extracts text from the JSON document and formats it for optimal vectorization
*
* @param jsonDocument The JSON document to prepare
* @param options Configuration options for preparation
* @returns A string ready for vectorization
*/
export declare function prepareJsonForVectorization(jsonDocument: any, options?: {
priorityFields?: string[];
excludeFields?: string[];
includeFieldNames?: boolean;
maxDepth?: number;
}): string;
/**
* Extracts text from a specific field in a JSON document
* This is useful for searching within specific fields
*
* @param jsonDocument The JSON document to extract from
* @param fieldPath The path to the field (e.g., "user.name" or "addresses[0].city")
* @returns The extracted text or empty string if field not found
*/
export declare function extractFieldFromJson(jsonDocument: any, fieldPath: string): string;

179
dist/utils/jsonProcessing.js vendored Normal file
View file

@ -0,0 +1,179 @@
/**
* Utility functions for processing JSON documents for vectorization and search
*/
/**
* Extracts text from a JSON object for vectorization
* This function recursively processes the JSON object and extracts text from all fields
* It can also prioritize specific fields if provided
*
* @param jsonObject The JSON object to extract text from
* @param options Configuration options for text extraction
* @returns A string containing the extracted text
*/
export function extractTextFromJson(jsonObject, options = {}) {
// Set default options
const { priorityFields = [], excludeFields = [], includeFieldNames = true, maxDepth = 5, currentDepth = 0, fieldPath = [] } = options;
// If input is not an object or array, or we've reached max depth, return as string
if (jsonObject === null ||
jsonObject === undefined ||
typeof jsonObject !== 'object' ||
currentDepth >= maxDepth) {
return String(jsonObject || '');
}
const extractedText = [];
const priorityText = [];
// Process arrays
if (Array.isArray(jsonObject)) {
for (let i = 0; i < jsonObject.length; i++) {
const value = jsonObject[i];
const newPath = [...fieldPath, i.toString()];
// Recursively extract text from array items
const itemText = extractTextFromJson(value, {
priorityFields,
excludeFields,
includeFieldNames,
maxDepth,
currentDepth: currentDepth + 1,
fieldPath: newPath
});
if (itemText) {
extractedText.push(itemText);
}
}
}
// Process objects
else {
for (const [key, value] of Object.entries(jsonObject)) {
// Skip excluded fields
if (excludeFields.includes(key)) {
continue;
}
const newPath = [...fieldPath, key];
const fullPath = newPath.join('.');
// Check if this is a priority field
const isPriority = priorityFields.some(field => {
// Exact match
if (field === key)
return true;
// Path match
if (field === fullPath)
return true;
// Wildcard match (e.g., "user.*" matches "user.name", "user.email", etc.)
if (field.endsWith('.*') && fullPath.startsWith(field.slice(0, -2)))
return true;
return false;
});
// Get the field value as text
let fieldText;
if (typeof value === 'object' && value !== null) {
// Recursively extract text from nested objects
fieldText = extractTextFromJson(value, {
priorityFields,
excludeFields,
includeFieldNames,
maxDepth,
currentDepth: currentDepth + 1,
fieldPath: newPath
});
}
else {
fieldText = String(value || '');
}
// Add field name if requested
if (includeFieldNames && fieldText) {
fieldText = `${key}: ${fieldText}`;
}
// Add to appropriate collection
if (fieldText) {
if (isPriority) {
priorityText.push(fieldText);
}
else {
extractedText.push(fieldText);
}
}
}
}
// Combine priority text (repeated for emphasis) and regular text
return [...priorityText, ...priorityText, ...extractedText].join(' ');
}
/**
* Prepares a JSON document for vectorization
* This function extracts text from the JSON document and formats it for optimal vectorization
*
* @param jsonDocument The JSON document to prepare
* @param options Configuration options for preparation
* @returns A string ready for vectorization
*/
export function prepareJsonForVectorization(jsonDocument, options = {}) {
// If input is a string, try to parse it as JSON
let document = jsonDocument;
if (typeof jsonDocument === 'string') {
try {
document = JSON.parse(jsonDocument);
}
catch (e) {
// If parsing fails, treat it as a plain string
return jsonDocument;
}
}
// If not an object after parsing, return as is
if (typeof document !== 'object' || document === null) {
return String(document || '');
}
// Extract text from the document
return extractTextFromJson(document, options);
}
/**
* Extracts text from a specific field in a JSON document
* This is useful for searching within specific fields
*
* @param jsonDocument The JSON document to extract from
* @param fieldPath The path to the field (e.g., "user.name" or "addresses[0].city")
* @returns The extracted text or empty string if field not found
*/
export function extractFieldFromJson(jsonDocument, fieldPath) {
// If input is a string, try to parse it as JSON
let document = jsonDocument;
if (typeof jsonDocument === 'string') {
try {
document = JSON.parse(jsonDocument);
}
catch (e) {
// If parsing fails, return empty string
return '';
}
}
// If not an object after parsing, return empty string
if (typeof document !== 'object' || document === null) {
return '';
}
// Parse the field path
const parts = fieldPath.split('.');
let current = document;
// Navigate through the path
for (const part of parts) {
// Handle array indexing (e.g., "addresses[0]")
const match = part.match(/^([^[]+)(?:\[(\d+)\])?$/);
if (!match) {
return '';
}
const [, key, indexStr] = match;
// Move to the next level
current = current[key];
// If we have an array index, access that element
if (indexStr !== undefined && Array.isArray(current)) {
const index = parseInt(indexStr, 10);
current = current[index];
}
// If we've reached a null or undefined value, return empty string
if (current === null || current === undefined) {
return '';
}
}
// Convert the final value to string
return typeof current === 'object'
? JSON.stringify(current)
: String(current);
}
//# sourceMappingURL=jsonProcessing.js.map

1
dist/utils/jsonProcessing.js.map vendored Normal file
View file

@ -0,0 +1 @@
{"version":3,"file":"jsonProcessing.js","sourceRoot":"","sources":["../../src/utils/jsonProcessing.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH;;;;;;;;GAQG;AACH,MAAM,UAAU,mBAAmB,CACjC,UAAe,EACf,UAOI,EAAE;IAEN,sBAAsB;IACtB,MAAM,EACJ,cAAc,GAAG,EAAE,EACnB,aAAa,GAAG,EAAE,EAClB,iBAAiB,GAAG,IAAI,EACxB,QAAQ,GAAG,CAAC,EACZ,YAAY,GAAG,CAAC,EAChB,SAAS,GAAG,EAAE,EACf,GAAG,OAAO,CAAA;IAEX,mFAAmF;IACnF,IACE,UAAU,KAAK,IAAI;QACnB,UAAU,KAAK,SAAS;QACxB,OAAO,UAAU,KAAK,QAAQ;QAC9B,YAAY,IAAI,QAAQ,EACxB,CAAC;QACD,OAAO,MAAM,CAAC,UAAU,IAAI,EAAE,CAAC,CAAA;IACjC,CAAC;IAED,MAAM,aAAa,GAAa,EAAE,CAAA;IAClC,MAAM,YAAY,GAAa,EAAE,CAAA;IAEjC,iBAAiB;IACjB,IAAI,KAAK,CAAC,OAAO,CAAC,UAAU,CAAC,EAAE,CAAC;QAC9B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,UAAU,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YAC3C,MAAM,KAAK,GAAG,UAAU,CAAC,CAAC,CAAC,CAAA;YAC3B,MAAM,OAAO,GAAG,CAAC,GAAG,SAAS,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAA;YAE5C,4CAA4C;YAC5C,MAAM,QAAQ,GAAG,mBAAmB,CAAC,KAAK,EAAE;gBAC1C,cAAc;gBACd,aAAa;gBACb,iBAAiB;gBACjB,QAAQ;gBACR,YAAY,EAAE,YAAY,GAAG,CAAC;gBAC9B,SAAS,EAAE,OAAO;aACnB,CAAC,CAAA;YAEF,IAAI,QAAQ,EAAE,CAAC;gBACb,aAAa,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAA;YAC9B,CAAC;QACH,CAAC;IACH,CAAC;IACD,kBAAkB;SACb,CAAC;QACJ,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,UAAU,CAAC,EAAE,CAAC;YACtD,uBAAuB;YACvB,IAAI,aAAa,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;gBAChC,SAAQ;YACV,CAAC;YAED,MAAM,OAAO,GAAG,CAAC,GAAG,SAAS,EAAE,GAAG,CAAC,CAAA;YACnC,MAAM,QAAQ,GAAG,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;YAElC,oCAAoC;YACpC,MAAM,UAAU,GAAG,cAAc,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE;gBAC7C,cAAc;gBACd,IAAI,KAAK,KAAK,GAAG;oBAAE,OAAO,IAAI,CAAA;gBAC9B,aAAa;gBACb,IAAI,KAAK,KAAK,QAAQ;oBAAE,OAAO,IAAI,CAAA;gBACnC,0EAA0E;gBAC1E,IAAI,KAAK,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,QAAQ,CAAC,UAAU,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;oBAAE,OAAO,IAAI,CAAA;gBAChF,OAAO,KAAK,CAAA;YACd,CAAC,CAAC,CAAA;YAEF,8BAA8B;YAC9B,IAAI,SAAiB,CAAA;YAErB,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,KAAK,IAAI,EAAE,CAAC;gBAChD,+CAA+C;gBAC/C,SAAS,GAAG,mBAAmB,CAAC,KAAK,EAAE;oBACrC,cAAc;oBACd,aAAa;oBACb,iBAAiB;oBACjB,QAAQ;oBACR,YAAY,EAAE,YAAY,GAAG,CAAC;oBAC9B,SAAS,EAAE,OAAO;iBACnB,CAAC,CAAA;YACJ,CAAC;iBAAM,CAAC;gBACN,SAAS,GAAG,MAAM,CAAC,KAAK,IAAI,EAAE,CAAC,CAAA;YACjC,CAAC;YAED,8BAA8B;YAC9B,IAAI,iBAAiB,IAAI,SAAS,EAAE,CAAC;gBACnC,SAAS,GAAG,GAAG,GAAG,KAAK,SAAS,EAAE,CAAA;YACpC,CAAC;YAED,gCAAgC;YAChC,IAAI,SAAS,EAAE,CAAC;gBACd,IAAI,UAAU,EAAE,CAAC;oBACf,YAAY,CAAC,IAAI,CAAC,SAAS,CAAC,CAAA;gBAC9B,CAAC;qBAAM,CAAC;oBACN,aAAa,CAAC,IAAI,CAAC,SAAS,CAAC,CAAA;gBAC/B,CAAC;YACH,CAAC;QACH,CAAC;IACH,CAAC;IAED,iEAAiE;IACjE,OAAO,CAAC,GAAG,YAAY,EAAE,GAAG,YAAY,EAAE,GAAG,aAAa,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;AACvE,CAAC;AAED;;;;;;;GAOG;AACH,MAAM,UAAU,2BAA2B,CACzC,YAAiB,EACjB,UAKI,EAAE;IAEN,gDAAgD;IAChD,IAAI,QAAQ,GAAG,YAAY,CAAA;IAC3B,IAAI,OAAO,YAAY,KAAK,QAAQ,EAAE,CAAC;QACrC,IAAI,CAAC;YACH,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,CAAA;QACrC,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,+CAA+C;YAC/C,OAAO,YAAY,CAAA;QACrB,CAAC;IACH,CAAC;IAED,+CAA+C;IAC/C,IAAI,OAAO,QAAQ,KAAK,QAAQ,IAAI,QAAQ,KAAK,IAAI,EAAE,CAAC;QACtD,OAAO,MAAM,CAAC,QAAQ,IAAI,EAAE,CAAC,CAAA;IAC/B,CAAC;IAED,iCAAiC;IACjC,OAAO,mBAAmB,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAA;AAC/C,CAAC;AAED;;;;;;;GAOG;AACH,MAAM,UAAU,oBAAoB,CAClC,YAAiB,EACjB,SAAiB;IAEjB,gDAAgD;IAChD,IAAI,QAAQ,GAAG,YAAY,CAAA;IAC3B,IAAI,OAAO,YAAY,KAAK,QAAQ,EAAE,CAAC;QACrC,IAAI,CAAC;YACH,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,CAAA;QACrC,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,wCAAwC;YACxC,OAAO,EAAE,CAAA;QACX,CAAC;IACH,CAAC;IAED,sDAAsD;IACtD,IAAI,OAAO,QAAQ,KAAK,QAAQ,IAAI,QAAQ,KAAK,IAAI,EAAE,CAAC;QACtD,OAAO,EAAE,CAAA;IACX,CAAC;IAED,uBAAuB;IACvB,MAAM,KAAK,GAAG,SAAS,CAAC,KAAK,CAAC,GAAG,CAAC,CAAA;IAClC,IAAI,OAAO,GAAG,QAAQ,CAAA;IAEtB,4BAA4B;IAC5B,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,+CAA+C;QAC/C,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,yBAAyB,CAAC,CAAA;QACnD,IAAI,CAAC,KAAK,EAAE,CAAC;YACX,OAAO,EAAE,CAAA;QACX,CAAC;QAED,MAAM,CAAC,EAAE,GAAG,EAAE,QAAQ,CAAC,GAAG,KAAK,CAAA;QAE/B,yBAAyB;QACzB,OAAO,GAAG,OAAO,CAAC,GAAG,CAAC,CAAA;QAEtB,iDAAiD;QACjD,IAAI,QAAQ,KAAK,SAAS,IAAI,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC;YACrD,MAAM,KAAK,GAAG,QAAQ,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAA;YACpC,OAAO,GAAG,OAAO,CAAC,KAAK,CAAC,CAAA;QAC1B,CAAC;QAED,kEAAkE;QAClE,IAAI,OAAO,KAAK,IAAI,IAAI,OAAO,KAAK,SAAS,EAAE,CAAC;YAC9C,OAAO,EAAE,CAAA;QACX,CAAC;IACH,CAAC;IAED,oCAAoC;IACpC,OAAO,OAAO,OAAO,KAAK,QAAQ;QAChC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC;QACzB,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAA;AACrB,CAAC"}

79
dist/utils/logger.d.ts vendored Normal file
View file

@ -0,0 +1,79 @@
/**
* Centralized logging utility for Brainy
* Provides configurable log levels and consistent logging across the codebase
* Automatically reduces logging in production environments to minimize costs
*/
export declare enum LogLevel {
ERROR = 0,
WARN = 1,
INFO = 2,
DEBUG = 3,
TRACE = 4
}
export interface LoggerConfig {
level: LogLevel;
modules?: {
[moduleName: string]: LogLevel;
};
timestamps?: boolean;
includeModule?: boolean;
handler?: (level: LogLevel, module: string, message: string, ...args: any[]) => void;
}
declare class Logger {
private static instance;
private config;
private constructor();
private applyEnvironmentDefaults;
static getInstance(): Logger;
configure(config: Partial<LoggerConfig>): void;
private shouldLog;
private formatMessage;
private log;
error(module: string, message: string, ...args: any[]): void;
warn(module: string, message: string, ...args: any[]): void;
info(module: string, message: string, ...args: any[]): void;
debug(module: string, message: string, ...args: any[]): void;
trace(module: string, message: string, ...args: any[]): void;
createModuleLogger(module: string): {
error: (message: string, ...args: any[]) => void;
warn: (message: string, ...args: any[]) => void;
info: (message: string, ...args: any[]) => void;
debug: (message: string, ...args: any[]) => void;
trace: (message: string, ...args: any[]) => void;
};
}
export declare const logger: Logger;
export declare function createModuleLogger(module: string): {
error: (message: string, ...args: any[]) => void;
warn: (message: string, ...args: any[]) => void;
info: (message: string, ...args: any[]) => void;
debug: (message: string, ...args: any[]) => void;
trace: (message: string, ...args: any[]) => void;
};
export declare function configureLogger(config: Partial<LoggerConfig>): void;
/**
* Smart console replacement that automatically reduces logging in production
* Dramatically reduces Google Cloud Run logging costs
*
* Usage: Replace console.log with smartConsole.log, etc.
*/
export declare const smartConsole: {
log: (message?: any, ...args: any[]) => void;
info: (message?: any, ...args: any[]) => void;
warn: (message?: any, ...args: any[]) => void;
error: (message?: any, ...args: any[]) => void;
debug: (message?: any, ...args: any[]) => void;
trace: (message?: any, ...args: any[]) => void;
};
/**
* Production-optimized logging functions
* These only log in non-production environments or when explicitly enabled
*/
export declare const prodLog: {
error: (message?: any, ...args: any[]) => void;
warn: (message?: any, ...args: any[]) => void;
info: (message?: any, ...args: any[]) => void;
debug: (message?: any, ...args: any[]) => void;
log: (message?: any, ...args: any[]) => void;
};
export {};

217
dist/utils/logger.js vendored Normal file
View file

@ -0,0 +1,217 @@
/**
* Centralized logging utility for Brainy
* Provides configurable log levels and consistent logging across the codebase
* Automatically reduces logging in production environments to minimize costs
*/
import { isProductionEnvironment, getLogLevel } from './environment.js';
export var LogLevel;
(function (LogLevel) {
LogLevel[LogLevel["ERROR"] = 0] = "ERROR";
LogLevel[LogLevel["WARN"] = 1] = "WARN";
LogLevel[LogLevel["INFO"] = 2] = "INFO";
LogLevel[LogLevel["DEBUG"] = 3] = "DEBUG";
LogLevel[LogLevel["TRACE"] = 4] = "TRACE";
})(LogLevel || (LogLevel = {}));
class Logger {
constructor() {
this.config = {
level: LogLevel.ERROR, // Default to ERROR in production for cost optimization
timestamps: false, // Disable timestamps in production to reduce log size
includeModule: true
};
// Auto-detect production environment and set appropriate defaults
this.applyEnvironmentDefaults();
// Set log level from environment variable if available (overrides auto-detection)
const envLogLevel = process.env.BRAINY_LOG_LEVEL;
if (envLogLevel) {
const level = LogLevel[envLogLevel.toUpperCase()];
if (level !== undefined) {
this.config.level = level;
}
}
// Parse module-specific log levels
const moduleLogLevels = process.env.BRAINY_MODULE_LOG_LEVELS;
if (moduleLogLevels) {
try {
this.config.modules = JSON.parse(moduleLogLevels);
}
catch (e) {
// Ignore parsing errors
}
}
}
applyEnvironmentDefaults() {
const envLogLevel = getLogLevel();
// Convert environment log level to Logger LogLevel
switch (envLogLevel) {
case 'silent':
this.config.level = -1; // Below ERROR to silence all logs
break;
case 'error':
this.config.level = LogLevel.ERROR;
this.config.timestamps = false; // Minimize log size in production
break;
case 'warn':
this.config.level = LogLevel.WARN;
this.config.timestamps = false;
break;
case 'info':
this.config.level = LogLevel.INFO;
this.config.timestamps = true;
break;
case 'verbose':
this.config.level = LogLevel.DEBUG;
this.config.timestamps = true;
break;
}
// In production environments, be extra conservative to minimize costs
if (isProductionEnvironment()) {
this.config.level = Math.min(this.config.level, LogLevel.ERROR);
this.config.timestamps = false;
this.config.includeModule = false; // Reduce log size
}
}
static getInstance() {
if (!Logger.instance) {
Logger.instance = new Logger();
}
return Logger.instance;
}
configure(config) {
this.config = { ...this.config, ...config };
}
shouldLog(level, module) {
// Check module-specific level first
if (this.config.modules && this.config.modules[module] !== undefined) {
return level <= this.config.modules[module];
}
// Otherwise use global level
return level <= this.config.level;
}
formatMessage(level, module, message) {
const parts = [];
if (this.config.timestamps) {
parts.push(`[${new Date().toISOString()}]`);
}
parts.push(`[${LogLevel[level]}]`);
if (this.config.includeModule) {
parts.push(`[${module}]`);
}
parts.push(message);
return parts.join(' ');
}
log(level, module, message, ...args) {
if (!this.shouldLog(level, module)) {
return;
}
if (this.config.handler) {
this.config.handler(level, module, message, ...args);
return;
}
const formattedMessage = this.formatMessage(level, module, message);
switch (level) {
case LogLevel.ERROR:
console.error(formattedMessage, ...args);
break;
case LogLevel.WARN:
console.warn(formattedMessage, ...args);
break;
case LogLevel.INFO:
console.info(formattedMessage, ...args);
break;
case LogLevel.DEBUG:
case LogLevel.TRACE:
console.log(formattedMessage, ...args);
break;
}
}
error(module, message, ...args) {
this.log(LogLevel.ERROR, module, message, ...args);
}
warn(module, message, ...args) {
this.log(LogLevel.WARN, module, message, ...args);
}
info(module, message, ...args) {
this.log(LogLevel.INFO, module, message, ...args);
}
debug(module, message, ...args) {
this.log(LogLevel.DEBUG, module, message, ...args);
}
trace(module, message, ...args) {
this.log(LogLevel.TRACE, module, message, ...args);
}
// Create a module-specific logger
createModuleLogger(module) {
return {
error: (message, ...args) => this.error(module, message, ...args),
warn: (message, ...args) => this.warn(module, message, ...args),
info: (message, ...args) => this.info(module, message, ...args),
debug: (message, ...args) => this.debug(module, message, ...args),
trace: (message, ...args) => this.trace(module, message, ...args)
};
}
}
// Export singleton instance
export const logger = Logger.getInstance();
// Export convenience function for creating module loggers
export function createModuleLogger(module) {
return logger.createModuleLogger(module);
}
// Export function to configure logger
export function configureLogger(config) {
logger.configure(config);
}
/**
* Smart console replacement that automatically reduces logging in production
* Dramatically reduces Google Cloud Run logging costs
*
* Usage: Replace console.log with smartConsole.log, etc.
*/
export const smartConsole = {
log: (message, ...args) => {
if (logger['shouldLog'](LogLevel.INFO, 'console')) {
console.log(message, ...args);
}
},
info: (message, ...args) => {
if (logger['shouldLog'](LogLevel.INFO, 'console')) {
console.info(message, ...args);
}
},
warn: (message, ...args) => {
if (logger['shouldLog'](LogLevel.WARN, 'console')) {
console.warn(message, ...args);
}
},
error: (message, ...args) => {
if (logger['shouldLog'](LogLevel.ERROR, 'console')) {
console.error(message, ...args);
}
},
debug: (message, ...args) => {
if (logger['shouldLog'](LogLevel.DEBUG, 'console')) {
console.debug(message, ...args);
}
},
trace: (message, ...args) => {
if (logger['shouldLog'](LogLevel.TRACE, 'console')) {
console.trace(message, ...args);
}
}
};
/**
* Production-optimized logging functions
* These only log in non-production environments or when explicitly enabled
*/
export const prodLog = {
// Only log errors in production (always visible)
error: (message, ...args) => {
console.error(message, ...args);
},
// These are suppressed in production unless BRAINY_LOG_LEVEL is set
warn: (message, ...args) => smartConsole.warn(message, ...args),
info: (message, ...args) => smartConsole.info(message, ...args),
debug: (message, ...args) => smartConsole.debug(message, ...args),
log: (message, ...args) => smartConsole.log(message, ...args)
};
//# sourceMappingURL=logger.js.map

1
dist/utils/logger.js.map vendored Normal file
View file

File diff suppressed because one or more lines are too long

79
dist/utils/metadataFilter.d.ts vendored Normal file
View file

@ -0,0 +1,79 @@
/**
* Smart metadata filtering for vector search
* Filters DURING search to ensure relevant results
* Simple API that just works without configuration
*/
import { SearchResult, HNSWNoun } from '../coreTypes.js';
/**
* MongoDB-style query operators
*/
export interface QueryOperators {
$eq?: any;
$ne?: any;
$gt?: any;
$gte?: any;
$lt?: any;
$lte?: any;
$in?: any[];
$nin?: any[];
$exists?: boolean;
$regex?: string | RegExp;
$includes?: any;
$all?: any[];
$size?: number;
$and?: MetadataFilter[];
$or?: MetadataFilter[];
$not?: MetadataFilter;
}
/**
* Metadata filter definition
*/
export interface MetadataFilter {
[key: string]: any | QueryOperators;
}
/**
* Options for metadata filtering
*/
export interface MetadataFilterOptions {
metadata?: MetadataFilter;
scoring?: {
vectorWeight?: number;
metadataWeight?: number;
metadataBoosts?: Record<string, number | ((value: any, query: any) => number)>;
};
}
/**
* Check if metadata matches the filter
*/
export declare function matchesMetadataFilter(metadata: any, filter: MetadataFilter): boolean;
/**
* Calculate metadata boost score
*/
export declare function calculateMetadataScore(metadata: any, filter: MetadataFilter, scoring?: MetadataFilterOptions['scoring']): number;
/**
* Apply compound scoring to search results
*/
export declare function applyCompoundScoring<T>(results: SearchResult<T>[], filter: MetadataFilter, scoring?: MetadataFilterOptions['scoring']): SearchResult<T>[];
/**
* Filter search results by metadata
*/
export declare function filterSearchResultsByMetadata<T>(results: SearchResult<T>[], filter: MetadataFilter): SearchResult<T>[];
/**
* Filter nouns by metadata before search
*/
export declare function filterNounsByMetadata(nouns: HNSWNoun[], filter: MetadataFilter): HNSWNoun[];
/**
* Aggregate search results for faceted search
*/
export interface FacetConfig {
field: string;
limit?: number;
}
export interface FacetResult {
[value: string]: number;
}
export interface AggregationResult<T> {
results: SearchResult<T>[];
facets: Record<string, FacetResult>;
}
export declare function aggregateSearchResults<T>(results: SearchResult<T>[], facets: Record<string, FacetConfig>): AggregationResult<T>;

229
dist/utils/metadataFilter.js vendored Normal file
View file

@ -0,0 +1,229 @@
/**
* Smart metadata filtering for vector search
* Filters DURING search to ensure relevant results
* Simple API that just works without configuration
*/
/**
* Check if a value matches a query with operators
*/
function matchesQuery(value, query) {
// Direct equality check
if (typeof query !== 'object' || query === null || Array.isArray(query)) {
return value === query;
}
// Check for MongoDB-style operators
for (const [op, operand] of Object.entries(query)) {
switch (op) {
case '$eq':
if (value !== operand)
return false;
break;
case '$ne':
if (value === operand)
return false;
break;
case '$gt':
if (typeof value !== 'number' || typeof operand !== 'number' || !(value > operand))
return false;
break;
case '$gte':
if (typeof value !== 'number' || typeof operand !== 'number' || !(value >= operand))
return false;
break;
case '$lt':
if (typeof value !== 'number' || typeof operand !== 'number' || !(value < operand))
return false;
break;
case '$lte':
if (typeof value !== 'number' || typeof operand !== 'number' || !(value <= operand))
return false;
break;
case '$in':
if (!Array.isArray(operand) || !operand.includes(value))
return false;
break;
case '$nin':
if (!Array.isArray(operand) || operand.includes(value))
return false;
break;
case '$exists':
if ((value !== undefined) !== operand)
return false;
break;
case '$regex':
const regex = typeof operand === 'string' ? new RegExp(operand) : operand;
if (!(regex instanceof RegExp) || !regex.test(String(value)))
return false;
break;
case '$includes':
if (!Array.isArray(value) || !value.includes(operand))
return false;
break;
case '$all':
if (!Array.isArray(value) || !Array.isArray(operand))
return false;
for (const item of operand) {
if (!value.includes(item))
return false;
}
break;
case '$size':
if (!Array.isArray(value) || value.length !== operand)
return false;
break;
default:
// Unknown operator, treat as field name
if (!matchesFieldQuery(value, op, operand))
return false;
}
}
return true;
}
/**
* Check if a field matches a query
*/
function matchesFieldQuery(obj, field, query) {
const value = getNestedValue(obj, field);
return matchesQuery(value, query);
}
/**
* Get nested value from object using dot notation
*/
function getNestedValue(obj, path) {
const parts = path.split('.');
let current = obj;
for (const part of parts) {
if (current === null || current === undefined) {
return undefined;
}
current = current[part];
}
return current;
}
/**
* Check if metadata matches the filter
*/
export function matchesMetadataFilter(metadata, filter) {
if (!filter || Object.keys(filter).length === 0) {
return true;
}
for (const [key, query] of Object.entries(filter)) {
// Handle logical operators
if (key === '$and') {
if (!Array.isArray(query))
return false;
for (const subFilter of query) {
if (!matchesMetadataFilter(metadata, subFilter))
return false;
}
continue;
}
if (key === '$or') {
if (!Array.isArray(query))
return false;
let matched = false;
for (const subFilter of query) {
if (matchesMetadataFilter(metadata, subFilter)) {
matched = true;
break;
}
}
if (!matched)
return false;
continue;
}
if (key === '$not') {
if (matchesMetadataFilter(metadata, query))
return false;
continue;
}
// Handle field queries
const value = getNestedValue(metadata, key);
if (!matchesQuery(value, query)) {
return false;
}
}
return true;
}
/**
* Calculate metadata boost score
*/
export function calculateMetadataScore(metadata, filter, scoring) {
if (!scoring || !scoring.metadataBoosts) {
return 0;
}
let score = 0;
for (const [field, boost] of Object.entries(scoring.metadataBoosts)) {
const value = getNestedValue(metadata, field);
if (typeof boost === 'function') {
score += boost(value, filter);
}
else if (value !== undefined) {
// Check if the field matches the filter
const fieldFilter = filter[field];
if (fieldFilter && matchesQuery(value, fieldFilter)) {
score += boost;
}
}
}
return score;
}
/**
* Apply compound scoring to search results
*/
export function applyCompoundScoring(results, filter, scoring) {
if (!scoring || (!scoring.vectorWeight && !scoring.metadataWeight)) {
return results;
}
const vectorWeight = scoring.vectorWeight ?? 1.0;
const metadataWeight = scoring.metadataWeight ?? 0.0;
return results.map(result => {
const metadataScore = calculateMetadataScore(result.metadata, filter, scoring);
const combinedScore = (result.score * vectorWeight) + (metadataScore * metadataWeight);
return {
...result,
score: combinedScore
};
}).sort((a, b) => b.score - a.score); // Re-sort by combined score
}
/**
* Filter search results by metadata
*/
export function filterSearchResultsByMetadata(results, filter) {
if (!filter || Object.keys(filter).length === 0) {
return results;
}
return results.filter(result => matchesMetadataFilter(result.metadata, filter));
}
/**
* Filter nouns by metadata before search
*/
export function filterNounsByMetadata(nouns, filter) {
if (!filter || Object.keys(filter).length === 0) {
return nouns;
}
return nouns.filter(noun => matchesMetadataFilter(noun.metadata, filter));
}
export function aggregateSearchResults(results, facets) {
const facetResults = {};
for (const [facetName, config] of Object.entries(facets)) {
const counts = {};
for (const result of results) {
const value = getNestedValue(result.metadata, config.field);
if (value !== undefined) {
const key = String(value);
counts[key] = (counts[key] || 0) + 1;
}
}
// Sort by count and apply limit
const sorted = Object.entries(counts)
.sort((a, b) => b[1] - a[1])
.slice(0, config.limit || 10);
facetResults[facetName] = Object.fromEntries(sorted);
}
return {
results,
facets: facetResults
};
}
//# sourceMappingURL=metadataFilter.js.map

1
dist/utils/metadataFilter.js.map vendored Normal file
View file

File diff suppressed because one or more lines are too long

154
dist/utils/metadataIndex.d.ts vendored Normal file
View file

@ -0,0 +1,154 @@
/**
* Metadata Index System
* Maintains inverted indexes for fast metadata filtering
* Automatically updates indexes when data changes
*/
import { StorageAdapter } from '../coreTypes.js';
export interface MetadataIndexEntry {
field: string;
value: string | number | boolean;
ids: Set<string>;
lastUpdated: number;
}
export interface FieldIndexData {
values: Record<string, number>;
lastUpdated: number;
}
export interface MetadataIndexStats {
totalEntries: number;
totalIds: number;
fieldsIndexed: string[];
lastRebuild: number;
indexSize: number;
}
export interface MetadataIndexConfig {
maxIndexSize?: number;
rebuildThreshold?: number;
autoOptimize?: boolean;
indexedFields?: string[];
excludeFields?: string[];
}
/**
* Manages metadata indexes for fast filtering
* Maintains inverted indexes: field+value -> list of IDs
*/
export declare class MetadataIndexManager {
private storage;
private config;
private indexCache;
private dirtyEntries;
private isRebuilding;
private metadataCache;
private fieldIndexes;
private dirtyFields;
private lastFlushTime;
private autoFlushThreshold;
constructor(storage: StorageAdapter, config?: MetadataIndexConfig);
/**
* Get index key for field and value
*/
private getIndexKey;
/**
* Generate field index filename for filter discovery
*/
private getFieldIndexFilename;
/**
* Generate value chunk filename for scalable storage
*/
private getValueChunkFilename;
/**
* Make a value safe for use in filenames
*/
private makeSafeFilename;
/**
* Normalize value for consistent indexing
*/
private normalizeValue;
/**
* Create a short hash for long values to avoid filesystem filename limits
*/
private hashValue;
/**
* Check if field should be indexed
*/
private shouldIndexField;
/**
* Extract indexable field-value pairs from metadata
*/
private extractIndexableFields;
/**
* Add item to metadata indexes
*/
addToIndex(id: string, metadata: any, skipFlush?: boolean): Promise<void>;
/**
* Update field index with value count
*/
private updateFieldIndex;
/**
* Remove item from metadata indexes
*/
removeFromIndex(id: string, metadata?: any): Promise<void>;
/**
* Get IDs for a specific field-value combination with caching
*/
getIds(field: string, value: any): Promise<string[]>;
/**
* Get all available values for a field (for filter discovery)
*/
getFilterValues(field: string): Promise<string[]>;
/**
* Get all indexed fields (for filter discovery)
*/
getFilterFields(): Promise<string[]>;
/**
* Convert MongoDB-style filter to simple field-value criteria for indexing
*/
private convertFilterToCriteria;
/**
* Get IDs matching MongoDB-style metadata filter using indexes where possible
*/
getIdsForFilter(filter: any): Promise<string[]>;
/**
* Get IDs matching multiple criteria (intersection) - LEGACY METHOD
* @deprecated Use getIdsForFilter instead
*/
getIdsForCriteria(criteria: Record<string, any>): Promise<string[]>;
/**
* Flush dirty entries to storage (non-blocking version)
*/
flush(): Promise<void>;
/**
* Yield control back to the Node.js event loop
* Prevents blocking during long-running operations
*/
private yieldToEventLoop;
/**
* Load field index from storage
*/
private loadFieldIndex;
/**
* Save field index to storage
*/
private saveFieldIndex;
/**
* Get index statistics
*/
getStats(): Promise<MetadataIndexStats>;
/**
* Rebuild entire index from scratch using pagination
* Non-blocking version that yields control back to event loop
*/
rebuild(): Promise<void>;
/**
* Load index entry from storage using safe filenames
*/
private loadIndexEntry;
/**
* Save index entry to storage using safe filenames
*/
private saveIndexEntry;
/**
* Delete index entry from storage using safe filenames
*/
private deleteIndexEntry;
}

770
dist/utils/metadataIndex.js vendored Normal file
View file

@ -0,0 +1,770 @@
/**
* Metadata Index System
* Maintains inverted indexes for fast metadata filtering
* Automatically updates indexes when data changes
*/
import { MetadataIndexCache } from './metadataIndexCache.js';
import { prodLog } from './logger.js';
/**
* Manages metadata indexes for fast filtering
* Maintains inverted indexes: field+value -> list of IDs
*/
export class MetadataIndexManager {
constructor(storage, config = {}) {
this.indexCache = new Map();
this.dirtyEntries = new Set();
this.isRebuilding = false;
this.fieldIndexes = new Map();
this.dirtyFields = new Set();
this.lastFlushTime = Date.now();
this.autoFlushThreshold = 10; // Start with 10 for more frequent non-blocking flushes
this.storage = storage;
this.config = {
maxIndexSize: config.maxIndexSize ?? 10000,
rebuildThreshold: config.rebuildThreshold ?? 0.1,
autoOptimize: config.autoOptimize ?? true,
indexedFields: config.indexedFields ?? [],
excludeFields: config.excludeFields ?? ['id', 'createdAt', 'updatedAt', 'embedding', 'vector', 'embeddings', 'vectors']
};
// Initialize metadata cache with similar config to search cache
this.metadataCache = new MetadataIndexCache({
maxAge: 5 * 60 * 1000, // 5 minutes
maxSize: 500, // 500 entries (field indexes + value chunks)
enabled: true
});
}
/**
* Get index key for field and value
*/
getIndexKey(field, value) {
const normalizedValue = this.normalizeValue(value);
return `${field}:${normalizedValue}`;
}
/**
* Generate field index filename for filter discovery
*/
getFieldIndexFilename(field) {
return `field_${field}`;
}
/**
* Generate value chunk filename for scalable storage
*/
getValueChunkFilename(field, value, chunkIndex = 0) {
const normalizedValue = this.normalizeValue(value);
const safeValue = this.makeSafeFilename(normalizedValue);
return `${field}_${safeValue}_chunk${chunkIndex}`;
}
/**
* Make a value safe for use in filenames
*/
makeSafeFilename(value) {
// Replace unsafe characters and limit length
return value
.replace(/[^a-zA-Z0-9-_]/g, '_')
.substring(0, 50)
.toLowerCase();
}
/**
* Normalize value for consistent indexing
*/
normalizeValue(value) {
if (value === null || value === undefined)
return '__NULL__';
if (typeof value === 'boolean')
return value ? '__TRUE__' : '__FALSE__';
if (typeof value === 'number')
return value.toString();
if (Array.isArray(value)) {
const joined = value.map(v => this.normalizeValue(v)).join(',');
// Hash very long array values to avoid filesystem limits
if (joined.length > 100) {
return this.hashValue(joined);
}
return joined;
}
const stringValue = String(value).toLowerCase().trim();
// Hash very long string values to avoid filesystem limits
if (stringValue.length > 100) {
return this.hashValue(stringValue);
}
return stringValue;
}
/**
* Create a short hash for long values to avoid filesystem filename limits
*/
hashValue(value) {
// Simple hash function to create shorter keys
let hash = 0;
for (let i = 0; i < value.length; i++) {
const char = value.charCodeAt(i);
hash = ((hash << 5) - hash) + char;
hash = hash & hash; // Convert to 32-bit integer
}
return `__HASH_${Math.abs(hash).toString(36)}`;
}
/**
* Check if field should be indexed
*/
shouldIndexField(field) {
if (this.config.excludeFields.includes(field))
return false;
if (this.config.indexedFields.length > 0) {
return this.config.indexedFields.includes(field);
}
return true;
}
/**
* Extract indexable field-value pairs from metadata
*/
extractIndexableFields(metadata) {
const fields = [];
const extract = (obj, prefix = '') => {
for (const [key, value] of Object.entries(obj)) {
const fullKey = prefix ? `${prefix}.${key}` : key;
if (!this.shouldIndexField(fullKey))
continue;
if (value && typeof value === 'object' && !Array.isArray(value)) {
// Recurse into nested objects
extract(value, fullKey);
}
else {
// Index this field
fields.push({ field: fullKey, value });
// If it's an array, also index each element
if (Array.isArray(value)) {
for (const item of value) {
fields.push({ field: fullKey, value: item });
}
}
}
}
};
if (metadata && typeof metadata === 'object') {
extract(metadata);
}
return fields;
}
/**
* Add item to metadata indexes
*/
async addToIndex(id, metadata, skipFlush = false) {
const fields = this.extractIndexableFields(metadata);
for (let i = 0; i < fields.length; i++) {
const { field, value } = fields[i];
const key = this.getIndexKey(field, value);
// Get or create index entry
let entry = this.indexCache.get(key);
if (!entry) {
const loadedEntry = await this.loadIndexEntry(key);
entry = loadedEntry ?? {
field,
value: this.normalizeValue(value),
ids: new Set(),
lastUpdated: Date.now()
};
this.indexCache.set(key, entry);
}
// Add ID to entry
entry.ids.add(id);
entry.lastUpdated = Date.now();
this.dirtyEntries.add(key);
// Update field index
await this.updateFieldIndex(field, value, 1);
// Yield to event loop every 5 fields to prevent blocking
if (i % 5 === 4) {
await this.yieldToEventLoop();
}
}
// Adaptive auto-flush based on usage patterns
if (!skipFlush) {
const timeSinceLastFlush = Date.now() - this.lastFlushTime;
const shouldAutoFlush = this.dirtyEntries.size >= this.autoFlushThreshold || // Size threshold
(this.dirtyEntries.size > 10 && timeSinceLastFlush > 5000); // Time threshold (5 seconds)
if (shouldAutoFlush) {
const startTime = Date.now();
await this.flush();
const flushTime = Date.now() - startTime;
// Adapt threshold based on flush performance
if (flushTime < 50) {
// Fast flush, can handle more entries
this.autoFlushThreshold = Math.min(200, this.autoFlushThreshold * 1.2);
}
else if (flushTime > 200) {
// Slow flush, reduce batch size
this.autoFlushThreshold = Math.max(20, this.autoFlushThreshold * 0.8);
}
// Yield to event loop after flush to prevent blocking
await this.yieldToEventLoop();
}
}
// Invalidate cache for these fields
for (const { field } of fields) {
this.metadataCache.invalidatePattern(`field_values_${field}`);
}
}
/**
* Update field index with value count
*/
async updateFieldIndex(field, value, delta) {
let fieldIndex = this.fieldIndexes.get(field);
if (!fieldIndex) {
// Load from storage if not in memory
fieldIndex = await this.loadFieldIndex(field) ?? {
values: {},
lastUpdated: Date.now()
};
this.fieldIndexes.set(field, fieldIndex);
}
const normalizedValue = this.normalizeValue(value);
fieldIndex.values[normalizedValue] = (fieldIndex.values[normalizedValue] || 0) + delta;
// Remove if count drops to 0
if (fieldIndex.values[normalizedValue] <= 0) {
delete fieldIndex.values[normalizedValue];
}
fieldIndex.lastUpdated = Date.now();
this.dirtyFields.add(field);
}
/**
* Remove item from metadata indexes
*/
async removeFromIndex(id, metadata) {
if (metadata) {
// Remove from specific field indexes
const fields = this.extractIndexableFields(metadata);
for (const { field, value } of fields) {
const key = this.getIndexKey(field, value);
let entry = this.indexCache.get(key);
if (!entry) {
const loadedEntry = await this.loadIndexEntry(key);
entry = loadedEntry ?? undefined;
}
if (entry) {
entry.ids.delete(id);
entry.lastUpdated = Date.now();
this.dirtyEntries.add(key);
// Update field index
await this.updateFieldIndex(field, value, -1);
// If no IDs left, mark for cleanup
if (entry.ids.size === 0) {
this.indexCache.delete(key);
await this.deleteIndexEntry(key);
}
}
// Invalidate cache
this.metadataCache.invalidatePattern(`field_values_${field}`);
}
}
else {
// Remove from all indexes (slower, requires scanning)
for (const [key, entry] of this.indexCache.entries()) {
if (entry.ids.has(id)) {
entry.ids.delete(id);
entry.lastUpdated = Date.now();
this.dirtyEntries.add(key);
if (entry.ids.size === 0) {
this.indexCache.delete(key);
await this.deleteIndexEntry(key);
}
}
}
}
}
/**
* Get IDs for a specific field-value combination with caching
*/
async getIds(field, value) {
const key = this.getIndexKey(field, value);
// Check metadata cache first
const cacheKey = `ids_${key}`;
const cachedIds = this.metadataCache.get(cacheKey);
if (cachedIds) {
return cachedIds;
}
// Try in-memory cache
let entry = this.indexCache.get(key);
// Load from storage if not cached
if (!entry) {
const loadedEntry = await this.loadIndexEntry(key);
if (loadedEntry) {
entry = loadedEntry;
this.indexCache.set(key, entry);
}
}
const ids = entry ? Array.from(entry.ids) : [];
// Cache the result
this.metadataCache.set(cacheKey, ids);
return ids;
}
/**
* Get all available values for a field (for filter discovery)
*/
async getFilterValues(field) {
// Check cache first
const cacheKey = `field_values_${field}`;
const cachedValues = this.metadataCache.get(cacheKey);
if (cachedValues) {
return cachedValues;
}
// Check in-memory field indexes first
let fieldIndex = this.fieldIndexes.get(field);
// If not in memory, load from storage
if (!fieldIndex) {
const loaded = await this.loadFieldIndex(field);
if (loaded) {
fieldIndex = loaded;
this.fieldIndexes.set(field, loaded);
}
}
if (!fieldIndex) {
return [];
}
const values = Object.keys(fieldIndex.values);
// Cache the result
this.metadataCache.set(cacheKey, values);
return values;
}
/**
* Get all indexed fields (for filter discovery)
*/
async getFilterFields() {
// Check cache first
const cacheKey = 'all_filter_fields';
const cachedFields = this.metadataCache.get(cacheKey);
if (cachedFields) {
return cachedFields;
}
// Get fields from in-memory indexes and storage
const fields = new Set(this.fieldIndexes.keys());
// Also scan storage for persisted field indexes (in case not loaded)
// This would require a new storage method to list field indexes
// For now, just use in-memory fields
const fieldsArray = Array.from(fields);
// Cache the result
this.metadataCache.set(cacheKey, fieldsArray);
return fieldsArray;
}
/**
* Convert MongoDB-style filter to simple field-value criteria for indexing
*/
convertFilterToCriteria(filter) {
const criteria = [];
if (!filter || typeof filter !== 'object') {
return criteria;
}
for (const [key, value] of Object.entries(filter)) {
// Skip logical operators for now - handle them separately
if (key.startsWith('$'))
continue;
if (value && typeof value === 'object' && !Array.isArray(value)) {
// Handle MongoDB operators
for (const [op, operand] of Object.entries(value)) {
switch (op) {
case '$in':
if (Array.isArray(operand)) {
criteria.push({ field: key, values: operand });
}
break;
case '$eq':
criteria.push({ field: key, values: [operand] });
break;
case '$includes':
// For $includes, the operand is the value we're looking for in an array field
criteria.push({ field: key, values: [operand] });
break;
// For other operators, we can't use index efficiently, skip for now
default:
break;
}
}
}
else {
// Direct value or array
const values = Array.isArray(value) ? value : [value];
criteria.push({ field: key, values });
}
}
return criteria;
}
/**
* Get IDs matching MongoDB-style metadata filter using indexes where possible
*/
async getIdsForFilter(filter) {
if (!filter || Object.keys(filter).length === 0) {
return [];
}
// Handle logical operators
if (filter.$and && Array.isArray(filter.$and)) {
// For $and, we need intersection of all sub-filters
const allIds = [];
for (const subFilter of filter.$and) {
const subIds = await this.getIdsForFilter(subFilter);
allIds.push(subIds);
}
if (allIds.length === 0)
return [];
if (allIds.length === 1)
return allIds[0];
// Intersection of all sets
return allIds.reduce((intersection, currentSet) => intersection.filter(id => currentSet.includes(id)));
}
if (filter.$or && Array.isArray(filter.$or)) {
// For $or, we need union of all sub-filters
const unionIds = new Set();
for (const subFilter of filter.$or) {
const subIds = await this.getIdsForFilter(subFilter);
subIds.forEach(id => unionIds.add(id));
}
return Array.from(unionIds);
}
// Handle regular field filters
const criteria = this.convertFilterToCriteria(filter);
const idSets = [];
for (const { field, values } of criteria) {
const unionIds = new Set();
for (const value of values) {
const ids = await this.getIds(field, value);
ids.forEach(id => unionIds.add(id));
}
idSets.push(Array.from(unionIds));
}
if (idSets.length === 0)
return [];
if (idSets.length === 1)
return idSets[0];
// Intersection of all field criteria (implicit $and)
return idSets.reduce((intersection, currentSet) => intersection.filter(id => currentSet.includes(id)));
}
/**
* Get IDs matching multiple criteria (intersection) - LEGACY METHOD
* @deprecated Use getIdsForFilter instead
*/
async getIdsForCriteria(criteria) {
return this.getIdsForFilter(criteria);
}
/**
* Flush dirty entries to storage (non-blocking version)
*/
async flush() {
if (this.dirtyEntries.size === 0 && this.dirtyFields.size === 0) {
return; // Nothing to flush
}
// Process in smaller batches to avoid blocking
const BATCH_SIZE = 20;
const allPromises = [];
// Flush value entries in batches
const dirtyEntriesArray = Array.from(this.dirtyEntries);
for (let i = 0; i < dirtyEntriesArray.length; i += BATCH_SIZE) {
const batch = dirtyEntriesArray.slice(i, i + BATCH_SIZE);
const batchPromises = batch.map(key => {
const entry = this.indexCache.get(key);
return entry ? this.saveIndexEntry(key, entry) : Promise.resolve();
});
allPromises.push(...batchPromises);
// Yield to event loop between batches
if (i + BATCH_SIZE < dirtyEntriesArray.length) {
await this.yieldToEventLoop();
}
}
// Flush field indexes in batches
const dirtyFieldsArray = Array.from(this.dirtyFields);
for (let i = 0; i < dirtyFieldsArray.length; i += BATCH_SIZE) {
const batch = dirtyFieldsArray.slice(i, i + BATCH_SIZE);
const batchPromises = batch.map(field => {
const fieldIndex = this.fieldIndexes.get(field);
return fieldIndex ? this.saveFieldIndex(field, fieldIndex) : Promise.resolve();
});
allPromises.push(...batchPromises);
// Yield to event loop between batches
if (i + BATCH_SIZE < dirtyFieldsArray.length) {
await this.yieldToEventLoop();
}
}
// Wait for all operations to complete
await Promise.all(allPromises);
this.dirtyEntries.clear();
this.dirtyFields.clear();
this.lastFlushTime = Date.now();
}
/**
* Yield control back to the Node.js event loop
* Prevents blocking during long-running operations
*/
async yieldToEventLoop() {
return new Promise(resolve => setImmediate(resolve));
}
/**
* Load field index from storage
*/
async loadFieldIndex(field) {
try {
const filename = this.getFieldIndexFilename(field);
const cacheKey = `field_index_${filename}`;
// Check cache first
const cached = this.metadataCache.get(cacheKey);
if (cached) {
return cached;
}
// Load from storage
const indexId = `__metadata_field_index__${filename}`;
const data = await this.storage.getMetadata(indexId);
if (data) {
const fieldIndex = {
values: data.values || {},
lastUpdated: data.lastUpdated || Date.now()
};
// Cache it
this.metadataCache.set(cacheKey, fieldIndex);
return fieldIndex;
}
}
catch (error) {
// Field index doesn't exist yet
}
return null;
}
/**
* Save field index to storage
*/
async saveFieldIndex(field, fieldIndex) {
const filename = this.getFieldIndexFilename(field);
const indexId = `__metadata_field_index__${filename}`;
await this.storage.saveMetadata(indexId, {
values: fieldIndex.values,
lastUpdated: fieldIndex.lastUpdated
});
// Invalidate cache
this.metadataCache.invalidatePattern(`field_index_${filename}`);
}
/**
* Get index statistics
*/
async getStats() {
const fields = new Set();
let totalEntries = 0;
let totalIds = 0;
for (const entry of this.indexCache.values()) {
fields.add(entry.field);
totalEntries++;
totalIds += entry.ids.size;
}
return {
totalEntries,
totalIds,
fieldsIndexed: Array.from(fields),
lastRebuild: 0, // TODO: track rebuild timestamp
indexSize: totalEntries * 100 // rough estimate
};
}
/**
* Rebuild entire index from scratch using pagination
* Non-blocking version that yields control back to event loop
*/
async rebuild() {
if (this.isRebuilding)
return;
this.isRebuilding = true;
try {
prodLog.info('🔄 Starting non-blocking metadata index rebuild with batch processing to prevent socket exhaustion...');
prodLog.info(`📊 Storage adapter: ${this.storage.constructor.name}`);
prodLog.info(`🔧 Batch processing available: ${!!this.storage.getMetadataBatch}`);
// Clear existing indexes
this.indexCache.clear();
this.dirtyEntries.clear();
this.fieldIndexes.clear();
this.dirtyFields.clear();
// Rebuild noun metadata indexes using pagination
let nounOffset = 0;
const nounLimit = 25; // Even smaller batches during initialization to prevent socket exhaustion
let hasMoreNouns = true;
let totalNounsProcessed = 0;
while (hasMoreNouns) {
const result = await this.storage.getNouns({
pagination: { offset: nounOffset, limit: nounLimit }
});
// CRITICAL FIX: Use batch metadata reading to prevent socket exhaustion
const nounIds = result.items.map(noun => noun.id);
let metadataBatch;
if (this.storage.getMetadataBatch) {
// Use batch reading if available (prevents socket exhaustion)
prodLog.info(`📦 Processing metadata batch ${Math.floor(totalNounsProcessed / nounLimit) + 1} (${nounIds.length} items)...`);
metadataBatch = await this.storage.getMetadataBatch(nounIds);
const successRate = ((metadataBatch.size / nounIds.length) * 100).toFixed(1);
prodLog.info(`✅ Batch loaded ${metadataBatch.size}/${nounIds.length} metadata objects (${successRate}% success)`);
}
else {
// Fallback to individual calls with strict concurrency control
prodLog.warn(`⚠️ FALLBACK: Storage adapter missing getMetadataBatch - using individual calls with concurrency limit`);
metadataBatch = new Map();
const CONCURRENCY_LIMIT = 3; // Very conservative limit
for (let i = 0; i < nounIds.length; i += CONCURRENCY_LIMIT) {
const batch = nounIds.slice(i, i + CONCURRENCY_LIMIT);
const batchPromises = batch.map(async (id) => {
try {
const metadata = await this.storage.getMetadata(id);
return { id, metadata };
}
catch (error) {
prodLog.debug(`Failed to read metadata for ${id}:`, error);
return { id, metadata: null };
}
});
const batchResults = await Promise.all(batchPromises);
for (const { id, metadata } of batchResults) {
if (metadata) {
metadataBatch.set(id, metadata);
}
}
// Yield between batches to prevent socket exhaustion
await this.yieldToEventLoop();
}
}
// Process the metadata batch
for (const noun of result.items) {
const metadata = metadataBatch.get(noun.id);
if (metadata) {
// Skip flush during rebuild for performance
await this.addToIndex(noun.id, metadata, true);
}
}
// Yield after processing the entire batch
await this.yieldToEventLoop();
totalNounsProcessed += result.items.length;
hasMoreNouns = result.hasMore;
nounOffset += nounLimit;
// Progress logging and event loop yield after each batch
if (totalNounsProcessed % 100 === 0 || !hasMoreNouns) {
prodLog.debug(`📊 Indexed ${totalNounsProcessed} nouns...`);
}
await this.yieldToEventLoop();
}
// Rebuild verb metadata indexes using pagination
let verbOffset = 0;
const verbLimit = 25; // Even smaller batches during initialization to prevent socket exhaustion
let hasMoreVerbs = true;
let totalVerbsProcessed = 0;
while (hasMoreVerbs) {
const result = await this.storage.getVerbs({
pagination: { offset: verbOffset, limit: verbLimit }
});
// CRITICAL FIX: Use batch verb metadata reading to prevent socket exhaustion
const verbIds = result.items.map(verb => verb.id);
let verbMetadataBatch;
if (this.storage.getVerbMetadataBatch) {
// Use batch reading if available (prevents socket exhaustion)
verbMetadataBatch = await this.storage.getVerbMetadataBatch(verbIds);
prodLog.debug(`📦 Batch loaded ${verbMetadataBatch.size}/${verbIds.length} verb metadata objects`);
}
else {
// Fallback to individual calls with strict concurrency control
verbMetadataBatch = new Map();
const CONCURRENCY_LIMIT = 3; // Very conservative limit to prevent socket exhaustion
for (let i = 0; i < verbIds.length; i += CONCURRENCY_LIMIT) {
const batch = verbIds.slice(i, i + CONCURRENCY_LIMIT);
const batchPromises = batch.map(async (id) => {
try {
const metadata = await this.storage.getVerbMetadata(id);
return { id, metadata };
}
catch (error) {
prodLog.debug(`Failed to read verb metadata for ${id}:`, error);
return { id, metadata: null };
}
});
const batchResults = await Promise.all(batchPromises);
for (const { id, metadata } of batchResults) {
if (metadata) {
verbMetadataBatch.set(id, metadata);
}
}
// Yield between batches to prevent socket exhaustion
await this.yieldToEventLoop();
}
}
// Process the verb metadata batch
for (const verb of result.items) {
const metadata = verbMetadataBatch.get(verb.id);
if (metadata) {
// Skip flush during rebuild for performance
await this.addToIndex(verb.id, metadata, true);
}
}
// Yield after processing the entire batch
await this.yieldToEventLoop();
totalVerbsProcessed += result.items.length;
hasMoreVerbs = result.hasMore;
verbOffset += verbLimit;
// Progress logging and event loop yield after each batch
if (totalVerbsProcessed % 100 === 0 || !hasMoreVerbs) {
prodLog.debug(`🔗 Indexed ${totalVerbsProcessed} verbs...`);
}
await this.yieldToEventLoop();
}
// Flush to storage with final yield
prodLog.debug('💾 Flushing metadata index to storage...');
await this.flush();
await this.yieldToEventLoop();
prodLog.info(`✅ Metadata index rebuild completed! Processed ${totalNounsProcessed} nouns and ${totalVerbsProcessed} verbs`);
prodLog.info(`🎯 Initial indexing may show minor socket timeouts - this is expected and doesn't affect data processing`);
}
finally {
this.isRebuilding = false;
}
}
/**
* Load index entry from storage using safe filenames
*/
async loadIndexEntry(key) {
try {
// Extract field and value from key
const [field, value] = key.split(':', 2);
const filename = this.getValueChunkFilename(field, value);
// Load from metadata indexes directory with safe filename
const indexId = `__metadata_index__${filename}`;
const data = await this.storage.getMetadata(indexId);
if (data) {
return {
field: data.field,
value: data.value,
ids: new Set(data.ids || []),
lastUpdated: data.lastUpdated || Date.now()
};
}
}
catch (error) {
// Index entry doesn't exist yet
}
return null;
}
/**
* Save index entry to storage using safe filenames
*/
async saveIndexEntry(key, entry) {
const data = {
field: entry.field,
value: entry.value,
ids: Array.from(entry.ids),
lastUpdated: entry.lastUpdated
};
// Extract field and value from key for safe filename generation
const [field, value] = key.split(':', 2);
const filename = this.getValueChunkFilename(field, value);
// Store metadata indexes with safe filename
const indexId = `__metadata_index__${filename}`;
await this.storage.saveMetadata(indexId, data);
}
/**
* Delete index entry from storage using safe filenames
*/
async deleteIndexEntry(key) {
try {
const [field, value] = key.split(':', 2);
const filename = this.getValueChunkFilename(field, value);
const indexId = `__metadata_index__${filename}`;
await this.storage.saveMetadata(indexId, null);
}
catch (error) {
// Entry might not exist
}
}
}
//# sourceMappingURL=metadataIndex.js.map

1
dist/utils/metadataIndex.js.map vendored Normal file
View file

File diff suppressed because one or more lines are too long

60
dist/utils/metadataIndexCache.d.ts vendored Normal file
View file

@ -0,0 +1,60 @@
/**
* MetadataIndexCache - Caches metadata index data for improved performance
* Reuses the same pattern as SearchCache for consistency
*/
export interface MetadataCacheEntry {
data: any;
timestamp: number;
hits: number;
}
export interface MetadataIndexCacheConfig {
maxAge?: number;
maxSize?: number;
enabled?: boolean;
hitCountWeight?: number;
}
export declare class MetadataIndexCache {
private cache;
private maxAge;
private maxSize;
private enabled;
private hitCountWeight;
private hits;
private misses;
private evictions;
constructor(config?: MetadataIndexCacheConfig);
/**
* Get cached entry
*/
get(key: string): any | undefined;
/**
* Set cache entry
*/
set(key: string, data: any): void;
/**
* Evict least valuable entry based on age and hit count
*/
private evictLeastValuable;
/**
* Invalidate cache entries matching a pattern
*/
invalidatePattern(pattern: string): void;
/**
* Clear all cache entries
*/
clear(): void;
/**
* Get cache statistics
*/
getStats(): {
size: number;
hits: number;
misses: number;
hitRate: number;
evictions: number;
};
/**
* Get estimated memory usage
*/
getMemoryUsage(): number;
}

119
dist/utils/metadataIndexCache.js vendored Normal file
View file

@ -0,0 +1,119 @@
/**
* MetadataIndexCache - Caches metadata index data for improved performance
* Reuses the same pattern as SearchCache for consistency
*/
export class MetadataIndexCache {
constructor(config = {}) {
this.cache = new Map();
// Cache statistics
this.hits = 0;
this.misses = 0;
this.evictions = 0;
this.maxAge = config.maxAge ?? 5 * 60 * 1000; // 5 minutes
this.maxSize = config.maxSize ?? 500; // More entries than SearchCache since indexes are smaller
this.enabled = config.enabled ?? true;
this.hitCountWeight = config.hitCountWeight ?? 0.3;
}
/**
* Get cached entry
*/
get(key) {
if (!this.enabled)
return undefined;
const entry = this.cache.get(key);
if (!entry) {
this.misses++;
return undefined;
}
// Check if entry is expired
if (Date.now() - entry.timestamp > this.maxAge) {
this.cache.delete(key);
this.misses++;
return undefined;
}
// Update hit count
entry.hits++;
this.hits++;
return entry.data;
}
/**
* Set cache entry
*/
set(key, data) {
if (!this.enabled)
return;
// Evict entries if at max size
if (this.cache.size >= this.maxSize) {
this.evictLeastValuable();
}
this.cache.set(key, {
data,
timestamp: Date.now(),
hits: 0
});
}
/**
* Evict least valuable entry based on age and hit count
*/
evictLeastValuable() {
let leastValuableKey = null;
let lowestScore = Infinity;
for (const [key, entry] of this.cache.entries()) {
const age = Date.now() - entry.timestamp;
const ageScore = age / this.maxAge;
const hitScore = entry.hits * this.hitCountWeight;
const score = hitScore - ageScore;
if (score < lowestScore) {
lowestScore = score;
leastValuableKey = key;
}
}
if (leastValuableKey) {
this.cache.delete(leastValuableKey);
this.evictions++;
}
}
/**
* Invalidate cache entries matching a pattern
*/
invalidatePattern(pattern) {
const keysToDelete = [];
for (const key of this.cache.keys()) {
if (key.includes(pattern)) {
keysToDelete.push(key);
}
}
keysToDelete.forEach(key => this.cache.delete(key));
}
/**
* Clear all cache entries
*/
clear() {
this.cache.clear();
}
/**
* Get cache statistics
*/
getStats() {
return {
size: this.cache.size,
hits: this.hits,
misses: this.misses,
hitRate: this.hits / (this.hits + this.misses) || 0,
evictions: this.evictions
};
}
/**
* Get estimated memory usage
*/
getMemoryUsage() {
// Rough estimate: 100 bytes per entry + data size
let totalSize = 0;
for (const entry of this.cache.values()) {
totalSize += 100; // Base overhead
totalSize += JSON.stringify(entry.data).length * 2; // Unicode chars
}
return totalSize;
}
}
//# sourceMappingURL=metadataIndexCache.js.map

1
dist/utils/metadataIndexCache.js.map vendored Normal file
View file

@ -0,0 +1 @@
{"version":3,"file":"metadataIndexCache.js","sourceRoot":"","sources":["../../src/utils/metadataIndexCache.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAeH,MAAM,OAAO,kBAAkB;IAY7B,YAAY,SAAmC,EAAE;QAXzC,UAAK,GAAG,IAAI,GAAG,EAA8B,CAAA;QAMrD,mBAAmB;QACX,SAAI,GAAG,CAAC,CAAA;QACR,WAAM,GAAG,CAAC,CAAA;QACV,cAAS,GAAG,CAAC,CAAA;QAGnB,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC,MAAM,IAAI,CAAC,GAAG,EAAE,GAAG,IAAI,CAAA,CAAC,YAAY;QACzD,IAAI,CAAC,OAAO,GAAG,MAAM,CAAC,OAAO,IAAI,GAAG,CAAA,CAAC,0DAA0D;QAC/F,IAAI,CAAC,OAAO,GAAG,MAAM,CAAC,OAAO,IAAI,IAAI,CAAA;QACrC,IAAI,CAAC,cAAc,GAAG,MAAM,CAAC,cAAc,IAAI,GAAG,CAAA;IACpD,CAAC;IAED;;OAEG;IACH,GAAG,CAAC,GAAW;QACb,IAAI,CAAC,IAAI,CAAC,OAAO;YAAE,OAAO,SAAS,CAAA;QAEnC,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,CAAA;QACjC,IAAI,CAAC,KAAK,EAAE,CAAC;YACX,IAAI,CAAC,MAAM,EAAE,CAAA;YACb,OAAO,SAAS,CAAA;QAClB,CAAC;QAED,4BAA4B;QAC5B,IAAI,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK,CAAC,SAAS,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC;YAC/C,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,CAAA;YACtB,IAAI,CAAC,MAAM,EAAE,CAAA;YACb,OAAO,SAAS,CAAA;QAClB,CAAC;QAED,mBAAmB;QACnB,KAAK,CAAC,IAAI,EAAE,CAAA;QACZ,IAAI,CAAC,IAAI,EAAE,CAAA;QACX,OAAO,KAAK,CAAC,IAAI,CAAA;IACnB,CAAC;IAED;;OAEG;IACH,GAAG,CAAC,GAAW,EAAE,IAAS;QACxB,IAAI,CAAC,IAAI,CAAC,OAAO;YAAE,OAAM;QAEzB,+BAA+B;QAC/B,IAAI,IAAI,CAAC,KAAK,CAAC,IAAI,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;YACpC,IAAI,CAAC,kBAAkB,EAAE,CAAA;QAC3B,CAAC;QAED,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,EAAE;YAClB,IAAI;YACJ,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE;YACrB,IAAI,EAAE,CAAC;SACR,CAAC,CAAA;IACJ,CAAC;IAED;;OAEG;IACK,kBAAkB;QACxB,IAAI,gBAAgB,GAAkB,IAAI,CAAA;QAC1C,IAAI,WAAW,GAAG,QAAQ,CAAA;QAE1B,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,IAAI,CAAC,KAAK,CAAC,OAAO,EAAE,EAAE,CAAC;YAChD,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK,CAAC,SAAS,CAAA;YACxC,MAAM,QAAQ,GAAG,GAAG,GAAG,IAAI,CAAC,MAAM,CAAA;YAClC,MAAM,QAAQ,GAAG,KAAK,CAAC,IAAI,GAAG,IAAI,CAAC,cAAc,CAAA;YACjD,MAAM,KAAK,GAAG,QAAQ,GAAG,QAAQ,CAAA;YAEjC,IAAI,KAAK,GAAG,WAAW,EAAE,CAAC;gBACxB,WAAW,GAAG,KAAK,CAAA;gBACnB,gBAAgB,GAAG,GAAG,CAAA;YACxB,CAAC;QACH,CAAC;QAED,IAAI,gBAAgB,EAAE,CAAC;YACrB,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,gBAAgB,CAAC,CAAA;YACnC,IAAI,CAAC,SAAS,EAAE,CAAA;QAClB,CAAC;IACH,CAAC;IAED;;OAEG;IACH,iBAAiB,CAAC,OAAe;QAC/B,MAAM,YAAY,GAAa,EAAE,CAAA;QACjC,KAAK,MAAM,GAAG,IAAI,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE,EAAE,CAAC;YACpC,IAAI,GAAG,CAAC,QAAQ,CAAC,OAAO,CAAC,EAAE,CAAC;gBAC1B,YAAY,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;YACxB,CAAC;QACH,CAAC;QACD,YAAY,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAA;IACrD,CAAC;IAED;;OAEG;IACH,KAAK;QACH,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,CAAA;IACpB,CAAC;IAED;;OAEG;IACH,QAAQ;QACN,OAAO;YACL,IAAI,EAAE,IAAI,CAAC,KAAK,CAAC,IAAI;YACrB,IAAI,EAAE,IAAI,CAAC,IAAI;YACf,MAAM,EAAE,IAAI,CAAC,MAAM;YACnB,OAAO,EAAE,IAAI,CAAC,IAAI,GAAG,CAAC,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC;YACnD,SAAS,EAAE,IAAI,CAAC,SAAS;SAC1B,CAAA;IACH,CAAC;IAED;;OAEG;IACH,cAAc;QACZ,kDAAkD;QAClD,IAAI,SAAS,GAAG,CAAC,CAAA;QACjB,KAAK,MAAM,KAAK,IAAI,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,EAAE,CAAC;YACxC,SAAS,IAAI,GAAG,CAAA,CAAC,gBAAgB;YACjC,SAAS,IAAI,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,MAAM,GAAG,CAAC,CAAA,CAAC,gBAAgB;QACrE,CAAC;QACD,OAAO,SAAS,CAAA;IAClB,CAAC;CACF"}

58
dist/utils/operationUtils.d.ts vendored Normal file
View file

@ -0,0 +1,58 @@
/**
* Utility functions for timeout and retry logic
* Used by storage adapters to handle network operations reliably
*/
export interface TimeoutConfig {
get?: number;
add?: number;
delete?: number;
}
export interface RetryConfig {
maxRetries?: number;
initialDelay?: number;
maxDelay?: number;
backoffMultiplier?: number;
}
export interface OperationConfig {
timeouts?: TimeoutConfig;
retryPolicy?: RetryConfig;
}
export declare const DEFAULT_TIMEOUTS: Required<TimeoutConfig>;
export declare const DEFAULT_RETRY_POLICY: Required<RetryConfig>;
/**
* Wraps a promise with a timeout
*/
export declare function withTimeout<T>(promise: Promise<T>, timeoutMs: number, operation: string): Promise<T>;
/**
* Executes an operation with retry logic and exponential backoff
*/
export declare function withRetry<T>(operation: () => Promise<T>, operationName: string, config?: RetryConfig): Promise<T>;
/**
* Executes an operation with both timeout and retry logic
*/
export declare function withTimeoutAndRetry<T>(operation: () => Promise<T>, operationName: string, timeoutMs: number, retryConfig?: RetryConfig): Promise<T>;
/**
* Creates a configured operation executor for a specific operation type
*/
export declare function createOperationExecutor(operationType: keyof TimeoutConfig, config?: OperationConfig): <T>(operation: () => Promise<T>, operationName: string) => Promise<T>;
/**
* Storage operation executors for different operation types
*/
export declare class StorageOperationExecutors {
private getExecutor;
private addExecutor;
private deleteExecutor;
constructor(config?: OperationConfig);
/**
* Execute a get operation with timeout and retry
*/
executeGet<T>(operation: () => Promise<T>, operationName: string): Promise<T>;
/**
* Execute an add operation with timeout and retry
*/
executeAdd<T>(operation: () => Promise<T>, operationName: string): Promise<T>;
/**
* Execute a delete operation with timeout and retry
*/
executeDelete<T>(operation: () => Promise<T>, operationName: string): Promise<T>;
}

126
dist/utils/operationUtils.js vendored Normal file
View file

@ -0,0 +1,126 @@
/**
* Utility functions for timeout and retry logic
* Used by storage adapters to handle network operations reliably
*/
import { BrainyError } from '../errors/brainyError.js';
// Default configuration values
export const DEFAULT_TIMEOUTS = {
get: 30000, // 30 seconds
add: 60000, // 1 minute
delete: 30000 // 30 seconds
};
export const DEFAULT_RETRY_POLICY = {
maxRetries: 3,
initialDelay: 1000,
maxDelay: 10000,
backoffMultiplier: 2
};
/**
* Wraps a promise with a timeout
*/
export function withTimeout(promise, timeoutMs, operation) {
return new Promise((resolve, reject) => {
const timeoutId = setTimeout(() => {
reject(BrainyError.timeout(operation, timeoutMs));
}, timeoutMs);
promise
.then((result) => {
clearTimeout(timeoutId);
resolve(result);
})
.catch((error) => {
clearTimeout(timeoutId);
reject(error);
});
});
}
/**
* Calculates the delay for exponential backoff
*/
function calculateBackoffDelay(attemptNumber, initialDelay, maxDelay, backoffMultiplier) {
const delay = initialDelay * Math.pow(backoffMultiplier, attemptNumber - 1);
return Math.min(delay, maxDelay);
}
/**
* Sleeps for the specified number of milliseconds
*/
function sleep(ms) {
return new Promise(resolve => setTimeout(resolve, ms));
}
/**
* Executes an operation with retry logic and exponential backoff
*/
export async function withRetry(operation, operationName, config = {}) {
const { maxRetries = DEFAULT_RETRY_POLICY.maxRetries, initialDelay = DEFAULT_RETRY_POLICY.initialDelay, maxDelay = DEFAULT_RETRY_POLICY.maxDelay, backoffMultiplier = DEFAULT_RETRY_POLICY.backoffMultiplier } = config;
let lastError;
for (let attempt = 1; attempt <= maxRetries + 1; attempt++) {
try {
return await operation();
}
catch (error) {
lastError = error instanceof Error ? error : new Error(String(error));
// If this is the last attempt, don't retry
if (attempt > maxRetries) {
break;
}
// Check if the error is retryable
if (!BrainyError.isRetryable(lastError)) {
throw BrainyError.fromError(lastError, operationName);
}
// Calculate delay for exponential backoff
const delay = calculateBackoffDelay(attempt, initialDelay, maxDelay, backoffMultiplier);
console.warn(`Operation '${operationName}' failed on attempt ${attempt}/${maxRetries + 1}. ` +
`Retrying in ${delay}ms. Error: ${lastError.message}`);
// Wait before retrying
await sleep(delay);
}
}
// All retries exhausted
throw BrainyError.retryExhausted(operationName, maxRetries, lastError);
}
/**
* Executes an operation with both timeout and retry logic
*/
export async function withTimeoutAndRetry(operation, operationName, timeoutMs, retryConfig = {}) {
return withRetry(() => withTimeout(operation(), timeoutMs, operationName), operationName, retryConfig);
}
/**
* Creates a configured operation executor for a specific operation type
*/
export function createOperationExecutor(operationType, config = {}) {
const timeouts = { ...DEFAULT_TIMEOUTS, ...config.timeouts };
const retryPolicy = { ...DEFAULT_RETRY_POLICY, ...config.retryPolicy };
const timeoutMs = timeouts[operationType];
return async function executeOperation(operation, operationName) {
return withTimeoutAndRetry(operation, operationName, timeoutMs, retryPolicy);
};
}
/**
* Storage operation executors for different operation types
*/
export class StorageOperationExecutors {
constructor(config = {}) {
this.getExecutor = createOperationExecutor('get', config);
this.addExecutor = createOperationExecutor('add', config);
this.deleteExecutor = createOperationExecutor('delete', config);
}
/**
* Execute a get operation with timeout and retry
*/
async executeGet(operation, operationName) {
return this.getExecutor(operation, operationName);
}
/**
* Execute an add operation with timeout and retry
*/
async executeAdd(operation, operationName) {
return this.addExecutor(operation, operationName);
}
/**
* Execute a delete operation with timeout and retry
*/
async executeDelete(operation, operationName) {
return this.deleteExecutor(operation, operationName);
}
}
//# sourceMappingURL=operationUtils.js.map

1
dist/utils/operationUtils.js.map vendored Normal file
View file

@ -0,0 +1 @@
{"version":3,"file":"operationUtils.js","sourceRoot":"","sources":["../../src/utils/operationUtils.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EAAE,WAAW,EAAE,MAAM,0BAA0B,CAAA;AAoBtD,+BAA+B;AAC/B,MAAM,CAAC,MAAM,gBAAgB,GAA4B;IACrD,GAAG,EAAE,KAAK,EAAO,aAAa;IAC9B,GAAG,EAAE,KAAK,EAAO,WAAW;IAC5B,MAAM,EAAE,KAAK,CAAI,aAAa;CACjC,CAAA;AAED,MAAM,CAAC,MAAM,oBAAoB,GAA0B;IACvD,UAAU,EAAE,CAAC;IACb,YAAY,EAAE,IAAI;IAClB,QAAQ,EAAE,KAAK;IACf,iBAAiB,EAAE,CAAC;CACvB,CAAA;AAED;;GAEG;AACH,MAAM,UAAU,WAAW,CACvB,OAAmB,EACnB,SAAiB,EACjB,SAAiB;IAEjB,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;QACnC,MAAM,SAAS,GAAG,UAAU,CAAC,GAAG,EAAE;YAC9B,MAAM,CAAC,WAAW,CAAC,OAAO,CAAC,SAAS,EAAE,SAAS,CAAC,CAAC,CAAA;QACrD,CAAC,EAAE,SAAS,CAAC,CAAA;QAEb,OAAO;aACF,IAAI,CAAC,CAAC,MAAM,EAAE,EAAE;YACb,YAAY,CAAC,SAAS,CAAC,CAAA;YACvB,OAAO,CAAC,MAAM,CAAC,CAAA;QACnB,CAAC,CAAC;aACD,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE;YACb,YAAY,CAAC,SAAS,CAAC,CAAA;YACvB,MAAM,CAAC,KAAK,CAAC,CAAA;QACjB,CAAC,CAAC,CAAA;IACV,CAAC,CAAC,CAAA;AACN,CAAC;AAED;;GAEG;AACH,SAAS,qBAAqB,CAC1B,aAAqB,EACrB,YAAoB,EACpB,QAAgB,EAChB,iBAAyB;IAEzB,MAAM,KAAK,GAAG,YAAY,GAAG,IAAI,CAAC,GAAG,CAAC,iBAAiB,EAAE,aAAa,GAAG,CAAC,CAAC,CAAA;IAC3E,OAAO,IAAI,CAAC,GAAG,CAAC,KAAK,EAAE,QAAQ,CAAC,CAAA;AACpC,CAAC;AAED;;GAEG;AACH,SAAS,KAAK,CAAC,EAAU;IACrB,OAAO,IAAI,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,CAAA;AAC1D,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,SAAS,CAC3B,SAA2B,EAC3B,aAAqB,EACrB,SAAsB,EAAE;IAExB,MAAM,EACF,UAAU,GAAG,oBAAoB,CAAC,UAAU,EAC5C,YAAY,GAAG,oBAAoB,CAAC,YAAY,EAChD,QAAQ,GAAG,oBAAoB,CAAC,QAAQ,EACxC,iBAAiB,GAAG,oBAAoB,CAAC,iBAAiB,EAC7D,GAAG,MAAM,CAAA;IAEV,IAAI,SAA4B,CAAA;IAEhC,KAAK,IAAI,OAAO,GAAG,CAAC,EAAE,OAAO,IAAI,UAAU,GAAG,CAAC,EAAE,OAAO,EAAE,EAAE,CAAC;QACzD,IAAI,CAAC;YACD,OAAO,MAAM,SAAS,EAAE,CAAA;QAC5B,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACb,SAAS,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAA;YAErE,2CAA2C;YAC3C,IAAI,OAAO,GAAG,UAAU,EAAE,CAAC;gBACvB,MAAK;YACT,CAAC;YAED,kCAAkC;YAClC,IAAI,CAAC,WAAW,CAAC,WAAW,CAAC,SAAS,CAAC,EAAE,CAAC;gBACtC,MAAM,WAAW,CAAC,SAAS,CAAC,SAAS,EAAE,aAAa,CAAC,CAAA;YACzD,CAAC;YAED,0CAA0C;YAC1C,MAAM,KAAK,GAAG,qBAAqB,CAAC,OAAO,EAAE,YAAY,EAAE,QAAQ,EAAE,iBAAiB,CAAC,CAAA;YAEvF,OAAO,CAAC,IAAI,CACR,cAAc,aAAa,uBAAuB,OAAO,IAAI,UAAU,GAAG,CAAC,IAAI;gBAC/E,eAAe,KAAK,cAAc,SAAS,CAAC,OAAO,EAAE,CACxD,CAAA;YAED,uBAAuB;YACvB,MAAM,KAAK,CAAC,KAAK,CAAC,CAAA;QACtB,CAAC;IACL,CAAC;IAED,wBAAwB;IACxB,MAAM,WAAW,CAAC,cAAc,CAAC,aAAa,EAAE,UAAU,EAAE,SAAS,CAAC,CAAA;AAC1E,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,mBAAmB,CACrC,SAA2B,EAC3B,aAAqB,EACrB,SAAiB,EACjB,cAA2B,EAAE;IAE7B,OAAO,SAAS,CACZ,GAAG,EAAE,CAAC,WAAW,CAAC,SAAS,EAAE,EAAE,SAAS,EAAE,aAAa,CAAC,EACxD,aAAa,EACb,WAAW,CACd,CAAA;AACL,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,uBAAuB,CACnC,aAAkC,EAClC,SAA0B,EAAE;IAE5B,MAAM,QAAQ,GAAG,EAAE,GAAG,gBAAgB,EAAE,GAAG,MAAM,CAAC,QAAQ,EAAE,CAAA;IAC5D,MAAM,WAAW,GAAG,EAAE,GAAG,oBAAoB,EAAE,GAAG,MAAM,CAAC,WAAW,EAAE,CAAA;IACtE,MAAM,SAAS,GAAG,QAAQ,CAAC,aAAa,CAAC,CAAA;IAEzC,OAAO,KAAK,UAAU,gBAAgB,CAClC,SAA2B,EAC3B,aAAqB;QAErB,OAAO,mBAAmB,CAAC,SAAS,EAAE,aAAa,EAAE,SAAS,EAAE,WAAW,CAAC,CAAA;IAChF,CAAC,CAAA;AACL,CAAC;AAED;;GAEG;AACH,MAAM,OAAO,yBAAyB;IAKlC,YAAY,SAA0B,EAAE;QACpC,IAAI,CAAC,WAAW,GAAG,uBAAuB,CAAC,KAAK,EAAE,MAAM,CAAC,CAAA;QACzD,IAAI,CAAC,WAAW,GAAG,uBAAuB,CAAC,KAAK,EAAE,MAAM,CAAC,CAAA;QACzD,IAAI,CAAC,cAAc,GAAG,uBAAuB,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAA;IACnE,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,UAAU,CAAI,SAA2B,EAAE,aAAqB;QAClE,OAAO,IAAI,CAAC,WAAW,CAAC,SAAS,EAAE,aAAa,CAAC,CAAA;IACrD,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,UAAU,CAAI,SAA2B,EAAE,aAAqB;QAClE,OAAO,IAAI,CAAC,WAAW,CAAC,SAAS,EAAE,aAAa,CAAC,CAAA;IACrD,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,aAAa,CAAI,SAA2B,EAAE,aAAqB;QACrE,OAAO,IAAI,CAAC,cAAc,CAAC,SAAS,EAAE,aAAa,CAAC,CAAA;IACxD,CAAC;CACJ"}

114
dist/utils/performanceMonitor.d.ts vendored Normal file
View file

@ -0,0 +1,114 @@
/**
* Performance Monitor
* Automatically tracks and optimizes system performance
* Provides real-time insights and auto-tuning recommendations
*/
interface PerformanceMetrics {
totalOperations: number;
successfulOperations: number;
failedOperations: number;
averageLatency: number;
p95Latency: number;
p99Latency: number;
operationsPerSecond: number;
bytesPerSecond: number;
memoryUsage: number;
cpuUsage: number;
socketUtilization: number;
queueDepth: number;
errorRate: number;
healthScore: number;
}
interface PerformanceTrend {
metric: string;
direction: 'improving' | 'degrading' | 'stable';
changeRate: number;
prediction: number;
}
/**
* Comprehensive performance monitoring and optimization
*/
export declare class PerformanceMonitor {
private logger;
private metrics;
private history;
private maxHistorySize;
private operationLatencies;
private operationSizes;
private lastReset;
private resetInterval;
private lastCpuUsage;
private lastCpuCheck;
private thresholds;
private recommendations;
private autoOptimizeEnabled;
private lastOptimization;
private optimizationInterval;
/**
* Track an operation completion
*/
trackOperation(success: boolean, latency: number, bytes?: number): void;
/**
* Update all metrics
*/
private updateMetrics;
/**
* Update resource metrics
*/
private updateResourceMetrics;
/**
* Calculate overall health score
*/
private calculateHealthScore;
/**
* Check for alert conditions
*/
private checkAlerts;
/**
* Auto-optimize system based on metrics
*/
private autoOptimize;
/**
* Analyze performance trends
*/
private analyzeTrends;
/**
* Reset counters
*/
private resetCounters;
/**
* Get current metrics
*/
getMetrics(): Readonly<PerformanceMetrics>;
/**
* Get performance trends
*/
getTrends(): PerformanceTrend[];
/**
* Get recommendations
*/
getRecommendations(): string[];
/**
* Get performance report
*/
getReport(): {
metrics: PerformanceMetrics;
trends: PerformanceTrend[];
recommendations: string[];
socketConfig: any;
backpressureStatus: any;
};
/**
* Enable/disable auto-optimization
*/
setAutoOptimize(enabled: boolean): void;
/**
* Reset all metrics and history
*/
reset(): void;
}
/**
* Get the global performance monitor instance
*/
export declare function getGlobalPerformanceMonitor(): PerformanceMonitor;
export {};

384
dist/utils/performanceMonitor.js vendored Normal file
View file

@ -0,0 +1,384 @@
/**
* Performance Monitor
* Automatically tracks and optimizes system performance
* Provides real-time insights and auto-tuning recommendations
*/
import { createModuleLogger } from './logger.js';
import { getGlobalSocketManager } from './adaptiveSocketManager.js';
import { getGlobalBackpressure } from './adaptiveBackpressure.js';
/**
* Comprehensive performance monitoring and optimization
*/
export class PerformanceMonitor {
constructor() {
this.logger = createModuleLogger('PerformanceMonitor');
// Current metrics
this.metrics = {
totalOperations: 0,
successfulOperations: 0,
failedOperations: 0,
averageLatency: 0,
p95Latency: 0,
p99Latency: 0,
operationsPerSecond: 0,
bytesPerSecond: 0,
memoryUsage: 0,
cpuUsage: 0,
socketUtilization: 0,
queueDepth: 0,
errorRate: 0,
healthScore: 100
};
// Historical data for trend analysis
this.history = [];
this.maxHistorySize = 1000;
// Operation tracking
this.operationLatencies = [];
this.operationSizes = [];
this.lastReset = Date.now();
this.resetInterval = 60000; // Reset counters every minute
// CPU tracking
this.lastCpuUsage = process.cpuUsage ? process.cpuUsage() : null;
this.lastCpuCheck = Date.now();
// Alert thresholds
this.thresholds = {
errorRate: 0.05, // 5% error rate
latencyP95: 5000, // 5 second P95
memoryUsage: 0.8, // 80% memory
cpuUsage: 0.9, // 90% CPU
healthScore: 70 // Health score below 70
};
// Optimization recommendations
this.recommendations = [];
// Auto-optimization state
this.autoOptimizeEnabled = true;
this.lastOptimization = Date.now();
this.optimizationInterval = 30000; // Optimize every 30 seconds
}
/**
* Track an operation completion
*/
trackOperation(success, latency, bytes = 0) {
// Update counters
this.metrics.totalOperations++;
if (success) {
this.metrics.successfulOperations++;
}
else {
this.metrics.failedOperations++;
}
// Track latency
this.operationLatencies.push(latency);
if (this.operationLatencies.length > 10000) {
this.operationLatencies = this.operationLatencies.slice(-5000);
}
// Track size
if (bytes > 0) {
this.operationSizes.push(bytes);
if (this.operationSizes.length > 10000) {
this.operationSizes = this.operationSizes.slice(-5000);
}
}
// Update metrics periodically
this.updateMetrics();
}
/**
* Update all metrics
*/
updateMetrics() {
const now = Date.now();
const timeSinceReset = (now - this.lastReset) / 1000;
// Calculate latency percentiles
if (this.operationLatencies.length > 0) {
const sorted = [...this.operationLatencies].sort((a, b) => a - b);
const p95Index = Math.floor(sorted.length * 0.95);
const p99Index = Math.floor(sorted.length * 0.99);
this.metrics.averageLatency = sorted.reduce((a, b) => a + b, 0) / sorted.length;
this.metrics.p95Latency = sorted[p95Index] || 0;
this.metrics.p99Latency = sorted[p99Index] || 0;
}
// Calculate throughput
if (timeSinceReset > 0) {
this.metrics.operationsPerSecond = this.metrics.totalOperations / timeSinceReset;
const totalBytes = this.operationSizes.reduce((a, b) => a + b, 0);
this.metrics.bytesPerSecond = totalBytes / timeSinceReset;
}
// Calculate error rate
this.metrics.errorRate = this.metrics.totalOperations > 0
? this.metrics.failedOperations / this.metrics.totalOperations
: 0;
// Update resource metrics
this.updateResourceMetrics();
// Calculate health score
this.calculateHealthScore();
// Store in history
this.history.push({ ...this.metrics });
if (this.history.length > this.maxHistorySize) {
this.history.shift();
}
// Check for alerts
this.checkAlerts();
// Auto-optimize if enabled
if (this.autoOptimizeEnabled && now - this.lastOptimization > this.optimizationInterval) {
this.autoOptimize();
this.lastOptimization = now;
}
// Reset counters periodically
if (now - this.lastReset > this.resetInterval) {
this.resetCounters();
}
}
/**
* Update resource metrics
*/
updateResourceMetrics() {
// Memory usage
if (typeof process !== 'undefined' && process.memoryUsage) {
const memUsage = process.memoryUsage();
this.metrics.memoryUsage = memUsage.heapUsed / memUsage.heapTotal;
}
// CPU usage (Node.js only)
if (this.lastCpuUsage && process.cpuUsage) {
const currentCpuUsage = process.cpuUsage();
const now = Date.now();
const timeDiff = now - this.lastCpuCheck;
if (timeDiff > 1000) { // Update CPU every second
const userDiff = currentCpuUsage.user - this.lastCpuUsage.user;
const systemDiff = currentCpuUsage.system - this.lastCpuUsage.system;
const totalDiff = userDiff + systemDiff;
// CPU percentage (approximate)
this.metrics.cpuUsage = totalDiff / (timeDiff * 1000);
this.lastCpuUsage = currentCpuUsage;
this.lastCpuCheck = now;
}
}
// Get metrics from socket manager
const socketMetrics = getGlobalSocketManager().getMetrics();
this.metrics.socketUtilization = socketMetrics.socketUtilization;
// Get metrics from backpressure system
const backpressureStatus = getGlobalBackpressure().getStatus();
this.metrics.queueDepth = backpressureStatus.queueLength;
}
/**
* Calculate overall health score
*/
calculateHealthScore() {
let score = 100;
// Deduct points for high error rate
if (this.metrics.errorRate > 0.01) {
score -= Math.min(30, this.metrics.errorRate * 300);
}
// Deduct points for high latency
if (this.metrics.p95Latency > 3000) {
score -= Math.min(20, (this.metrics.p95Latency - 3000) / 100);
}
// Deduct points for high memory usage
if (this.metrics.memoryUsage > 0.7) {
score -= Math.min(20, (this.metrics.memoryUsage - 0.7) * 66);
}
// Deduct points for high CPU usage
if (this.metrics.cpuUsage > 0.8) {
score -= Math.min(15, (this.metrics.cpuUsage - 0.8) * 75);
}
// Deduct points for low throughput
if (this.metrics.operationsPerSecond < 1 && this.metrics.totalOperations > 10) {
score -= 10;
}
// Deduct points for queue depth
if (this.metrics.queueDepth > 100) {
score -= Math.min(15, this.metrics.queueDepth / 20);
}
this.metrics.healthScore = Math.max(0, Math.min(100, score));
}
/**
* Check for alert conditions
*/
checkAlerts() {
const alerts = [];
if (this.metrics.errorRate > this.thresholds.errorRate) {
alerts.push(`High error rate: ${(this.metrics.errorRate * 100).toFixed(1)}%`);
}
if (this.metrics.p95Latency > this.thresholds.latencyP95) {
alerts.push(`High P95 latency: ${this.metrics.p95Latency}ms`);
}
if (this.metrics.memoryUsage > this.thresholds.memoryUsage) {
alerts.push(`High memory usage: ${(this.metrics.memoryUsage * 100).toFixed(1)}%`);
}
if (this.metrics.cpuUsage > this.thresholds.cpuUsage) {
alerts.push(`High CPU usage: ${(this.metrics.cpuUsage * 100).toFixed(1)}%`);
}
if (this.metrics.healthScore < this.thresholds.healthScore) {
alerts.push(`Low health score: ${this.metrics.healthScore.toFixed(0)}`);
}
if (alerts.length > 0) {
this.logger.warn('Performance alerts', { alerts, metrics: this.metrics });
}
}
/**
* Auto-optimize system based on metrics
*/
autoOptimize() {
this.recommendations = [];
// Analyze trends
const trends = this.analyzeTrends();
// Generate recommendations based on metrics and trends
if (this.metrics.errorRate > 0.02) {
this.recommendations.push('Reduce load or increase timeouts due to high error rate');
}
if (this.metrics.p95Latency > 3000) {
this.recommendations.push('Increase batch size or socket limits to improve latency');
}
if (this.metrics.memoryUsage > 0.7) {
this.recommendations.push('Reduce cache sizes or batch sizes to free memory');
}
if (this.metrics.queueDepth > 50) {
this.recommendations.push('Increase concurrency limits to reduce queue depth');
}
// Check for degrading trends
trends.forEach(trend => {
if (trend.direction === 'degrading' && Math.abs(trend.changeRate) > 0.1) {
this.recommendations.push(`${trend.metric} is degrading at ${(trend.changeRate * 100).toFixed(1)}% per minute`);
}
});
// Log recommendations if any
if (this.recommendations.length > 0) {
this.logger.info('Performance optimization recommendations', {
recommendations: this.recommendations,
metrics: this.metrics
});
}
}
/**
* Analyze performance trends
*/
analyzeTrends() {
const trends = [];
if (this.history.length < 10) {
return trends; // Not enough data
}
// Get recent history
const recent = this.history.slice(-20);
const older = this.history.slice(-40, -20);
// Compare key metrics
const metricsToAnalyze = [
'errorRate',
'averageLatency',
'operationsPerSecond',
'memoryUsage',
'healthScore'
];
metricsToAnalyze.forEach(metric => {
const recentAvg = recent.reduce((sum, m) => sum + m[metric], 0) / recent.length;
const olderAvg = older.length > 0
? older.reduce((sum, m) => sum + m[metric], 0) / older.length
: recentAvg;
const changeRate = olderAvg !== 0 ? (recentAvg - olderAvg) / olderAvg : 0;
let direction = 'stable';
if (Math.abs(changeRate) > 0.05) { // 5% threshold
// For error rate and latency, increase is bad
if (metric === 'errorRate' || metric === 'averageLatency' || metric === 'memoryUsage') {
direction = changeRate > 0 ? 'degrading' : 'improving';
}
else {
// For throughput and health score, increase is good
direction = changeRate > 0 ? 'improving' : 'degrading';
}
}
// Simple linear prediction
const prediction = recentAvg + (recentAvg * changeRate);
trends.push({
metric,
direction,
changeRate,
prediction
});
});
return trends;
}
/**
* Reset counters
*/
resetCounters() {
this.metrics.totalOperations = 0;
this.metrics.successfulOperations = 0;
this.metrics.failedOperations = 0;
this.operationSizes = [];
this.lastReset = Date.now();
}
/**
* Get current metrics
*/
getMetrics() {
return { ...this.metrics };
}
/**
* Get performance trends
*/
getTrends() {
return this.analyzeTrends();
}
/**
* Get recommendations
*/
getRecommendations() {
return [...this.recommendations];
}
/**
* Get performance report
*/
getReport() {
return {
metrics: this.getMetrics(),
trends: this.getTrends(),
recommendations: this.getRecommendations(),
socketConfig: getGlobalSocketManager().getConfig(),
backpressureStatus: getGlobalBackpressure().getStatus()
};
}
/**
* Enable/disable auto-optimization
*/
setAutoOptimize(enabled) {
this.autoOptimizeEnabled = enabled;
this.logger.info(`Auto-optimization ${enabled ? 'enabled' : 'disabled'}`);
}
/**
* Reset all metrics and history
*/
reset() {
this.metrics = {
totalOperations: 0,
successfulOperations: 0,
failedOperations: 0,
averageLatency: 0,
p95Latency: 0,
p99Latency: 0,
operationsPerSecond: 0,
bytesPerSecond: 0,
memoryUsage: 0,
cpuUsage: 0,
socketUtilization: 0,
queueDepth: 0,
errorRate: 0,
healthScore: 100
};
this.history = [];
this.operationLatencies = [];
this.operationSizes = [];
this.recommendations = [];
this.lastReset = Date.now();
this.logger.info('Performance monitor reset');
}
}
// Global singleton instance
let globalMonitor = null;
/**
* Get the global performance monitor instance
*/
export function getGlobalPerformanceMonitor() {
if (!globalMonitor) {
globalMonitor = new PerformanceMonitor();
}
return globalMonitor;
}
//# sourceMappingURL=performanceMonitor.js.map

1
dist/utils/performanceMonitor.js.map vendored Normal file
View file

File diff suppressed because one or more lines are too long

91
dist/utils/requestCoalescer.d.ts vendored Normal file
View file

@ -0,0 +1,91 @@
/**
* Request Coalescer
* Batches and deduplicates operations to reduce S3 API calls
* Automatically flushes based on size, time, or pressure
*/
interface CoalescedOperation {
type: 'write' | 'read' | 'delete';
key: string;
data?: any;
resolve: (value: any) => void;
reject: (error: any) => void;
timestamp: number;
}
interface BatchStats {
totalOperations: number;
coalescedOperations: number;
deduplicated: number;
batchesProcessed: number;
averageBatchSize: number;
}
/**
* Coalesces multiple operations into efficient batches
*/
export declare class RequestCoalescer {
private logger;
private writeQueue;
private readQueue;
private deleteQueue;
private maxBatchSize;
private maxBatchAge;
private minBatchSize;
private flushTimer;
private lastFlush;
private stats;
private processor;
constructor(processor: (batch: CoalescedOperation[]) => Promise<void>, options?: {
maxBatchSize?: number;
maxBatchAge?: number;
minBatchSize?: number;
});
/**
* Add a write operation to be coalesced
*/
write(key: string, data: any): Promise<void>;
/**
* Add a read operation to be coalesced
*/
read(key: string): Promise<any>;
/**
* Add a delete operation to be coalesced
*/
delete(key: string): Promise<void>;
/**
* Check if we should flush the queues
*/
private checkFlush;
/**
* Flush all queued operations
*/
flush(reason?: string): Promise<void>;
/**
* Get current statistics
*/
getStats(): BatchStats;
/**
* Get current queue sizes
*/
getQueueSizes(): {
writes: number;
reads: number;
deletes: number;
total: number;
};
/**
* Adjust batch parameters based on load
*/
adjustParameters(pending: number): void;
/**
* Force immediate flush of all operations
*/
forceFlush(): Promise<void>;
}
/**
* Get or create a coalescer for a storage instance
*/
export declare function getCoalescer(storageId: string, processor: (batch: any[]) => Promise<void>): RequestCoalescer;
/**
* Clear all coalescers
*/
export declare function clearCoalescers(): void;
export {};

324
dist/utils/requestCoalescer.js vendored Normal file
View file

@ -0,0 +1,324 @@
/**
* Request Coalescer
* Batches and deduplicates operations to reduce S3 API calls
* Automatically flushes based on size, time, or pressure
*/
import { createModuleLogger } from './logger.js';
/**
* Coalesces multiple operations into efficient batches
*/
export class RequestCoalescer {
constructor(processor, options) {
this.logger = createModuleLogger('RequestCoalescer');
// Operation queues by type
this.writeQueue = new Map();
this.readQueue = new Map();
this.deleteQueue = new Map();
// Batch configuration
this.maxBatchSize = 100;
this.maxBatchAge = 100; // ms - flush quickly under load
this.minBatchSize = 10; // Don't flush until we have enough
// Flush timers
this.flushTimer = null;
this.lastFlush = Date.now();
// Statistics
this.stats = {
totalOperations: 0,
coalescedOperations: 0,
deduplicated: 0,
batchesProcessed: 0,
averageBatchSize: 0
};
this.processor = processor;
if (options) {
this.maxBatchSize = options.maxBatchSize || this.maxBatchSize;
this.maxBatchAge = options.maxBatchAge || this.maxBatchAge;
this.minBatchSize = options.minBatchSize || this.minBatchSize;
}
}
/**
* Add a write operation to be coalesced
*/
async write(key, data) {
return new Promise((resolve, reject) => {
// Check if we already have a pending write for this key
const existing = this.writeQueue.get(key);
if (existing && existing.length > 0) {
// Replace the data but resolve all promises
const last = existing[existing.length - 1];
last.data = data; // Use latest data
// Add this promise to be resolved
existing.push({
type: 'write',
key,
data,
resolve,
reject,
timestamp: Date.now()
});
this.stats.deduplicated++;
}
else {
// New write operation
this.writeQueue.set(key, [{
type: 'write',
key,
data,
resolve,
reject,
timestamp: Date.now()
}]);
}
this.stats.totalOperations++;
this.checkFlush();
});
}
/**
* Add a read operation to be coalesced
*/
async read(key) {
return new Promise((resolve, reject) => {
// Check if we already have a pending read for this key
const existing = this.readQueue.get(key);
if (existing && existing.length > 0) {
// Coalesce with existing read
existing.push({
type: 'read',
key,
resolve,
reject,
timestamp: Date.now()
});
this.stats.deduplicated++;
}
else {
// New read operation
this.readQueue.set(key, [{
type: 'read',
key,
resolve,
reject,
timestamp: Date.now()
}]);
}
this.stats.totalOperations++;
this.checkFlush();
});
}
/**
* Add a delete operation to be coalesced
*/
async delete(key) {
return new Promise((resolve, reject) => {
// Cancel any pending writes for this key
if (this.writeQueue.has(key)) {
const writes = this.writeQueue.get(key);
writes.forEach(op => op.reject(new Error('Cancelled by delete')));
this.writeQueue.delete(key);
this.stats.deduplicated += writes.length;
}
// Cancel any pending reads for this key
if (this.readQueue.has(key)) {
const reads = this.readQueue.get(key);
reads.forEach(op => op.resolve(null)); // Return null for deleted items
this.readQueue.delete(key);
this.stats.deduplicated += reads.length;
}
// Check if we already have a pending delete
const existing = this.deleteQueue.get(key);
if (existing && existing.length > 0) {
// Coalesce with existing delete
existing.push({
type: 'delete',
key,
resolve,
reject,
timestamp: Date.now()
});
this.stats.deduplicated++;
}
else {
// New delete operation
this.deleteQueue.set(key, [{
type: 'delete',
key,
resolve,
reject,
timestamp: Date.now()
}]);
}
this.stats.totalOperations++;
this.checkFlush();
});
}
/**
* Check if we should flush the queues
*/
checkFlush() {
const totalSize = this.writeQueue.size + this.readQueue.size + this.deleteQueue.size;
const now = Date.now();
const age = now - this.lastFlush;
// Immediate flush conditions
if (totalSize >= this.maxBatchSize) {
this.flush('size_limit');
return;
}
// Age-based flush
if (age >= this.maxBatchAge && totalSize >= this.minBatchSize) {
this.flush('age_limit');
return;
}
// Schedule a flush if not already scheduled
if (!this.flushTimer && totalSize > 0) {
const delay = Math.max(10, this.maxBatchAge - age);
this.flushTimer = setTimeout(() => {
this.flush('timer');
}, delay);
}
}
/**
* Flush all queued operations
*/
async flush(reason = 'manual') {
// Clear timer
if (this.flushTimer) {
clearTimeout(this.flushTimer);
this.flushTimer = null;
}
// Collect all operations into a single batch
const batch = [];
// Process deletes first (highest priority)
this.deleteQueue.forEach((ops) => {
// Only take the first operation per key (others are duplicates)
if (ops.length > 0) {
batch.push(ops[0]);
this.stats.coalescedOperations += ops.length;
}
});
// Then writes
this.writeQueue.forEach((ops) => {
if (ops.length > 0) {
// Use the last write (most recent data)
const lastWrite = ops[ops.length - 1];
batch.push(lastWrite);
this.stats.coalescedOperations += ops.length;
}
});
// Then reads
this.readQueue.forEach((ops) => {
if (ops.length > 0) {
batch.push(ops[0]);
this.stats.coalescedOperations += ops.length;
}
});
// Clear queues
const allOps = [
...Array.from(this.deleteQueue.values()).flat(),
...Array.from(this.writeQueue.values()).flat(),
...Array.from(this.readQueue.values()).flat()
];
this.deleteQueue.clear();
this.writeQueue.clear();
this.readQueue.clear();
if (batch.length === 0) {
return;
}
// Update stats
this.stats.batchesProcessed++;
this.stats.averageBatchSize =
(this.stats.averageBatchSize * (this.stats.batchesProcessed - 1) + batch.length) /
this.stats.batchesProcessed;
this.logger.debug(`Flushing batch of ${batch.length} operations (${allOps.length} total) - reason: ${reason}`);
// Process the batch
try {
await this.processor(batch);
// Resolve all promises
allOps.forEach(op => {
if (op.type === 'read') {
// Find the result for this read
const result = batch.find(b => b.key === op.key && b.type === 'read');
op.resolve(result?.data || null);
}
else {
op.resolve(undefined);
}
});
}
catch (error) {
// Reject all promises
allOps.forEach(op => op.reject(error));
this.logger.error('Batch processing failed:', error);
}
this.lastFlush = Date.now();
}
/**
* Get current statistics
*/
getStats() {
return { ...this.stats };
}
/**
* Get current queue sizes
*/
getQueueSizes() {
return {
writes: this.writeQueue.size,
reads: this.readQueue.size,
deletes: this.deleteQueue.size,
total: this.writeQueue.size + this.readQueue.size + this.deleteQueue.size
};
}
/**
* Adjust batch parameters based on load
*/
adjustParameters(pending) {
if (pending > 10000) {
// Extreme load - batch aggressively
this.maxBatchSize = 500;
this.maxBatchAge = 50;
this.minBatchSize = 50;
}
else if (pending > 1000) {
// High load - larger batches
this.maxBatchSize = 200;
this.maxBatchAge = 100;
this.minBatchSize = 20;
}
else if (pending > 100) {
// Moderate load
this.maxBatchSize = 100;
this.maxBatchAge = 200;
this.minBatchSize = 10;
}
else {
// Low load - optimize for latency
this.maxBatchSize = 50;
this.maxBatchAge = 500;
this.minBatchSize = 5;
}
}
/**
* Force immediate flush of all operations
*/
async forceFlush() {
await this.flush('force');
}
}
// Global coalescer instances by storage type
const coalescers = new Map();
/**
* Get or create a coalescer for a storage instance
*/
export function getCoalescer(storageId, processor) {
if (!coalescers.has(storageId)) {
coalescers.set(storageId, new RequestCoalescer(processor));
}
return coalescers.get(storageId);
}
/**
* Clear all coalescers
*/
export function clearCoalescers() {
coalescers.clear();
}
//# sourceMappingURL=requestCoalescer.js.map

1
dist/utils/requestCoalescer.js.map vendored Normal file
View file

File diff suppressed because one or more lines are too long

93
dist/utils/searchCache.d.ts vendored Normal file
View file

@ -0,0 +1,93 @@
/**
* SearchCache - Caches search results for improved performance
*/
import { SearchResult } from '../coreTypes.js';
export interface CacheEntry<T = any> {
results: SearchResult<T>[];
timestamp: number;
hits: number;
}
export interface SearchCacheConfig {
maxAge?: number;
maxSize?: number;
enabled?: boolean;
hitCountWeight?: number;
}
export declare class SearchCache<T = any> {
private cache;
private maxAge;
private maxSize;
private enabled;
private hitCountWeight;
private hits;
private misses;
private evictions;
constructor(config?: SearchCacheConfig);
/**
* Generate cache key from search parameters
*/
getCacheKey(query: any, k: number, options?: Record<string, any>): string;
/**
* Get cached results if available and not expired
*/
get(key: string): SearchResult<T>[] | null;
/**
* Cache search results
*/
set(key: string, results: SearchResult<T>[]): void;
/**
* Evict the oldest entry based on timestamp and hit count
*/
private evictOldest;
/**
* Clear all cached results
*/
clear(): void;
/**
* Invalidate cache entries that might be affected by data changes
*/
invalidate(pattern?: string | RegExp): void;
/**
* Smart invalidation for real-time data updates
* Only clears cache if it's getting stale or if data changes significantly
*/
invalidateOnDataChange(changeType?: 'add' | 'update' | 'delete'): void;
/**
* Check if cache entries have expired and remove them
* This is especially important in distributed scenarios where
* real-time updates might be delayed or missed
*/
cleanupExpiredEntries(): number;
/**
* Get cache statistics
*/
getStats(): {
hits: number;
misses: number;
evictions: number;
hitRate: number;
size: number;
maxSize: number;
enabled: boolean;
};
/**
* Enable or disable caching
*/
setEnabled(enabled: boolean): void;
/**
* Get memory usage estimate in bytes
*/
getMemoryUsage(): number;
/**
* Get current cache configuration
*/
getConfig(): SearchCacheConfig;
/**
* Update cache configuration dynamically
*/
updateConfig(newConfig: Partial<SearchCacheConfig>): void;
/**
* Evict entries if cache exceeds maxSize
*/
private evictIfNeeded;
}

248
dist/utils/searchCache.js vendored Normal file
View file

@ -0,0 +1,248 @@
/**
* SearchCache - Caches search results for improved performance
*/
export class SearchCache {
constructor(config = {}) {
this.cache = new Map();
// Cache statistics
this.hits = 0;
this.misses = 0;
this.evictions = 0;
this.maxAge = config.maxAge ?? 5 * 60 * 1000; // 5 minutes
this.maxSize = config.maxSize ?? 100;
this.enabled = config.enabled ?? true;
this.hitCountWeight = config.hitCountWeight ?? 0.3;
}
/**
* Generate cache key from search parameters
*/
getCacheKey(query, k, options = {}) {
// Create a normalized key that ignores order of options
const normalizedOptions = Object.keys(options)
.sort()
.reduce((acc, key) => {
// Skip cache-related options
if (key === 'skipCache' || key === 'useStreaming')
return acc;
acc[key] = options[key];
return acc;
}, {});
return JSON.stringify({
query: typeof query === 'object' ? JSON.stringify(query) : query,
k,
...normalizedOptions
});
}
/**
* Get cached results if available and not expired
*/
get(key) {
if (!this.enabled)
return null;
const entry = this.cache.get(key);
if (!entry) {
this.misses++;
return null;
}
// Check if expired
if (Date.now() - entry.timestamp > this.maxAge) {
this.cache.delete(key);
this.misses++;
return null;
}
// Update hit count and statistics
entry.hits++;
this.hits++;
return entry.results;
}
/**
* Cache search results
*/
set(key, results) {
if (!this.enabled)
return;
// Evict if cache is full
if (this.cache.size >= this.maxSize) {
this.evictOldest();
}
this.cache.set(key, {
results: [...results], // Deep copy to prevent mutations
timestamp: Date.now(),
hits: 0
});
}
/**
* Evict the oldest entry based on timestamp and hit count
*/
evictOldest() {
let oldestKey = null;
let oldestScore = Infinity;
const now = Date.now();
for (const [key, entry] of this.cache.entries()) {
// Score combines age and inverse hit count
const age = now - entry.timestamp;
const hitScore = entry.hits > 0 ? 1 / entry.hits : 1;
const score = age + (hitScore * this.hitCountWeight * this.maxAge);
if (score < oldestScore) {
oldestScore = score;
oldestKey = key;
}
}
if (oldestKey) {
this.cache.delete(oldestKey);
this.evictions++;
}
}
/**
* Clear all cached results
*/
clear() {
this.cache.clear();
this.hits = 0;
this.misses = 0;
this.evictions = 0;
}
/**
* Invalidate cache entries that might be affected by data changes
*/
invalidate(pattern) {
if (!pattern) {
this.clear();
return;
}
const keysToDelete = [];
for (const key of this.cache.keys()) {
const shouldDelete = typeof pattern === 'string'
? key.includes(pattern)
: pattern.test(key);
if (shouldDelete) {
keysToDelete.push(key);
}
}
keysToDelete.forEach(key => this.cache.delete(key));
}
/**
* Smart invalidation for real-time data updates
* Only clears cache if it's getting stale or if data changes significantly
*/
invalidateOnDataChange(changeType) {
// For now, clear all caches on data changes to ensure consistency
// In the future, we could implement more sophisticated invalidation
// based on the type of change and affected data
this.clear();
}
/**
* Check if cache entries have expired and remove them
* This is especially important in distributed scenarios where
* real-time updates might be delayed or missed
*/
cleanupExpiredEntries() {
const now = Date.now();
const keysToDelete = [];
for (const [key, entry] of this.cache.entries()) {
if (now - entry.timestamp > this.maxAge) {
keysToDelete.push(key);
}
}
keysToDelete.forEach(key => this.cache.delete(key));
return keysToDelete.length;
}
/**
* Get cache statistics
*/
getStats() {
const total = this.hits + this.misses;
return {
hits: this.hits,
misses: this.misses,
evictions: this.evictions,
hitRate: total > 0 ? this.hits / total : 0,
size: this.cache.size,
maxSize: this.maxSize,
enabled: this.enabled
};
}
/**
* Enable or disable caching
*/
setEnabled(enabled) {
Object.defineProperty(this, 'enabled', { value: enabled, writable: false });
if (!enabled) {
this.clear();
}
}
/**
* Get memory usage estimate in bytes
*/
getMemoryUsage() {
let totalSize = 0;
for (const [key, entry] of this.cache.entries()) {
// Estimate key size
totalSize += key.length * 2; // UTF-16 characters
// Estimate entry size
totalSize += JSON.stringify(entry.results).length * 2;
totalSize += 16; // timestamp + hits (8 bytes each)
}
return totalSize;
}
/**
* Get current cache configuration
*/
getConfig() {
return {
enabled: this.enabled,
maxSize: this.maxSize,
maxAge: this.maxAge,
hitCountWeight: this.hitCountWeight
};
}
/**
* Update cache configuration dynamically
*/
updateConfig(newConfig) {
if (newConfig.enabled !== undefined) {
this.enabled = newConfig.enabled;
}
if (newConfig.maxSize !== undefined) {
this.maxSize = newConfig.maxSize;
// Trigger eviction if current size exceeds new limit
this.evictIfNeeded();
}
if (newConfig.maxAge !== undefined) {
this.maxAge = newConfig.maxAge;
// Clean up entries that are now expired with new TTL
this.cleanupExpiredEntries();
}
if (newConfig.hitCountWeight !== undefined) {
this.hitCountWeight = newConfig.hitCountWeight;
}
}
/**
* Evict entries if cache exceeds maxSize
*/
evictIfNeeded() {
if (this.cache.size <= this.maxSize) {
return;
}
// Calculate eviction score for each entry (same logic as existing eviction)
const entries = Array.from(this.cache.entries()).map(([key, entry]) => {
const age = Date.now() - entry.timestamp;
const hitCount = entry.hits;
// Eviction score: lower is more likely to be evicted
// Combines age and hit count (weighted by hitCountWeight)
const ageScore = age / this.maxAge;
const hitScore = 1 / (hitCount + 1); // Inverse of hits (more hits = lower score)
const score = ageScore * (1 - this.hitCountWeight) + hitScore * this.hitCountWeight;
return { key, entry, score };
});
// Sort by score (lowest first - these will be evicted)
entries.sort((a, b) => a.score - b.score);
// Evict entries until we're under the limit
const toEvict = entries.slice(0, this.cache.size - this.maxSize);
toEvict.forEach(({ key }) => {
this.cache.delete(key);
this.evictions++;
});
}
}
//# sourceMappingURL=searchCache.js.map

1
dist/utils/searchCache.js.map vendored Normal file
View file

File diff suppressed because one or more lines are too long

28
dist/utils/statistics.d.ts vendored Normal file
View file

@ -0,0 +1,28 @@
/**
* Utility functions for retrieving statistics from Brainy
*/
import { BrainyData } from '../brainyData.js';
/**
* Get statistics about the current state of a BrainyData instance
* This function provides access to statistics at the root level of the library
*
* @param instance A BrainyData instance to get statistics from
* @param options Additional options for retrieving statistics
* @returns Object containing counts of nouns, verbs, metadata entries, and HNSW index size
* @throws Error if the instance is not provided or if statistics retrieval fails
*/
export declare function getStatistics(instance: BrainyData, options?: {
service?: string | string[];
}): Promise<{
nounCount: number;
verbCount: number;
metadataCount: number;
hnswIndexSize: number;
serviceBreakdown?: {
[service: string]: {
nounCount: number;
verbCount: number;
metadataCount: number;
};
};
}>;

25
dist/utils/statistics.js vendored Normal file
View file

@ -0,0 +1,25 @@
/**
* Utility functions for retrieving statistics from Brainy
*/
/**
* Get statistics about the current state of a BrainyData instance
* This function provides access to statistics at the root level of the library
*
* @param instance A BrainyData instance to get statistics from
* @param options Additional options for retrieving statistics
* @returns Object containing counts of nouns, verbs, metadata entries, and HNSW index size
* @throws Error if the instance is not provided or if statistics retrieval fails
*/
export async function getStatistics(instance, options = {}) {
if (!instance) {
throw new Error('BrainyData instance must be provided to getStatistics');
}
try {
return await instance.getStatistics(options);
}
catch (error) {
console.error('Failed to get statistics:', error);
throw new Error(`Failed to get statistics: ${error}`);
}
}
//# sourceMappingURL=statistics.js.map

1
dist/utils/statistics.js.map vendored Normal file
View file

@ -0,0 +1 @@
{"version":3,"file":"statistics.js","sourceRoot":"","sources":["../../src/utils/statistics.ts"],"names":[],"mappings":"AAAA;;GAEG;AAIH;;;;;;;;GAQG;AACH,MAAM,CAAC,KAAK,UAAU,aAAa,CAC/B,QAAoB,EACpB,UAEI,EAAE;IAcN,IAAI,CAAC,QAAQ,EAAE,CAAC;QACZ,MAAM,IAAI,KAAK,CAAC,uDAAuD,CAAC,CAAA;IAC5E,CAAC;IAED,IAAI,CAAC;QACD,OAAO,MAAM,QAAQ,CAAC,aAAa,CAAC,OAAO,CAAC,CAAA;IAChD,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACb,OAAO,CAAC,KAAK,CAAC,2BAA2B,EAAE,KAAK,CAAC,CAAA;QACjD,MAAM,IAAI,KAAK,CAAC,6BAA6B,KAAK,EAAE,CAAC,CAAA;IACzD,CAAC;AACL,CAAC"}

92
dist/utils/statisticsCollector.d.ts vendored Normal file
View file

@ -0,0 +1,92 @@
/**
* Lightweight statistics collector for Brainy
* Designed to have minimal performance impact even with millions of entries
*/
import { StatisticsData } from '../coreTypes.js';
export declare class StatisticsCollector {
private contentTypes;
private oldestTimestamp;
private newestTimestamp;
private updateTimestamps;
private searchMetrics;
private verbTypes;
private storageSizeCache;
private throttlingMetrics;
private readonly MAX_TIMESTAMPS;
private readonly MAX_SEARCH_TERMS;
private readonly SIZE_UPDATE_INTERVAL;
/**
* Track content type (very lightweight)
*/
trackContentType(type: string): void;
/**
* Track data update timestamp (lightweight)
*/
trackUpdate(timestamp?: number): void;
/**
* Track search performance (lightweight)
*/
trackSearch(searchTerm: string, durationMs: number): void;
/**
* Track verb type (lightweight)
*/
trackVerbType(type: string): void;
/**
* Update storage size estimates (called periodically, not on every operation)
*/
updateStorageSizes(sizes: {
nouns: number;
verbs: number;
metadata: number;
index: number;
}): void;
/**
* Track a throttling event
*/
trackThrottlingEvent(reason: string, service?: string): void;
/**
* Clear throttling state after successful operations
*/
clearThrottlingState(): void;
/**
* Track delayed operation
*/
trackDelayedOperation(delayMs: number): void;
/**
* Track retried operation
*/
trackRetriedOperation(): void;
/**
* Track operation failed due to throttling
*/
trackFailedDueToThrottling(): void;
/**
* Update throttling metrics from storage adapter
*/
updateThrottlingMetrics(metrics: {
currentlyThrottled: boolean;
lastThrottleTime: number;
consecutiveThrottleEvents: number;
currentBackoffMs: number;
totalThrottleEvents: number;
throttleEventsByHour: number[];
throttleReasons: Record<string, number>;
delayedOperations: number;
retriedOperations: number;
failedDueToThrottling: number;
totalDelayMs: number;
}): void;
/**
* Get comprehensive statistics
*/
getStatistics(): Partial<StatisticsData>;
/**
* Merge statistics from storage (for distributed systems)
*/
mergeFromStorage(stored: Partial<StatisticsData>): void;
/**
* Reset statistics (for testing)
*/
reset(): void;
private pruneSearchTerms;
}

366
dist/utils/statisticsCollector.js vendored Normal file
View file

@ -0,0 +1,366 @@
/**
* Lightweight statistics collector for Brainy
* Designed to have minimal performance impact even with millions of entries
*/
export class StatisticsCollector {
constructor() {
// Content type tracking (lightweight counters)
this.contentTypes = new Map();
// Data freshness tracking (only track timestamps, not full data)
this.oldestTimestamp = Date.now();
this.newestTimestamp = Date.now();
this.updateTimestamps = [];
// Search performance tracking (rolling window)
this.searchMetrics = {
totalSearches: 0,
totalSearchTimeMs: 0,
searchTimestamps: [],
topSearchTerms: new Map()
};
// Verb type tracking
this.verbTypes = new Map();
// Storage size estimates (updated periodically, not on every operation)
this.storageSizeCache = {
lastUpdated: 0,
sizes: {
nouns: 0,
verbs: 0,
metadata: 0,
index: 0
}
};
// Throttling metrics
this.throttlingMetrics = {
currentlyThrottled: false,
lastThrottleTime: 0,
consecutiveThrottleEvents: 0,
currentBackoffMs: 1000,
totalThrottleEvents: 0,
throttleEventsByHour: new Array(24).fill(0),
throttleReasons: new Map(),
delayedOperations: 0,
retriedOperations: 0,
failedDueToThrottling: 0,
totalDelayMs: 0,
serviceThrottling: new Map()
};
this.MAX_TIMESTAMPS = 1000; // Keep last 1000 timestamps
this.MAX_SEARCH_TERMS = 100; // Track top 100 search terms
this.SIZE_UPDATE_INTERVAL = 60000; // Update sizes every minute
}
/**
* Track content type (very lightweight)
*/
trackContentType(type) {
this.contentTypes.set(type, (this.contentTypes.get(type) || 0) + 1);
}
/**
* Track data update timestamp (lightweight)
*/
trackUpdate(timestamp) {
const ts = timestamp || Date.now();
// Update oldest/newest
if (ts < this.oldestTimestamp)
this.oldestTimestamp = ts;
if (ts > this.newestTimestamp)
this.newestTimestamp = ts;
// Add to rolling window
this.updateTimestamps.push({ timestamp: ts, count: 1 });
// Keep window size limited
if (this.updateTimestamps.length > this.MAX_TIMESTAMPS) {
this.updateTimestamps.shift();
}
}
/**
* Track search performance (lightweight)
*/
trackSearch(searchTerm, durationMs) {
this.searchMetrics.totalSearches++;
this.searchMetrics.totalSearchTimeMs += durationMs;
// Add to rolling window
this.searchMetrics.searchTimestamps.push({
timestamp: Date.now(),
count: 1
});
// Keep window size limited
if (this.searchMetrics.searchTimestamps.length > this.MAX_TIMESTAMPS) {
this.searchMetrics.searchTimestamps.shift();
}
// Track search term (limit to top N)
const termCount = (this.searchMetrics.topSearchTerms.get(searchTerm) || 0) + 1;
this.searchMetrics.topSearchTerms.set(searchTerm, termCount);
// Prune if too many terms
if (this.searchMetrics.topSearchTerms.size > this.MAX_SEARCH_TERMS * 2) {
this.pruneSearchTerms();
}
}
/**
* Track verb type (lightweight)
*/
trackVerbType(type) {
this.verbTypes.set(type, (this.verbTypes.get(type) || 0) + 1);
}
/**
* Update storage size estimates (called periodically, not on every operation)
*/
updateStorageSizes(sizes) {
this.storageSizeCache = {
lastUpdated: Date.now(),
sizes
};
}
/**
* Track a throttling event
*/
trackThrottlingEvent(reason, service) {
this.throttlingMetrics.currentlyThrottled = true;
this.throttlingMetrics.consecutiveThrottleEvents++;
this.throttlingMetrics.lastThrottleTime = Date.now();
this.throttlingMetrics.totalThrottleEvents++;
// Track by hour
const hourIndex = new Date().getHours();
this.throttlingMetrics.throttleEventsByHour[hourIndex]++;
// Track reason
const reasonCount = this.throttlingMetrics.throttleReasons.get(reason) || 0;
this.throttlingMetrics.throttleReasons.set(reason, reasonCount + 1);
// Track service-level throttling
if (service) {
const serviceInfo = this.throttlingMetrics.serviceThrottling.get(service) || {
throttleCount: 0,
lastThrottle: 0,
status: 'normal'
};
serviceInfo.throttleCount++;
serviceInfo.lastThrottle = Date.now();
serviceInfo.status = 'throttled';
this.throttlingMetrics.serviceThrottling.set(service, serviceInfo);
}
// Exponential backoff
this.throttlingMetrics.currentBackoffMs = Math.min(this.throttlingMetrics.currentBackoffMs * 2, 30000 // Max 30 seconds
);
}
/**
* Clear throttling state after successful operations
*/
clearThrottlingState() {
if (this.throttlingMetrics.consecutiveThrottleEvents > 0) {
this.throttlingMetrics.consecutiveThrottleEvents = 0;
this.throttlingMetrics.currentBackoffMs = 1000; // Reset to initial backoff
this.throttlingMetrics.currentlyThrottled = false;
// Update service statuses
for (const [, info] of this.throttlingMetrics.serviceThrottling) {
if (info.status === 'throttled') {
info.status = 'recovering';
}
else if (info.status === 'recovering') {
const timeSinceThrottle = Date.now() - info.lastThrottle;
if (timeSinceThrottle > 60000) { // 1 minute recovery period
info.status = 'normal';
}
}
}
}
}
/**
* Track delayed operation
*/
trackDelayedOperation(delayMs) {
this.throttlingMetrics.delayedOperations++;
this.throttlingMetrics.totalDelayMs += delayMs;
}
/**
* Track retried operation
*/
trackRetriedOperation() {
this.throttlingMetrics.retriedOperations++;
}
/**
* Track operation failed due to throttling
*/
trackFailedDueToThrottling() {
this.throttlingMetrics.failedDueToThrottling++;
}
/**
* Update throttling metrics from storage adapter
*/
updateThrottlingMetrics(metrics) {
this.throttlingMetrics.currentlyThrottled = metrics.currentlyThrottled;
this.throttlingMetrics.lastThrottleTime = metrics.lastThrottleTime;
this.throttlingMetrics.consecutiveThrottleEvents = metrics.consecutiveThrottleEvents;
this.throttlingMetrics.currentBackoffMs = metrics.currentBackoffMs;
this.throttlingMetrics.totalThrottleEvents = metrics.totalThrottleEvents;
this.throttlingMetrics.throttleEventsByHour = [...metrics.throttleEventsByHour];
// Update throttle reasons map
this.throttlingMetrics.throttleReasons.clear();
for (const [reason, count] of Object.entries(metrics.throttleReasons)) {
this.throttlingMetrics.throttleReasons.set(reason, count);
}
this.throttlingMetrics.delayedOperations = metrics.delayedOperations;
this.throttlingMetrics.retriedOperations = metrics.retriedOperations;
this.throttlingMetrics.failedDueToThrottling = metrics.failedDueToThrottling;
this.throttlingMetrics.totalDelayMs = metrics.totalDelayMs;
}
/**
* Get comprehensive statistics
*/
getStatistics() {
const now = Date.now();
const hourAgo = now - 3600000;
const dayAgo = now - 86400000;
const weekAgo = now - 604800000;
const monthAgo = now - 2592000000;
// Calculate data freshness
const updatesLastHour = this.updateTimestamps.filter(t => t.timestamp > hourAgo).length;
const updatesLastDay = this.updateTimestamps.filter(t => t.timestamp > dayAgo).length;
// Calculate age distribution
const ageDistribution = {
last24h: 0,
last7d: 0,
last30d: 0,
older: 0
};
// Estimate based on update patterns (not scanning all data)
const totalUpdates = this.updateTimestamps.length;
if (totalUpdates > 0) {
const recentUpdates = this.updateTimestamps.filter(t => t.timestamp > dayAgo).length;
const weekUpdates = this.updateTimestamps.filter(t => t.timestamp > weekAgo).length;
const monthUpdates = this.updateTimestamps.filter(t => t.timestamp > monthAgo).length;
ageDistribution.last24h = Math.round((recentUpdates / totalUpdates) * 100);
ageDistribution.last7d = Math.round(((weekUpdates - recentUpdates) / totalUpdates) * 100);
ageDistribution.last30d = Math.round(((monthUpdates - weekUpdates) / totalUpdates) * 100);
ageDistribution.older = 100 - ageDistribution.last24h - ageDistribution.last7d - ageDistribution.last30d;
}
// Calculate search metrics
const searchesLastHour = this.searchMetrics.searchTimestamps.filter(t => t.timestamp > hourAgo).length;
const searchesLastDay = this.searchMetrics.searchTimestamps.filter(t => t.timestamp > dayAgo).length;
const avgSearchTime = this.searchMetrics.totalSearches > 0
? this.searchMetrics.totalSearchTimeMs / this.searchMetrics.totalSearches
: 0;
// Get top search terms
const topSearchTerms = Array.from(this.searchMetrics.topSearchTerms.entries())
.sort((a, b) => b[1] - a[1])
.slice(0, 10)
.map(([term]) => term);
// Calculate storage metrics
const totalSize = Object.values(this.storageSizeCache.sizes).reduce((a, b) => a + b, 0);
// Calculate average delay for throttling
const averageDelayMs = this.throttlingMetrics.delayedOperations > 0
? this.throttlingMetrics.totalDelayMs / this.throttlingMetrics.delayedOperations
: 0;
// Convert service throttling map to record
const serviceThrottlingRecord = {};
for (const [service, info] of this.throttlingMetrics.serviceThrottling) {
serviceThrottlingRecord[service] = {
throttleCount: info.throttleCount,
lastThrottle: new Date(info.lastThrottle).toISOString(),
status: info.status
};
}
return {
contentTypes: Object.fromEntries(this.contentTypes),
dataFreshness: {
oldestEntry: new Date(this.oldestTimestamp).toISOString(),
newestEntry: new Date(this.newestTimestamp).toISOString(),
updatesLastHour,
updatesLastDay,
ageDistribution
},
storageMetrics: {
totalSizeBytes: totalSize,
nounsSizeBytes: this.storageSizeCache.sizes.nouns,
verbsSizeBytes: this.storageSizeCache.sizes.verbs,
metadataSizeBytes: this.storageSizeCache.sizes.metadata,
indexSizeBytes: this.storageSizeCache.sizes.index
},
searchMetrics: {
totalSearches: this.searchMetrics.totalSearches,
averageSearchTimeMs: avgSearchTime,
searchesLastHour,
searchesLastDay,
topSearchTerms
},
verbStatistics: {
totalVerbs: Array.from(this.verbTypes.values()).reduce((a, b) => a + b, 0),
verbTypes: Object.fromEntries(this.verbTypes),
averageConnectionsPerVerb: 2 // Verbs connect 2 nouns
},
throttlingMetrics: {
storage: {
currentlyThrottled: this.throttlingMetrics.currentlyThrottled,
lastThrottleTime: this.throttlingMetrics.lastThrottleTime > 0
? new Date(this.throttlingMetrics.lastThrottleTime).toISOString()
: undefined,
consecutiveThrottleEvents: this.throttlingMetrics.consecutiveThrottleEvents,
currentBackoffMs: this.throttlingMetrics.currentBackoffMs,
totalThrottleEvents: this.throttlingMetrics.totalThrottleEvents,
throttleEventsByHour: [...this.throttlingMetrics.throttleEventsByHour],
throttleReasons: Object.fromEntries(this.throttlingMetrics.throttleReasons)
},
operationImpact: {
delayedOperations: this.throttlingMetrics.delayedOperations,
retriedOperations: this.throttlingMetrics.retriedOperations,
failedDueToThrottling: this.throttlingMetrics.failedDueToThrottling,
averageDelayMs,
totalDelayMs: this.throttlingMetrics.totalDelayMs
},
serviceThrottling: Object.keys(serviceThrottlingRecord).length > 0
? serviceThrottlingRecord
: undefined
}
};
}
/**
* Merge statistics from storage (for distributed systems)
*/
mergeFromStorage(stored) {
// Merge content types
if (stored.contentTypes) {
for (const [type, count] of Object.entries(stored.contentTypes)) {
this.contentTypes.set(type, count);
}
}
// Merge verb types
if (stored.verbStatistics?.verbTypes) {
for (const [type, count] of Object.entries(stored.verbStatistics.verbTypes)) {
this.verbTypes.set(type, count);
}
}
// Merge search metrics
if (stored.searchMetrics) {
this.searchMetrics.totalSearches = stored.searchMetrics.totalSearches || 0;
this.searchMetrics.totalSearchTimeMs = (stored.searchMetrics.averageSearchTimeMs || 0) * this.searchMetrics.totalSearches;
}
// Merge data freshness
if (stored.dataFreshness) {
this.oldestTimestamp = new Date(stored.dataFreshness.oldestEntry).getTime();
this.newestTimestamp = new Date(stored.dataFreshness.newestEntry).getTime();
}
}
/**
* Reset statistics (for testing)
*/
reset() {
this.contentTypes.clear();
this.verbTypes.clear();
this.updateTimestamps = [];
this.searchMetrics = {
totalSearches: 0,
totalSearchTimeMs: 0,
searchTimestamps: [],
topSearchTerms: new Map()
};
this.oldestTimestamp = Date.now();
this.newestTimestamp = Date.now();
}
pruneSearchTerms() {
// Keep only top N search terms
const sorted = Array.from(this.searchMetrics.topSearchTerms.entries())
.sort((a, b) => b[1] - a[1])
.slice(0, this.MAX_SEARCH_TERMS);
this.searchMetrics.topSearchTerms.clear();
for (const [term, count] of sorted) {
this.searchMetrics.topSearchTerms.set(term, count);
}
}
}
//# sourceMappingURL=statisticsCollector.js.map

1
dist/utils/statisticsCollector.js.map vendored Normal file
View file

File diff suppressed because one or more lines are too long

7
dist/utils/textEncoding.d.ts vendored Normal file
View file

@ -0,0 +1,7 @@
/**
* Apply TextEncoder/TextDecoder patches for Node.js compatibility
* Simplified version for Transformers.js/ONNX Runtime
*/
export declare function applyTensorFlowPatch(): Promise<void>;
export declare function getTextEncoder(): TextEncoder;
export declare function getTextDecoder(): TextDecoder;

66
dist/utils/textEncoding.js vendored Normal file
View file

@ -0,0 +1,66 @@
import { isNode } from './environment.js';
// Simplified TextEncoder/TextDecoder utilities for Node.js compatibility
// No longer needs complex TensorFlow.js patches - only basic TextEncoder/TextDecoder
/**
* Flag to track if the patch has been applied
*/
let patchApplied = false;
/**
* Apply TextEncoder/TextDecoder patches for Node.js compatibility
* Simplified version for Transformers.js/ONNX Runtime
*/
export async function applyTensorFlowPatch() {
// Apply patches for all non-browser environments that might need TextEncoder/TextDecoder
const isBrowserEnv = typeof window !== 'undefined' && typeof document !== 'undefined';
if (isBrowserEnv || patchApplied) {
return; // Browser environments don't need these patches, and don't patch twice
}
if (!isNode()) {
return; // Only patch Node.js environments
}
try {
console.log('Brainy: Applying TextEncoder/TextDecoder patch for Node.js');
// Get the appropriate global object
const globalObj = (() => {
if (typeof globalThis !== 'undefined')
return globalThis;
if (typeof global !== 'undefined')
return global;
return {};
})();
// Make sure TextEncoder and TextDecoder are available globally
if (!globalObj.TextEncoder) {
globalObj.TextEncoder = TextEncoder;
}
if (!globalObj.TextDecoder) {
globalObj.TextDecoder = TextDecoder;
}
// Also set them on the global object for older code
if (typeof global !== 'undefined') {
if (!global.TextEncoder) {
global.TextEncoder = TextEncoder;
}
if (!global.TextDecoder) {
global.TextDecoder = TextDecoder;
}
}
patchApplied = true;
console.log('Brainy: TextEncoder/TextDecoder patches applied successfully');
}
catch (error) {
console.warn('Brainy: Failed to apply TextEncoder/TextDecoder patch:', error);
}
}
export function getTextEncoder() {
return new TextEncoder();
}
export function getTextDecoder() {
return new TextDecoder();
}
// Apply patch immediately if in Node.js
if (isNode()) {
applyTensorFlowPatch().catch((error) => {
console.warn('Failed to apply TextEncoder/TextDecoder patch at module load:', error);
});
}
//# sourceMappingURL=textEncoding.js.map

1
dist/utils/textEncoding.js.map vendored Normal file
View file

@ -0,0 +1 @@
{"version":3,"file":"textEncoding.js","sourceRoot":"","sources":["../../src/utils/textEncoding.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,kBAAkB,CAAA;AAEzC,yEAAyE;AACzE,qFAAqF;AAErF;;GAEG;AACH,IAAI,YAAY,GAAG,KAAK,CAAA;AAExB;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,oBAAoB;IACxC,yFAAyF;IACzF,MAAM,YAAY,GAAG,OAAO,MAAM,KAAK,WAAW,IAAI,OAAO,QAAQ,KAAK,WAAW,CAAA;IACrF,IAAI,YAAY,IAAI,YAAY,EAAE,CAAC;QACjC,OAAM,CAAC,uEAAuE;IAChF,CAAC;IAED,IAAI,CAAC,MAAM,EAAE,EAAE,CAAC;QACd,OAAM,CAAC,kCAAkC;IAC3C,CAAC;IAED,IAAI,CAAC;QACH,OAAO,CAAC,GAAG,CAAC,4DAA4D,CAAC,CAAA;QAEzE,oCAAoC;QACpC,MAAM,SAAS,GAAG,CAAC,GAAG,EAAE;YACtB,IAAI,OAAO,UAAU,KAAK,WAAW;gBAAE,OAAO,UAAU,CAAA;YACxD,IAAI,OAAO,MAAM,KAAK,WAAW;gBAAE,OAAO,MAAM,CAAA;YAChD,OAAO,EAAS,CAAA;QAClB,CAAC,CAAC,EAAE,CAAA;QAEJ,+DAA+D;QAC/D,IAAI,CAAC,SAAS,CAAC,WAAW,EAAE,CAAC;YAC3B,SAAS,CAAC,WAAW,GAAG,WAAW,CAAA;QACrC,CAAC;QACD,IAAI,CAAC,SAAS,CAAC,WAAW,EAAE,CAAC;YAC3B,SAAS,CAAC,WAAW,GAAG,WAAW,CAAA;QACrC,CAAC;QAED,oDAAoD;QACpD,IAAI,OAAO,MAAM,KAAK,WAAW,EAAE,CAAC;YAClC,IAAI,CAAC,MAAM,CAAC,WAAW,EAAE,CAAC;gBACxB,MAAM,CAAC,WAAW,GAAG,WAAW,CAAA;YAClC,CAAC;YACD,IAAI,CAAC,MAAM,CAAC,WAAW,EAAE,CAAC;gBACxB,MAAM,CAAC,WAAW,GAAG,WAAW,CAAA;YAClC,CAAC;QACH,CAAC;QAED,YAAY,GAAG,IAAI,CAAA;QACnB,OAAO,CAAC,GAAG,CAAC,8DAA8D,CAAC,CAAA;IAC7E,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,OAAO,CAAC,IAAI,CAAC,wDAAwD,EAAE,KAAK,CAAC,CAAA;IAC/E,CAAC;AACH,CAAC;AAED,MAAM,UAAU,cAAc;IAC5B,OAAO,IAAI,WAAW,EAAE,CAAA;AAC1B,CAAC;AAED,MAAM,UAAU,cAAc;IAC5B,OAAO,IAAI,WAAW,EAAE,CAAA;AAC1B,CAAC;AAED,wCAAwC;AACxC,IAAI,MAAM,EAAE,EAAE,CAAC;IACb,oBAAoB,EAAE,CAAC,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE;QACrC,OAAO,CAAC,IAAI,CAAC,+DAA+D,EAAE,KAAK,CAAC,CAAA;IACtF,CAAC,CAAC,CAAA;AACJ,CAAC"}

30
dist/utils/typeUtils.d.ts vendored Normal file
View file

@ -0,0 +1,30 @@
/**
* Type Utilities
*
* This module provides utility functions for working with the Brainy type system,
* particularly for accessing lists of noun and verb types.
*/
/**
* Returns an array of all available noun types
*
* @returns {string[]} Array of all noun type values
*/
export declare function getNounTypes(): string[];
/**
* Returns an array of all available verb types
*
* @returns {string[]} Array of all verb type values
*/
export declare function getVerbTypes(): string[];
/**
* Returns a map of noun type keys to their string values
*
* @returns {Record<string, string>} Map of noun type keys to values
*/
export declare function getNounTypeMap(): Record<string, string>;
/**
* Returns a map of verb type keys to their string values
*
* @returns {Record<string, string>} Map of verb type keys to values
*/
export declare function getVerbTypeMap(): Record<string, string>;

40
dist/utils/typeUtils.js vendored Normal file
View file

@ -0,0 +1,40 @@
/**
* Type Utilities
*
* This module provides utility functions for working with the Brainy type system,
* particularly for accessing lists of noun and verb types.
*/
import { NounType, VerbType } from '../types/graphTypes.js';
/**
* Returns an array of all available noun types
*
* @returns {string[]} Array of all noun type values
*/
export function getNounTypes() {
return Object.values(NounType);
}
/**
* Returns an array of all available verb types
*
* @returns {string[]} Array of all verb type values
*/
export function getVerbTypes() {
return Object.values(VerbType);
}
/**
* Returns a map of noun type keys to their string values
*
* @returns {Record<string, string>} Map of noun type keys to values
*/
export function getNounTypeMap() {
return { ...NounType };
}
/**
* Returns a map of verb type keys to their string values
*
* @returns {Record<string, string>} Map of verb type keys to values
*/
export function getVerbTypeMap() {
return { ...VerbType };
}
//# sourceMappingURL=typeUtils.js.map

1
dist/utils/typeUtils.js.map vendored Normal file
View file

@ -0,0 +1 @@
{"version":3,"file":"typeUtils.js","sourceRoot":"","sources":["../../src/utils/typeUtils.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,EAAE,QAAQ,EAAE,QAAQ,EAAE,MAAM,wBAAwB,CAAA;AAE3D;;;;GAIG;AACH,MAAM,UAAU,YAAY;IAC1B,OAAO,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAA;AAChC,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,YAAY;IAC1B,OAAO,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAA;AAChC,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,cAAc;IAC5B,OAAO,EAAE,GAAG,QAAQ,EAAE,CAAA;AACxB,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,cAAc;IAC5B,OAAO,EAAE,GAAG,QAAQ,EAAE,CAAA;AACxB,CAAC"}

17
dist/utils/version.d.ts vendored Normal file
View file

@ -0,0 +1,17 @@
/**
* Version utilities for Brainy
*/
/**
* Get the current Brainy package version
* @returns The current version string
*/
export declare function getBrainyVersion(): string;
/**
* Get version information for augmentation metadata
* @param service The service/augmentation name
* @returns Version metadata object
*/
export declare function getAugmentationVersion(service: string): {
augmentation: string;
version: string;
};

24
dist/utils/version.js vendored Normal file
View file

@ -0,0 +1,24 @@
/**
* Version utilities for Brainy
*/
// Package version - this should be updated during the build process
const BRAINY_VERSION = '0.41.0';
/**
* Get the current Brainy package version
* @returns The current version string
*/
export function getBrainyVersion() {
return BRAINY_VERSION;
}
/**
* Get version information for augmentation metadata
* @param service The service/augmentation name
* @returns Version metadata object
*/
export function getAugmentationVersion(service) {
return {
augmentation: service,
version: getBrainyVersion()
};
}
//# sourceMappingURL=version.js.map

1
dist/utils/version.js.map vendored Normal file
View file

@ -0,0 +1 @@
{"version":3,"file":"version.js","sourceRoot":"","sources":["../../src/utils/version.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,oEAAoE;AACpE,MAAM,cAAc,GAAG,QAAQ,CAAA;AAE/B;;;GAGG;AACH,MAAM,UAAU,gBAAgB;IAC9B,OAAO,cAAc,CAAA;AACvB,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,sBAAsB,CAAC,OAAe;IACpD,OAAO;QACL,YAAY,EAAE,OAAO;QACrB,OAAO,EAAE,gBAAgB,EAAE;KAC5B,CAAA;AACH,CAAC"}

17
dist/utils/workerUtils.d.ts vendored Normal file
View file

@ -0,0 +1,17 @@
/**
* Utility functions for executing functions in Worker Threads (Node.js) or Web Workers (Browser)
* This implementation leverages Node.js 24's improved Worker Threads API for better performance
*/
/**
* Execute a function in a separate thread
*
* @param fnString The function to execute as a string
* @param args The arguments to pass to the function
* @returns A promise that resolves with the result of the function
*/
export declare function executeInThread<T>(fnString: string, args: any): Promise<T>;
/**
* Clean up all worker pools
* This should be called when the application is shutting down
*/
export declare function cleanupWorkerPools(): void;

458
dist/utils/workerUtils.js vendored Normal file
View file

@ -0,0 +1,458 @@
/**
* Utility functions for executing functions in Worker Threads (Node.js) or Web Workers (Browser)
* This implementation leverages Node.js 24's improved Worker Threads API for better performance
*/
import { isBrowser, isNode } from './environment.js';
// Worker pool to reuse workers
const workerPool = new Map();
const MAX_POOL_SIZE = 4; // Adjust based on system capabilities
/**
* Execute a function in a separate thread
*
* @param fnString The function to execute as a string
* @param args The arguments to pass to the function
* @returns A promise that resolves with the result of the function
*/
export function executeInThread(fnString, args) {
if (isNode()) {
return executeInNodeWorker(fnString, args);
}
else if (isBrowser() && typeof window !== 'undefined' && window.Worker) {
return executeInWebWorker(fnString, args);
}
else {
// Fallback to main thread execution
try {
// Try different approaches to create a function from string
let fn;
try {
// First try with 'return' prefix
fn = new Function('return ' + fnString)();
}
catch (functionError) {
console.warn('Fallback: Error creating function with return syntax, trying alternative approaches', functionError);
try {
// Try wrapping in parentheses for function expressions
fn = new Function('return (' + fnString + ')')();
}
catch (wrapError) {
console.warn('Fallback: Error creating function with parentheses wrapping', wrapError);
try {
// Try direct approach for named functions
fn = new Function(fnString)();
}
catch (directError) {
console.warn('Fallback: Direct approach failed, trying with function wrapper', directError);
try {
// Try wrapping in a function that returns the function expression
fn = new Function('return function(args) { return (' + fnString + ')(args); }')();
}
catch (wrapperError) {
console.error('Fallback: All approaches to create function failed', wrapperError);
throw new Error('Failed to create function from string: ' +
functionError.message);
}
}
}
}
return Promise.resolve(fn(args));
}
catch (error) {
return Promise.reject(error);
}
}
}
/**
* Execute a function in a Node.js Worker Thread
* Optimized for Node.js 24 with improved Worker Threads performance
*/
function executeInNodeWorker(fnString, args) {
return new Promise((resolve, reject) => {
try {
// Dynamically import worker_threads (Node.js only)
import('node:worker_threads')
.then(({ Worker, isMainThread, parentPort, workerData }) => {
if (!isMainThread && parentPort) {
// We're inside a worker, execute the function
const fn = new Function('return ' + workerData.fnString)();
const result = fn(workerData.args);
parentPort.postMessage({ result });
return;
}
// Get a worker from the pool or create a new one
const workerId = `worker-${Math.random().toString(36).substring(2, 9)}`;
let worker;
if (workerPool.size < MAX_POOL_SIZE) {
// Create a new worker
worker = new Worker(`
import { parentPort, workerData } from 'node:worker_threads';
// Add TensorFlow.js platform patch for Node.js
if (typeof global !== 'undefined') {
try {
// Define a custom PlatformNode class
class PlatformNode {
constructor() {
// Create a util object with necessary methods
this.util = {
// Add isFloat32Array and isTypedArray directly to util
isFloat32Array: (arr) => {
return !!(
arr instanceof Float32Array ||
(arr &&
Object.prototype.toString.call(arr) === '[object Float32Array]')
);
},
isTypedArray: (arr) => {
return !!(ArrayBuffer.isView(arr) && !(arr instanceof DataView));
},
// Use native TextEncoder and TextDecoder
TextEncoder: TextEncoder,
TextDecoder: TextDecoder
};
// Initialize encoders using native constructors
this.textEncoder = new TextEncoder();
this.textDecoder = new TextDecoder();
}
// Define isFloat32Array directly on the instance
isFloat32Array(arr) {
return !!(
arr instanceof Float32Array ||
(arr && Object.prototype.toString.call(arr) === '[object Float32Array]')
);
}
// Define isTypedArray directly on the instance
isTypedArray(arr) {
return !!(ArrayBuffer.isView(arr) && !(arr instanceof DataView));
}
}
// Assign the PlatformNode class to the global object
global.PlatformNode = PlatformNode;
// Also create an instance and assign it to global.platformNode
global.platformNode = new PlatformNode();
// Ensure global.util exists and has the necessary methods
if (!global.util) {
global.util = {};
}
// Add isFloat32Array method if it doesn't exist
if (!global.util.isFloat32Array) {
global.util.isFloat32Array = (arr) => {
return !!(
arr instanceof Float32Array ||
(arr && Object.prototype.toString.call(arr) === '[object Float32Array]')
);
};
}
// Add isTypedArray method if it doesn't exist
if (!global.util.isTypedArray) {
global.util.isTypedArray = (arr) => {
return !!(ArrayBuffer.isView(arr) && !(arr instanceof DataView));
};
}
} catch (error) {
console.warn('Failed to apply TensorFlow.js platform patch:', error);
}
}
const fn = new Function('return ' + workerData.fnString)();
const result = fn(workerData.args);
parentPort.postMessage({ result });
`, {
eval: true,
workerData: { fnString, args }
});
workerPool.set(workerId, worker);
}
else {
// Reuse an existing worker
const poolKeys = Array.from(workerPool.keys());
const randomKey = poolKeys[Math.floor(Math.random() * poolKeys.length)];
worker = workerPool.get(randomKey);
// Terminate and recreate if the worker is busy
if (worker._busy) {
worker.terminate();
worker = new Worker(`
import { parentPort, workerData } from 'node:worker_threads';
// Add TensorFlow.js platform patch for Node.js
if (typeof global !== 'undefined') {
try {
// Define a custom PlatformNode class
class PlatformNode {
constructor() {
// Create a util object with necessary methods
this.util = {
// Use native TextEncoder and TextDecoder
TextEncoder: TextEncoder,
TextDecoder: TextDecoder
};
// Initialize encoders using native constructors
this.textEncoder = new TextEncoder();
this.textDecoder = new TextDecoder();
}
// Define isFloat32Array directly on the instance
isFloat32Array(arr) {
return !!(
arr instanceof Float32Array ||
(arr && Object.prototype.toString.call(arr) === '[object Float32Array]')
);
}
// Define isTypedArray directly on the instance
isTypedArray(arr) {
return !!(ArrayBuffer.isView(arr) && !(arr instanceof DataView));
}
}
// Assign the PlatformNode class to the global object
global.PlatformNode = PlatformNode;
// Also create an instance and assign it to global.platformNode
global.platformNode = new PlatformNode();
// Ensure global.util exists and has the necessary methods
if (!global.util) {
global.util = {};
}
// Add isFloat32Array method if it doesn't exist
if (!global.util.isFloat32Array) {
global.util.isFloat32Array = (arr) => {
return !!(
arr instanceof Float32Array ||
(arr && Object.prototype.toString.call(arr) === '[object Float32Array]')
);
};
}
// Add isTypedArray method if it doesn't exist
if (!global.util.isTypedArray) {
global.util.isTypedArray = (arr) => {
return !!(ArrayBuffer.isView(arr) && !(arr instanceof DataView));
};
}
} catch (error) {
console.warn('Failed to apply TensorFlow.js platform patch:', error);
}
}
const fn = new Function('return ' + workerData.fnString)();
const result = fn(workerData.args);
parentPort.postMessage({ result });
`, {
eval: true,
workerData: { fnString, args }
});
workerPool.set(randomKey, worker);
}
worker._busy = true;
}
worker.on('message', (message) => {
worker._busy = false;
resolve(message.result);
});
worker.on('error', (err) => {
worker._busy = false;
reject(err);
});
worker.on('exit', (code) => {
if (code !== 0) {
worker._busy = false;
reject(new Error(`Worker stopped with exit code ${code}`));
}
});
})
.catch(reject);
}
catch (error) {
reject(error);
}
});
}
/**
* Execute a function in a Web Worker (Browser environment)
*/
function executeInWebWorker(fnString, args) {
return new Promise((resolve, reject) => {
try {
// Use the dedicated worker.js file instead of creating a blob
// Try different approaches to locate the worker.js file
let workerPath = './worker.js';
try {
// First try to use the import.meta.url if available (modern browsers)
if (typeof import.meta !== 'undefined' && import.meta.url) {
const baseUrl = import.meta.url.substring(0, import.meta.url.lastIndexOf('/') + 1);
workerPath = `${baseUrl}worker.js`;
}
// Fallback to a relative path based on the unified.js location
else if (typeof document !== 'undefined') {
// Find the script tag that loaded unified.js
const scripts = document.getElementsByTagName('script');
for (let i = 0; i < scripts.length; i++) {
const src = scripts[i].src;
if (src && src.includes('unified.js')) {
// Get the directory path
workerPath =
src.substring(0, src.lastIndexOf('/') + 1) + 'worker.js';
break;
}
}
}
}
catch (e) {
console.warn('Could not determine worker path from import.meta.url, using relative path', e);
}
// If we couldn't determine the path, try some common locations
if (workerPath === './worker.js' && typeof window !== 'undefined') {
// Try to find the worker.js in the same directory as the current page
const pageUrl = window.location.href;
const pageDir = pageUrl.substring(0, pageUrl.lastIndexOf('/') + 1);
workerPath = `${pageDir}worker.js`;
// Also check for dist/worker.js
if (typeof document !== 'undefined') {
const distWorkerPath = `${pageDir}dist/worker.js`;
// Create a test request to see if the file exists
const xhr = new XMLHttpRequest();
xhr.open('HEAD', distWorkerPath, false);
try {
xhr.send();
if (xhr.status >= 200 && xhr.status < 300) {
workerPath = distWorkerPath;
}
}
catch (e) {
// Ignore errors, we'll use the default path
}
}
}
console.log('Using worker path:', workerPath);
// Try to create a worker, but fall back to inline worker or main thread execution if it fails
let worker;
try {
worker = new Worker(workerPath);
}
catch (error) {
console.warn('Failed to create Web Worker from file, trying inline worker:', error);
try {
// Create an inline worker using a Blob
const workerCode = `
// Brainy Inline Worker Script
console.log('Brainy Inline Worker: Started');
self.onmessage = function (e) {
try {
console.log('Brainy Inline Worker: Received message', e.data ? 'with data' : 'without data');
if (!e.data || !e.data.fnString) {
throw new Error('Invalid message: missing function string');
}
console.log('Brainy Inline Worker: Creating function from string');
const fn = new Function('return ' + e.data.fnString)();
console.log('Brainy Inline Worker: Executing function with args');
const result = fn(e.data.args);
console.log('Brainy Inline Worker: Function executed successfully, posting result');
self.postMessage({ result: result });
} catch (error) {
console.error('Brainy Inline Worker: Error executing function', error);
self.postMessage({
error: error.message,
stack: error.stack
});
}
};
`;
const blob = new Blob([workerCode], {
type: 'application/javascript'
});
const blobUrl = URL.createObjectURL(blob);
worker = new Worker(blobUrl);
console.log('Created inline worker using Blob URL');
}
catch (inlineWorkerError) {
console.warn('Failed to create inline Web Worker, falling back to main thread execution:', inlineWorkerError);
// Execute in main thread as fallback
try {
const fn = new Function('return ' + fnString)();
resolve(fn(args));
return;
}
catch (mainThreadError) {
reject(mainThreadError);
return;
}
}
}
// Set a timeout to prevent hanging
const timeoutId = setTimeout(() => {
console.warn('Web Worker execution timed out, falling back to main thread');
worker.terminate();
// Execute in main thread as fallback
try {
const fn = new Function('return ' + fnString)();
resolve(fn(args));
}
catch (mainThreadError) {
reject(mainThreadError);
}
}, 25000); // 25 second timeout (less than the 30 second test timeout)
worker.onmessage = function (e) {
clearTimeout(timeoutId);
if (e.data.error) {
reject(new Error(e.data.error));
}
else {
resolve(e.data.result);
}
worker.terminate();
};
worker.onerror = function (e) {
clearTimeout(timeoutId);
console.warn('Web Worker error, falling back to main thread execution:', e.message);
worker.terminate();
// Execute in main thread as fallback
try {
const fn = new Function('return ' + fnString)();
resolve(fn(args));
}
catch (mainThreadError) {
reject(mainThreadError);
}
};
worker.postMessage({ fnString, args });
}
catch (error) {
reject(error);
}
});
}
/**
* Clean up all worker pools
* This should be called when the application is shutting down
*/
export function cleanupWorkerPools() {
if (isNode()) {
import('node:worker_threads')
.then(({ Worker }) => {
for (const worker of workerPool.values()) {
worker.terminate();
}
workerPool.clear();
console.log('Worker pools cleaned up');
})
.catch(console.error);
}
}
//# sourceMappingURL=workerUtils.js.map

1
dist/utils/workerUtils.js.map vendored Normal file
View file

File diff suppressed because one or more lines are too long

93
dist/utils/writeBuffer.d.ts vendored Normal file
View file

@ -0,0 +1,93 @@
/**
* Write Buffer
* Accumulates writes and flushes them in bulk to reduce S3 operations
* Implements intelligent deduplication and compression
*/
interface FlushResult {
successful: number;
failed: number;
duration: number;
}
/**
* High-performance write buffer for bulk operations
*/
export declare class WriteBuffer<T> {
private logger;
private buffer;
private maxBufferSize;
private flushInterval;
private minFlushSize;
private maxRetries;
private flushTimer;
private isFlushing;
private lastFlush;
private pendingFlush;
private totalWrites;
private totalFlushes;
private failedWrites;
private duplicatesRemoved;
private writeFunction;
private type;
private backpressure;
constructor(type: 'noun' | 'verb' | 'metadata', writeFunction: (items: Map<string, T>) => Promise<void>, options?: {
maxBufferSize?: number;
flushInterval?: number;
minFlushSize?: number;
});
/**
* Add item to buffer
*/
add(id: string, data: T): Promise<void>;
/**
* Check if we should flush
*/
private checkFlush;
/**
* Flush buffer to storage
*/
flush(reason?: string): Promise<FlushResult>;
/**
* Perform the actual flush
*/
private doFlush;
/**
* Start periodic flush timer
*/
private startPeriodicFlush;
/**
* Stop periodic flush timer
*/
stop(): void;
/**
* Force flush all pending writes
*/
forceFlush(): Promise<FlushResult>;
/**
* Get buffer statistics
*/
getStats(): {
bufferSize: number;
totalWrites: number;
totalFlushes: number;
failedWrites: number;
duplicatesRemoved: number;
avgFlushSize: number;
};
/**
* Adjust parameters based on load
*/
adjustForLoad(pendingRequests: number): void;
}
/**
* Get or create a write buffer
*/
export declare function getWriteBuffer<T>(id: string, type: 'noun' | 'verb' | 'metadata', writeFunction: (items: Map<string, T>) => Promise<void>): WriteBuffer<T>;
/**
* Flush all write buffers
*/
export declare function flushAllBuffers(): Promise<void>;
/**
* Clear all write buffers
*/
export declare function clearWriteBuffers(): void;
export {};

328
dist/utils/writeBuffer.js vendored Normal file
View file

@ -0,0 +1,328 @@
/**
* Write Buffer
* Accumulates writes and flushes them in bulk to reduce S3 operations
* Implements intelligent deduplication and compression
*/
import { createModuleLogger } from './logger.js';
import { getGlobalBackpressure } from './adaptiveBackpressure.js';
/**
* High-performance write buffer for bulk operations
*/
export class WriteBuffer {
constructor(type, writeFunction, options) {
this.logger = createModuleLogger('WriteBuffer');
// Buffer storage
this.buffer = new Map();
// Configuration - More aggressive for high volume
this.maxBufferSize = 2000; // Allow larger buffers
this.flushInterval = 500; // Flush more frequently (0.5 seconds)
this.minFlushSize = 50; // Lower minimum to flush sooner
this.maxRetries = 3; // Maximum retry attempts
// State
this.flushTimer = null;
this.isFlushing = false;
this.lastFlush = Date.now();
this.pendingFlush = null;
// Statistics
this.totalWrites = 0;
this.totalFlushes = 0;
this.failedWrites = 0;
this.duplicatesRemoved = 0;
// Backpressure integration
this.backpressure = getGlobalBackpressure();
this.type = type;
this.writeFunction = writeFunction;
if (options) {
this.maxBufferSize = options.maxBufferSize || this.maxBufferSize;
this.flushInterval = options.flushInterval || this.flushInterval;
this.minFlushSize = options.minFlushSize || this.minFlushSize;
}
// Start periodic flush
this.startPeriodicFlush();
}
/**
* Add item to buffer
*/
async add(id, data) {
// Check if we're already at capacity
if (this.buffer.size >= this.maxBufferSize) {
// Wait for current flush to complete
if (this.pendingFlush) {
await this.pendingFlush;
}
// Force flush if still at capacity
if (this.buffer.size >= this.maxBufferSize) {
await this.flush('capacity');
}
}
// Check for duplicate and update if newer
const existing = this.buffer.get(id);
if (existing) {
// Update with newer data
existing.data = data;
existing.timestamp = Date.now();
this.duplicatesRemoved++;
}
else {
// Add new item
this.buffer.set(id, {
id,
data,
timestamp: Date.now(),
type: this.type,
retryCount: 0
});
}
this.totalWrites++;
// Log buffer growth periodically
if (this.totalWrites % 100 === 0) {
this.logger.info(`📈 BUFFER GROWTH: ${this.buffer.size} ${this.type} items buffered (${this.totalWrites} total writes, ${this.duplicatesRemoved} deduplicated)`);
}
// Check if we should flush
this.checkFlush();
}
/**
* Check if we should flush
*/
checkFlush() {
const bufferSize = this.buffer.size;
const timeSinceFlush = Date.now() - this.lastFlush;
// Immediate flush conditions
if (bufferSize >= this.maxBufferSize) {
this.flush('size');
return;
}
// Time-based flush with minimum size
if (timeSinceFlush >= this.flushInterval && bufferSize >= this.minFlushSize) {
this.flush('time');
return;
}
// Adaptive flush based on system load
const backpressureStatus = this.backpressure.getStatus();
if (backpressureStatus.queueLength > 1000 && bufferSize > 10) {
// System under pressure - flush smaller batches more frequently
this.flush('pressure');
}
}
/**
* Flush buffer to storage
*/
async flush(reason = 'manual') {
// Prevent concurrent flushes
if (this.isFlushing) {
if (this.pendingFlush) {
return this.pendingFlush;
}
return { successful: 0, failed: 0, duration: 0 };
}
// Nothing to flush
if (this.buffer.size === 0) {
return { successful: 0, failed: 0, duration: 0 };
}
this.isFlushing = true;
const startTime = Date.now();
// Create flush promise
this.pendingFlush = this.doFlush(reason, startTime);
try {
const result = await this.pendingFlush;
return result;
}
finally {
this.isFlushing = false;
this.pendingFlush = null;
}
}
/**
* Perform the actual flush
*/
async doFlush(reason, startTime) {
const itemsToFlush = new Map();
const flushingItems = new Map();
// Take items from buffer
let count = 0;
for (const [id, item] of this.buffer.entries()) {
itemsToFlush.set(id, item.data);
flushingItems.set(id, item);
count++;
// Limit batch size for better performance
if (count >= 500) {
break;
}
}
// Remove from buffer
for (const id of itemsToFlush.keys()) {
this.buffer.delete(id);
}
this.logger.warn(`🔄 BUFFERING: Flushing ${itemsToFlush.size} ${this.type} items (buffer had ${this.buffer.size + itemsToFlush.size}) - reason: ${reason}`);
try {
// Request permission from backpressure system
const opId = `flush-${Date.now()}`;
await this.backpressure.requestPermission(opId, 2); // Higher priority
try {
// Perform bulk write
await this.writeFunction(itemsToFlush);
// Success
this.backpressure.releasePermission(opId, true);
this.totalFlushes++;
this.lastFlush = Date.now();
const duration = Date.now() - startTime;
this.logger.warn(`🚀 BATCH FLUSH: ${itemsToFlush.size} ${this.type} items → 1 bulk S3 operation (${duration}ms, reason: ${reason})`);
return {
successful: itemsToFlush.size,
failed: 0,
duration
};
}
catch (error) {
// Release with error
this.backpressure.releasePermission(opId, false);
throw error;
}
}
catch (error) {
this.logger.error(`Flush failed: ${error}`);
// Put items back with retry count
for (const [id, item] of flushingItems.entries()) {
item.retryCount++;
if (item.retryCount < this.maxRetries) {
// Put back for retry
this.buffer.set(id, item);
}
else {
// Max retries exceeded
this.failedWrites++;
this.logger.error(`Max retries exceeded for ${this.type} ${id}`);
}
}
const duration = Date.now() - startTime;
return {
successful: 0,
failed: itemsToFlush.size,
duration
};
}
}
/**
* Start periodic flush timer
*/
startPeriodicFlush() {
if (this.flushTimer) {
return;
}
this.flushTimer = setInterval(() => {
if (this.buffer.size > 0) {
const timeSinceFlush = Date.now() - this.lastFlush;
// Flush if we have items and enough time has passed
if (timeSinceFlush >= this.flushInterval) {
this.flush('periodic').catch(error => {
this.logger.error('Periodic flush failed:', error);
});
}
}
}, Math.min(100, this.flushInterval / 2));
}
/**
* Stop periodic flush timer
*/
stop() {
if (this.flushTimer) {
clearInterval(this.flushTimer);
this.flushTimer = null;
}
}
/**
* Force flush all pending writes
*/
async forceFlush() {
// Flush everything regardless of size
const oldMinSize = this.minFlushSize;
this.minFlushSize = 0;
try {
const result = await this.flush('force');
// Flush any remaining items
while (this.buffer.size > 0) {
const additionalResult = await this.flush('force-remaining');
result.successful += additionalResult.successful;
result.failed += additionalResult.failed;
result.duration += additionalResult.duration;
}
return result;
}
finally {
this.minFlushSize = oldMinSize;
}
}
/**
* Get buffer statistics
*/
getStats() {
return {
bufferSize: this.buffer.size,
totalWrites: this.totalWrites,
totalFlushes: this.totalFlushes,
failedWrites: this.failedWrites,
duplicatesRemoved: this.duplicatesRemoved,
avgFlushSize: this.totalFlushes > 0 ? this.totalWrites / this.totalFlushes : 0
};
}
/**
* Adjust parameters based on load
*/
adjustForLoad(pendingRequests) {
if (pendingRequests > 10000) {
// Extreme load - buffer more aggressively
this.maxBufferSize = 5000;
this.flushInterval = 500;
this.minFlushSize = 500;
}
else if (pendingRequests > 1000) {
// High load
this.maxBufferSize = 2000;
this.flushInterval = 1000;
this.minFlushSize = 200;
}
else if (pendingRequests > 100) {
// Moderate load
this.maxBufferSize = 1000;
this.flushInterval = 2000;
this.minFlushSize = 100;
}
else {
// Low load - optimize for latency
this.maxBufferSize = 500;
this.flushInterval = 5000;
this.minFlushSize = 50;
}
}
}
// Global write buffers
const writeBuffers = new Map();
/**
* Get or create a write buffer
*/
export function getWriteBuffer(id, type, writeFunction) {
if (!writeBuffers.has(id)) {
writeBuffers.set(id, new WriteBuffer(type, writeFunction));
}
return writeBuffers.get(id);
}
/**
* Flush all write buffers
*/
export async function flushAllBuffers() {
const promises = [];
for (const buffer of writeBuffers.values()) {
promises.push(buffer.forceFlush());
}
await Promise.all(promises);
}
/**
* Clear all write buffers
*/
export function clearWriteBuffers() {
for (const buffer of writeBuffers.values()) {
buffer.stop();
}
writeBuffers.clear();
}
//# sourceMappingURL=writeBuffer.js.map

1
dist/utils/writeBuffer.js.map vendored Normal file
View file

File diff suppressed because one or more lines are too long