David Snelling
f8c45f2d8d
Open source vector database with HNSW indexing, graph relationships, and metadata facets. Features CLI with professional augmentation registry integration for discovering extensions and capabilities.
384 lines
No EOL
14 KiB
JavaScript
384 lines
No EOL
14 KiB
JavaScript
/**
|
|
* Performance Monitor
|
|
* Automatically tracks and optimizes system performance
|
|
* Provides real-time insights and auto-tuning recommendations
|
|
*/
|
|
import { createModuleLogger } from './logger.js';
|
|
import { getGlobalSocketManager } from './adaptiveSocketManager.js';
|
|
import { getGlobalBackpressure } from './adaptiveBackpressure.js';
|
|
/**
|
|
* Comprehensive performance monitoring and optimization
|
|
*/
|
|
export class PerformanceMonitor {
|
|
constructor() {
|
|
this.logger = createModuleLogger('PerformanceMonitor');
|
|
// Current metrics
|
|
this.metrics = {
|
|
totalOperations: 0,
|
|
successfulOperations: 0,
|
|
failedOperations: 0,
|
|
averageLatency: 0,
|
|
p95Latency: 0,
|
|
p99Latency: 0,
|
|
operationsPerSecond: 0,
|
|
bytesPerSecond: 0,
|
|
memoryUsage: 0,
|
|
cpuUsage: 0,
|
|
socketUtilization: 0,
|
|
queueDepth: 0,
|
|
errorRate: 0,
|
|
healthScore: 100
|
|
};
|
|
// Historical data for trend analysis
|
|
this.history = [];
|
|
this.maxHistorySize = 1000;
|
|
// Operation tracking
|
|
this.operationLatencies = [];
|
|
this.operationSizes = [];
|
|
this.lastReset = Date.now();
|
|
this.resetInterval = 60000; // Reset counters every minute
|
|
// CPU tracking
|
|
this.lastCpuUsage = process.cpuUsage ? process.cpuUsage() : null;
|
|
this.lastCpuCheck = Date.now();
|
|
// Alert thresholds
|
|
this.thresholds = {
|
|
errorRate: 0.05, // 5% error rate
|
|
latencyP95: 5000, // 5 second P95
|
|
memoryUsage: 0.8, // 80% memory
|
|
cpuUsage: 0.9, // 90% CPU
|
|
healthScore: 70 // Health score below 70
|
|
};
|
|
// Optimization recommendations
|
|
this.recommendations = [];
|
|
// Auto-optimization state
|
|
this.autoOptimizeEnabled = true;
|
|
this.lastOptimization = Date.now();
|
|
this.optimizationInterval = 30000; // Optimize every 30 seconds
|
|
}
|
|
/**
|
|
* Track an operation completion
|
|
*/
|
|
trackOperation(success, latency, bytes = 0) {
|
|
// Update counters
|
|
this.metrics.totalOperations++;
|
|
if (success) {
|
|
this.metrics.successfulOperations++;
|
|
}
|
|
else {
|
|
this.metrics.failedOperations++;
|
|
}
|
|
// Track latency
|
|
this.operationLatencies.push(latency);
|
|
if (this.operationLatencies.length > 10000) {
|
|
this.operationLatencies = this.operationLatencies.slice(-5000);
|
|
}
|
|
// Track size
|
|
if (bytes > 0) {
|
|
this.operationSizes.push(bytes);
|
|
if (this.operationSizes.length > 10000) {
|
|
this.operationSizes = this.operationSizes.slice(-5000);
|
|
}
|
|
}
|
|
// Update metrics periodically
|
|
this.updateMetrics();
|
|
}
|
|
/**
|
|
* Update all metrics
|
|
*/
|
|
updateMetrics() {
|
|
const now = Date.now();
|
|
const timeSinceReset = (now - this.lastReset) / 1000;
|
|
// Calculate latency percentiles
|
|
if (this.operationLatencies.length > 0) {
|
|
const sorted = [...this.operationLatencies].sort((a, b) => a - b);
|
|
const p95Index = Math.floor(sorted.length * 0.95);
|
|
const p99Index = Math.floor(sorted.length * 0.99);
|
|
this.metrics.averageLatency = sorted.reduce((a, b) => a + b, 0) / sorted.length;
|
|
this.metrics.p95Latency = sorted[p95Index] || 0;
|
|
this.metrics.p99Latency = sorted[p99Index] || 0;
|
|
}
|
|
// Calculate throughput
|
|
if (timeSinceReset > 0) {
|
|
this.metrics.operationsPerSecond = this.metrics.totalOperations / timeSinceReset;
|
|
const totalBytes = this.operationSizes.reduce((a, b) => a + b, 0);
|
|
this.metrics.bytesPerSecond = totalBytes / timeSinceReset;
|
|
}
|
|
// Calculate error rate
|
|
this.metrics.errorRate = this.metrics.totalOperations > 0
|
|
? this.metrics.failedOperations / this.metrics.totalOperations
|
|
: 0;
|
|
// Update resource metrics
|
|
this.updateResourceMetrics();
|
|
// Calculate health score
|
|
this.calculateHealthScore();
|
|
// Store in history
|
|
this.history.push({ ...this.metrics });
|
|
if (this.history.length > this.maxHistorySize) {
|
|
this.history.shift();
|
|
}
|
|
// Check for alerts
|
|
this.checkAlerts();
|
|
// Auto-optimize if enabled
|
|
if (this.autoOptimizeEnabled && now - this.lastOptimization > this.optimizationInterval) {
|
|
this.autoOptimize();
|
|
this.lastOptimization = now;
|
|
}
|
|
// Reset counters periodically
|
|
if (now - this.lastReset > this.resetInterval) {
|
|
this.resetCounters();
|
|
}
|
|
}
|
|
/**
|
|
* Update resource metrics
|
|
*/
|
|
updateResourceMetrics() {
|
|
// Memory usage
|
|
if (typeof process !== 'undefined' && process.memoryUsage) {
|
|
const memUsage = process.memoryUsage();
|
|
this.metrics.memoryUsage = memUsage.heapUsed / memUsage.heapTotal;
|
|
}
|
|
// CPU usage (Node.js only)
|
|
if (this.lastCpuUsage && process.cpuUsage) {
|
|
const currentCpuUsage = process.cpuUsage();
|
|
const now = Date.now();
|
|
const timeDiff = now - this.lastCpuCheck;
|
|
if (timeDiff > 1000) { // Update CPU every second
|
|
const userDiff = currentCpuUsage.user - this.lastCpuUsage.user;
|
|
const systemDiff = currentCpuUsage.system - this.lastCpuUsage.system;
|
|
const totalDiff = userDiff + systemDiff;
|
|
// CPU percentage (approximate)
|
|
this.metrics.cpuUsage = totalDiff / (timeDiff * 1000);
|
|
this.lastCpuUsage = currentCpuUsage;
|
|
this.lastCpuCheck = now;
|
|
}
|
|
}
|
|
// Get metrics from socket manager
|
|
const socketMetrics = getGlobalSocketManager().getMetrics();
|
|
this.metrics.socketUtilization = socketMetrics.socketUtilization;
|
|
// Get metrics from backpressure system
|
|
const backpressureStatus = getGlobalBackpressure().getStatus();
|
|
this.metrics.queueDepth = backpressureStatus.queueLength;
|
|
}
|
|
/**
|
|
* Calculate overall health score
|
|
*/
|
|
calculateHealthScore() {
|
|
let score = 100;
|
|
// Deduct points for high error rate
|
|
if (this.metrics.errorRate > 0.01) {
|
|
score -= Math.min(30, this.metrics.errorRate * 300);
|
|
}
|
|
// Deduct points for high latency
|
|
if (this.metrics.p95Latency > 3000) {
|
|
score -= Math.min(20, (this.metrics.p95Latency - 3000) / 100);
|
|
}
|
|
// Deduct points for high memory usage
|
|
if (this.metrics.memoryUsage > 0.7) {
|
|
score -= Math.min(20, (this.metrics.memoryUsage - 0.7) * 66);
|
|
}
|
|
// Deduct points for high CPU usage
|
|
if (this.metrics.cpuUsage > 0.8) {
|
|
score -= Math.min(15, (this.metrics.cpuUsage - 0.8) * 75);
|
|
}
|
|
// Deduct points for low throughput
|
|
if (this.metrics.operationsPerSecond < 1 && this.metrics.totalOperations > 10) {
|
|
score -= 10;
|
|
}
|
|
// Deduct points for queue depth
|
|
if (this.metrics.queueDepth > 100) {
|
|
score -= Math.min(15, this.metrics.queueDepth / 20);
|
|
}
|
|
this.metrics.healthScore = Math.max(0, Math.min(100, score));
|
|
}
|
|
/**
|
|
* Check for alert conditions
|
|
*/
|
|
checkAlerts() {
|
|
const alerts = [];
|
|
if (this.metrics.errorRate > this.thresholds.errorRate) {
|
|
alerts.push(`High error rate: ${(this.metrics.errorRate * 100).toFixed(1)}%`);
|
|
}
|
|
if (this.metrics.p95Latency > this.thresholds.latencyP95) {
|
|
alerts.push(`High P95 latency: ${this.metrics.p95Latency}ms`);
|
|
}
|
|
if (this.metrics.memoryUsage > this.thresholds.memoryUsage) {
|
|
alerts.push(`High memory usage: ${(this.metrics.memoryUsage * 100).toFixed(1)}%`);
|
|
}
|
|
if (this.metrics.cpuUsage > this.thresholds.cpuUsage) {
|
|
alerts.push(`High CPU usage: ${(this.metrics.cpuUsage * 100).toFixed(1)}%`);
|
|
}
|
|
if (this.metrics.healthScore < this.thresholds.healthScore) {
|
|
alerts.push(`Low health score: ${this.metrics.healthScore.toFixed(0)}`);
|
|
}
|
|
if (alerts.length > 0) {
|
|
this.logger.warn('Performance alerts', { alerts, metrics: this.metrics });
|
|
}
|
|
}
|
|
/**
|
|
* Auto-optimize system based on metrics
|
|
*/
|
|
autoOptimize() {
|
|
this.recommendations = [];
|
|
// Analyze trends
|
|
const trends = this.analyzeTrends();
|
|
// Generate recommendations based on metrics and trends
|
|
if (this.metrics.errorRate > 0.02) {
|
|
this.recommendations.push('Reduce load or increase timeouts due to high error rate');
|
|
}
|
|
if (this.metrics.p95Latency > 3000) {
|
|
this.recommendations.push('Increase batch size or socket limits to improve latency');
|
|
}
|
|
if (this.metrics.memoryUsage > 0.7) {
|
|
this.recommendations.push('Reduce cache sizes or batch sizes to free memory');
|
|
}
|
|
if (this.metrics.queueDepth > 50) {
|
|
this.recommendations.push('Increase concurrency limits to reduce queue depth');
|
|
}
|
|
// Check for degrading trends
|
|
trends.forEach(trend => {
|
|
if (trend.direction === 'degrading' && Math.abs(trend.changeRate) > 0.1) {
|
|
this.recommendations.push(`${trend.metric} is degrading at ${(trend.changeRate * 100).toFixed(1)}% per minute`);
|
|
}
|
|
});
|
|
// Log recommendations if any
|
|
if (this.recommendations.length > 0) {
|
|
this.logger.info('Performance optimization recommendations', {
|
|
recommendations: this.recommendations,
|
|
metrics: this.metrics
|
|
});
|
|
}
|
|
}
|
|
/**
|
|
* Analyze performance trends
|
|
*/
|
|
analyzeTrends() {
|
|
const trends = [];
|
|
if (this.history.length < 10) {
|
|
return trends; // Not enough data
|
|
}
|
|
// Get recent history
|
|
const recent = this.history.slice(-20);
|
|
const older = this.history.slice(-40, -20);
|
|
// Compare key metrics
|
|
const metricsToAnalyze = [
|
|
'errorRate',
|
|
'averageLatency',
|
|
'operationsPerSecond',
|
|
'memoryUsage',
|
|
'healthScore'
|
|
];
|
|
metricsToAnalyze.forEach(metric => {
|
|
const recentAvg = recent.reduce((sum, m) => sum + m[metric], 0) / recent.length;
|
|
const olderAvg = older.length > 0
|
|
? older.reduce((sum, m) => sum + m[metric], 0) / older.length
|
|
: recentAvg;
|
|
const changeRate = olderAvg !== 0 ? (recentAvg - olderAvg) / olderAvg : 0;
|
|
let direction = 'stable';
|
|
if (Math.abs(changeRate) > 0.05) { // 5% threshold
|
|
// For error rate and latency, increase is bad
|
|
if (metric === 'errorRate' || metric === 'averageLatency' || metric === 'memoryUsage') {
|
|
direction = changeRate > 0 ? 'degrading' : 'improving';
|
|
}
|
|
else {
|
|
// For throughput and health score, increase is good
|
|
direction = changeRate > 0 ? 'improving' : 'degrading';
|
|
}
|
|
}
|
|
// Simple linear prediction
|
|
const prediction = recentAvg + (recentAvg * changeRate);
|
|
trends.push({
|
|
metric,
|
|
direction,
|
|
changeRate,
|
|
prediction
|
|
});
|
|
});
|
|
return trends;
|
|
}
|
|
/**
|
|
* Reset counters
|
|
*/
|
|
resetCounters() {
|
|
this.metrics.totalOperations = 0;
|
|
this.metrics.successfulOperations = 0;
|
|
this.metrics.failedOperations = 0;
|
|
this.operationSizes = [];
|
|
this.lastReset = Date.now();
|
|
}
|
|
/**
|
|
* Get current metrics
|
|
*/
|
|
getMetrics() {
|
|
return { ...this.metrics };
|
|
}
|
|
/**
|
|
* Get performance trends
|
|
*/
|
|
getTrends() {
|
|
return this.analyzeTrends();
|
|
}
|
|
/**
|
|
* Get recommendations
|
|
*/
|
|
getRecommendations() {
|
|
return [...this.recommendations];
|
|
}
|
|
/**
|
|
* Get performance report
|
|
*/
|
|
getReport() {
|
|
return {
|
|
metrics: this.getMetrics(),
|
|
trends: this.getTrends(),
|
|
recommendations: this.getRecommendations(),
|
|
socketConfig: getGlobalSocketManager().getConfig(),
|
|
backpressureStatus: getGlobalBackpressure().getStatus()
|
|
};
|
|
}
|
|
/**
|
|
* Enable/disable auto-optimization
|
|
*/
|
|
setAutoOptimize(enabled) {
|
|
this.autoOptimizeEnabled = enabled;
|
|
this.logger.info(`Auto-optimization ${enabled ? 'enabled' : 'disabled'}`);
|
|
}
|
|
/**
|
|
* Reset all metrics and history
|
|
*/
|
|
reset() {
|
|
this.metrics = {
|
|
totalOperations: 0,
|
|
successfulOperations: 0,
|
|
failedOperations: 0,
|
|
averageLatency: 0,
|
|
p95Latency: 0,
|
|
p99Latency: 0,
|
|
operationsPerSecond: 0,
|
|
bytesPerSecond: 0,
|
|
memoryUsage: 0,
|
|
cpuUsage: 0,
|
|
socketUtilization: 0,
|
|
queueDepth: 0,
|
|
errorRate: 0,
|
|
healthScore: 100
|
|
};
|
|
this.history = [];
|
|
this.operationLatencies = [];
|
|
this.operationSizes = [];
|
|
this.recommendations = [];
|
|
this.lastReset = Date.now();
|
|
this.logger.info('Performance monitor reset');
|
|
}
|
|
}
|
|
// Global singleton instance
|
|
let globalMonitor = null;
|
|
/**
|
|
* Get the global performance monitor instance
|
|
*/
|
|
export function getGlobalPerformanceMonitor() {
|
|
if (!globalMonitor) {
|
|
globalMonitor = new PerformanceMonitor();
|
|
}
|
|
return globalMonitor;
|
|
}
|
|
//# sourceMappingURL=performanceMonitor.js.map
|