chore: recovery checkpoint - v3.0 API successfully recovered
CRITICAL CHECKPOINT - DO NOT PUSH TO GITHUB Recovery Status: - Successfully recovered brainy.ts from compiled JavaScript - All core v3.0 API methods functional (add, get, update, delete, relate, find, etc.) - Neural subsystem intact (562KB embedded patterns, NLP working) - Augmentation pipeline operational (20+ augmentations) - HNSW clustering system complete - Triple Intelligence compiled (needs constructor fix) - Test suite validates functionality Changes preserved: - 898 files with changes from last 3 days - 144,475 insertions - All augmentation improvements - All test coverage enhancements - Complete v3.0 feature set This is a LOCAL checkpoint only - contains recovered work after corruption incident. Created backup in .backups/brainy-full-20250910-151314.tar.gz Branch: recovery-checkpoint-20250910-151433 Date: Wed Sep 10 03:18:04 PM PDT 2025
This commit is contained in:
parent
f65455fb22
commit
8ff382ca3b
895 changed files with 143654 additions and 28268 deletions
|
|
@ -0,0 +1,384 @@
|
|||
/**
|
||||
* Performance Monitor
|
||||
* Automatically tracks and optimizes system performance
|
||||
* Provides real-time insights and auto-tuning recommendations
|
||||
*/
|
||||
import { createModuleLogger } from './logger.js';
|
||||
import { getGlobalSocketManager } from './adaptiveSocketManager.js';
|
||||
import { getGlobalBackpressure } from './adaptiveBackpressure.js';
|
||||
/**
|
||||
* Comprehensive performance monitoring and optimization
|
||||
*/
|
||||
export class PerformanceMonitor {
|
||||
constructor() {
|
||||
this.logger = createModuleLogger('PerformanceMonitor');
|
||||
// Current metrics
|
||||
this.metrics = {
|
||||
totalOperations: 0,
|
||||
successfulOperations: 0,
|
||||
failedOperations: 0,
|
||||
averageLatency: 0,
|
||||
p95Latency: 0,
|
||||
p99Latency: 0,
|
||||
operationsPerSecond: 0,
|
||||
bytesPerSecond: 0,
|
||||
memoryUsage: 0,
|
||||
cpuUsage: 0,
|
||||
socketUtilization: 0,
|
||||
queueDepth: 0,
|
||||
errorRate: 0,
|
||||
healthScore: 100
|
||||
};
|
||||
// Historical data for trend analysis
|
||||
this.history = [];
|
||||
this.maxHistorySize = 1000;
|
||||
// Operation tracking
|
||||
this.operationLatencies = [];
|
||||
this.operationSizes = [];
|
||||
this.lastReset = Date.now();
|
||||
this.resetInterval = 60000; // Reset counters every minute
|
||||
// CPU tracking
|
||||
this.lastCpuUsage = process.cpuUsage ? process.cpuUsage() : null;
|
||||
this.lastCpuCheck = Date.now();
|
||||
// Alert thresholds
|
||||
this.thresholds = {
|
||||
errorRate: 0.05, // 5% error rate
|
||||
latencyP95: 5000, // 5 second P95
|
||||
memoryUsage: 0.8, // 80% memory
|
||||
cpuUsage: 0.9, // 90% CPU
|
||||
healthScore: 70 // Health score below 70
|
||||
};
|
||||
// Optimization recommendations
|
||||
this.recommendations = [];
|
||||
// Auto-optimization state
|
||||
this.autoOptimizeEnabled = true;
|
||||
this.lastOptimization = Date.now();
|
||||
this.optimizationInterval = 30000; // Optimize every 30 seconds
|
||||
}
|
||||
/**
|
||||
* Track an operation completion
|
||||
*/
|
||||
trackOperation(success, latency, bytes = 0) {
|
||||
// Update counters
|
||||
this.metrics.totalOperations++;
|
||||
if (success) {
|
||||
this.metrics.successfulOperations++;
|
||||
}
|
||||
else {
|
||||
this.metrics.failedOperations++;
|
||||
}
|
||||
// Track latency
|
||||
this.operationLatencies.push(latency);
|
||||
if (this.operationLatencies.length > 10000) {
|
||||
this.operationLatencies = this.operationLatencies.slice(-5000);
|
||||
}
|
||||
// Track size
|
||||
if (bytes > 0) {
|
||||
this.operationSizes.push(bytes);
|
||||
if (this.operationSizes.length > 10000) {
|
||||
this.operationSizes = this.operationSizes.slice(-5000);
|
||||
}
|
||||
}
|
||||
// Update metrics periodically
|
||||
this.updateMetrics();
|
||||
}
|
||||
/**
|
||||
* Update all metrics
|
||||
*/
|
||||
updateMetrics() {
|
||||
const now = Date.now();
|
||||
const timeSinceReset = (now - this.lastReset) / 1000;
|
||||
// Calculate latency percentiles
|
||||
if (this.operationLatencies.length > 0) {
|
||||
const sorted = [...this.operationLatencies].sort((a, b) => a - b);
|
||||
const p95Index = Math.floor(sorted.length * 0.95);
|
||||
const p99Index = Math.floor(sorted.length * 0.99);
|
||||
this.metrics.averageLatency = sorted.reduce((a, b) => a + b, 0) / sorted.length;
|
||||
this.metrics.p95Latency = sorted[p95Index] || 0;
|
||||
this.metrics.p99Latency = sorted[p99Index] || 0;
|
||||
}
|
||||
// Calculate throughput
|
||||
if (timeSinceReset > 0) {
|
||||
this.metrics.operationsPerSecond = this.metrics.totalOperations / timeSinceReset;
|
||||
const totalBytes = this.operationSizes.reduce((a, b) => a + b, 0);
|
||||
this.metrics.bytesPerSecond = totalBytes / timeSinceReset;
|
||||
}
|
||||
// Calculate error rate
|
||||
this.metrics.errorRate = this.metrics.totalOperations > 0
|
||||
? this.metrics.failedOperations / this.metrics.totalOperations
|
||||
: 0;
|
||||
// Update resource metrics
|
||||
this.updateResourceMetrics();
|
||||
// Calculate health score
|
||||
this.calculateHealthScore();
|
||||
// Store in history
|
||||
this.history.push({ ...this.metrics });
|
||||
if (this.history.length > this.maxHistorySize) {
|
||||
this.history.shift();
|
||||
}
|
||||
// Check for alerts
|
||||
this.checkAlerts();
|
||||
// Auto-optimize if enabled
|
||||
if (this.autoOptimizeEnabled && now - this.lastOptimization > this.optimizationInterval) {
|
||||
this.autoOptimize();
|
||||
this.lastOptimization = now;
|
||||
}
|
||||
// Reset counters periodically
|
||||
if (now - this.lastReset > this.resetInterval) {
|
||||
this.resetCounters();
|
||||
}
|
||||
}
|
||||
/**
|
||||
* Update resource metrics
|
||||
*/
|
||||
updateResourceMetrics() {
|
||||
// Memory usage
|
||||
if (typeof process !== 'undefined' && process.memoryUsage) {
|
||||
const memUsage = process.memoryUsage();
|
||||
this.metrics.memoryUsage = memUsage.heapUsed / memUsage.heapTotal;
|
||||
}
|
||||
// CPU usage (Node.js only)
|
||||
if (this.lastCpuUsage && process.cpuUsage) {
|
||||
const currentCpuUsage = process.cpuUsage();
|
||||
const now = Date.now();
|
||||
const timeDiff = now - this.lastCpuCheck;
|
||||
if (timeDiff > 1000) { // Update CPU every second
|
||||
const userDiff = currentCpuUsage.user - this.lastCpuUsage.user;
|
||||
const systemDiff = currentCpuUsage.system - this.lastCpuUsage.system;
|
||||
const totalDiff = userDiff + systemDiff;
|
||||
// CPU percentage (approximate)
|
||||
this.metrics.cpuUsage = totalDiff / (timeDiff * 1000);
|
||||
this.lastCpuUsage = currentCpuUsage;
|
||||
this.lastCpuCheck = now;
|
||||
}
|
||||
}
|
||||
// Get metrics from socket manager
|
||||
const socketMetrics = getGlobalSocketManager().getMetrics();
|
||||
this.metrics.socketUtilization = socketMetrics.socketUtilization;
|
||||
// Get metrics from backpressure system
|
||||
const backpressureStatus = getGlobalBackpressure().getStatus();
|
||||
this.metrics.queueDepth = backpressureStatus.queueLength;
|
||||
}
|
||||
/**
|
||||
* Calculate overall health score
|
||||
*/
|
||||
calculateHealthScore() {
|
||||
let score = 100;
|
||||
// Deduct points for high error rate
|
||||
if (this.metrics.errorRate > 0.01) {
|
||||
score -= Math.min(30, this.metrics.errorRate * 300);
|
||||
}
|
||||
// Deduct points for high latency
|
||||
if (this.metrics.p95Latency > 3000) {
|
||||
score -= Math.min(20, (this.metrics.p95Latency - 3000) / 100);
|
||||
}
|
||||
// Deduct points for high memory usage
|
||||
if (this.metrics.memoryUsage > 0.7) {
|
||||
score -= Math.min(20, (this.metrics.memoryUsage - 0.7) * 66);
|
||||
}
|
||||
// Deduct points for high CPU usage
|
||||
if (this.metrics.cpuUsage > 0.8) {
|
||||
score -= Math.min(15, (this.metrics.cpuUsage - 0.8) * 75);
|
||||
}
|
||||
// Deduct points for low throughput
|
||||
if (this.metrics.operationsPerSecond < 1 && this.metrics.totalOperations > 10) {
|
||||
score -= 10;
|
||||
}
|
||||
// Deduct points for queue depth
|
||||
if (this.metrics.queueDepth > 100) {
|
||||
score -= Math.min(15, this.metrics.queueDepth / 20);
|
||||
}
|
||||
this.metrics.healthScore = Math.max(0, Math.min(100, score));
|
||||
}
|
||||
/**
|
||||
* Check for alert conditions
|
||||
*/
|
||||
checkAlerts() {
|
||||
const alerts = [];
|
||||
if (this.metrics.errorRate > this.thresholds.errorRate) {
|
||||
alerts.push(`High error rate: ${(this.metrics.errorRate * 100).toFixed(1)}%`);
|
||||
}
|
||||
if (this.metrics.p95Latency > this.thresholds.latencyP95) {
|
||||
alerts.push(`High P95 latency: ${this.metrics.p95Latency}ms`);
|
||||
}
|
||||
if (this.metrics.memoryUsage > this.thresholds.memoryUsage) {
|
||||
alerts.push(`High memory usage: ${(this.metrics.memoryUsage * 100).toFixed(1)}%`);
|
||||
}
|
||||
if (this.metrics.cpuUsage > this.thresholds.cpuUsage) {
|
||||
alerts.push(`High CPU usage: ${(this.metrics.cpuUsage * 100).toFixed(1)}%`);
|
||||
}
|
||||
if (this.metrics.healthScore < this.thresholds.healthScore) {
|
||||
alerts.push(`Low health score: ${this.metrics.healthScore.toFixed(0)}`);
|
||||
}
|
||||
if (alerts.length > 0) {
|
||||
this.logger.warn('Performance alerts', { alerts, metrics: this.metrics });
|
||||
}
|
||||
}
|
||||
/**
|
||||
* Auto-optimize system based on metrics
|
||||
*/
|
||||
autoOptimize() {
|
||||
this.recommendations = [];
|
||||
// Analyze trends
|
||||
const trends = this.analyzeTrends();
|
||||
// Generate recommendations based on metrics and trends
|
||||
if (this.metrics.errorRate > 0.02) {
|
||||
this.recommendations.push('Reduce load or increase timeouts due to high error rate');
|
||||
}
|
||||
if (this.metrics.p95Latency > 3000) {
|
||||
this.recommendations.push('Increase batch size or socket limits to improve latency');
|
||||
}
|
||||
if (this.metrics.memoryUsage > 0.7) {
|
||||
this.recommendations.push('Reduce cache sizes or batch sizes to free memory');
|
||||
}
|
||||
if (this.metrics.queueDepth > 50) {
|
||||
this.recommendations.push('Increase concurrency limits to reduce queue depth');
|
||||
}
|
||||
// Check for degrading trends
|
||||
trends.forEach(trend => {
|
||||
if (trend.direction === 'degrading' && Math.abs(trend.changeRate) > 0.1) {
|
||||
this.recommendations.push(`${trend.metric} is degrading at ${(trend.changeRate * 100).toFixed(1)}% per minute`);
|
||||
}
|
||||
});
|
||||
// Log recommendations if any
|
||||
if (this.recommendations.length > 0) {
|
||||
this.logger.info('Performance optimization recommendations', {
|
||||
recommendations: this.recommendations,
|
||||
metrics: this.metrics
|
||||
});
|
||||
}
|
||||
}
|
||||
/**
|
||||
* Analyze performance trends
|
||||
*/
|
||||
analyzeTrends() {
|
||||
const trends = [];
|
||||
if (this.history.length < 10) {
|
||||
return trends; // Not enough data
|
||||
}
|
||||
// Get recent history
|
||||
const recent = this.history.slice(-20);
|
||||
const older = this.history.slice(-40, -20);
|
||||
// Compare key metrics
|
||||
const metricsToAnalyze = [
|
||||
'errorRate',
|
||||
'averageLatency',
|
||||
'operationsPerSecond',
|
||||
'memoryUsage',
|
||||
'healthScore'
|
||||
];
|
||||
metricsToAnalyze.forEach(metric => {
|
||||
const recentAvg = recent.reduce((sum, m) => sum + m[metric], 0) / recent.length;
|
||||
const olderAvg = older.length > 0
|
||||
? older.reduce((sum, m) => sum + m[metric], 0) / older.length
|
||||
: recentAvg;
|
||||
const changeRate = olderAvg !== 0 ? (recentAvg - olderAvg) / olderAvg : 0;
|
||||
let direction = 'stable';
|
||||
if (Math.abs(changeRate) > 0.05) { // 5% threshold
|
||||
// For error rate and latency, increase is bad
|
||||
if (metric === 'errorRate' || metric === 'averageLatency' || metric === 'memoryUsage') {
|
||||
direction = changeRate > 0 ? 'degrading' : 'improving';
|
||||
}
|
||||
else {
|
||||
// For throughput and health score, increase is good
|
||||
direction = changeRate > 0 ? 'improving' : 'degrading';
|
||||
}
|
||||
}
|
||||
// Simple linear prediction
|
||||
const prediction = recentAvg + (recentAvg * changeRate);
|
||||
trends.push({
|
||||
metric,
|
||||
direction,
|
||||
changeRate,
|
||||
prediction
|
||||
});
|
||||
});
|
||||
return trends;
|
||||
}
|
||||
/**
|
||||
* Reset counters
|
||||
*/
|
||||
resetCounters() {
|
||||
this.metrics.totalOperations = 0;
|
||||
this.metrics.successfulOperations = 0;
|
||||
this.metrics.failedOperations = 0;
|
||||
this.operationSizes = [];
|
||||
this.lastReset = Date.now();
|
||||
}
|
||||
/**
|
||||
* Get current metrics
|
||||
*/
|
||||
getMetrics() {
|
||||
return { ...this.metrics };
|
||||
}
|
||||
/**
|
||||
* Get performance trends
|
||||
*/
|
||||
getTrends() {
|
||||
return this.analyzeTrends();
|
||||
}
|
||||
/**
|
||||
* Get recommendations
|
||||
*/
|
||||
getRecommendations() {
|
||||
return [...this.recommendations];
|
||||
}
|
||||
/**
|
||||
* Get performance report
|
||||
*/
|
||||
getReport() {
|
||||
return {
|
||||
metrics: this.getMetrics(),
|
||||
trends: this.getTrends(),
|
||||
recommendations: this.getRecommendations(),
|
||||
socketConfig: getGlobalSocketManager().getConfig(),
|
||||
backpressureStatus: getGlobalBackpressure().getStatus()
|
||||
};
|
||||
}
|
||||
/**
|
||||
* Enable/disable auto-optimization
|
||||
*/
|
||||
setAutoOptimize(enabled) {
|
||||
this.autoOptimizeEnabled = enabled;
|
||||
this.logger.info(`Auto-optimization ${enabled ? 'enabled' : 'disabled'}`);
|
||||
}
|
||||
/**
|
||||
* Reset all metrics and history
|
||||
*/
|
||||
reset() {
|
||||
this.metrics = {
|
||||
totalOperations: 0,
|
||||
successfulOperations: 0,
|
||||
failedOperations: 0,
|
||||
averageLatency: 0,
|
||||
p95Latency: 0,
|
||||
p99Latency: 0,
|
||||
operationsPerSecond: 0,
|
||||
bytesPerSecond: 0,
|
||||
memoryUsage: 0,
|
||||
cpuUsage: 0,
|
||||
socketUtilization: 0,
|
||||
queueDepth: 0,
|
||||
errorRate: 0,
|
||||
healthScore: 100
|
||||
};
|
||||
this.history = [];
|
||||
this.operationLatencies = [];
|
||||
this.operationSizes = [];
|
||||
this.recommendations = [];
|
||||
this.lastReset = Date.now();
|
||||
this.logger.info('Performance monitor reset');
|
||||
}
|
||||
}
|
||||
// Global singleton instance
|
||||
let globalMonitor = null;
|
||||
/**
|
||||
* Get the global performance monitor instance
|
||||
*/
|
||||
export function getGlobalPerformanceMonitor() {
|
||||
if (!globalMonitor) {
|
||||
globalMonitor = new PerformanceMonitor();
|
||||
}
|
||||
return globalMonitor;
|
||||
}
|
||||
//# sourceMappingURL=performanceMonitor.js.map
|
||||
Loading…
Add table
Add a link
Reference in a new issue