feat: add automatic adaptive performance optimization for high-volume scenarios
- Implement AdaptiveSocketManager for zero-config socket pool scaling - Add AdaptiveBackpressure for intelligent flow control with circuit breaker - Create PerformanceMonitor for real-time metrics and auto-optimization - Automatically adapt to load patterns without manual configuration - Self-healing system that learns from usage patterns - Dynamically adjust batch sizes based on system resources - Automatic recovery from socket exhaustion scenarios - No configuration required - system adapts automatically This addresses socket exhaustion issues reported by bluesky-package by providing automatic, adaptive resource management that scales based on actual load patterns.
This commit is contained in:
parent
e237e81700
commit
9842af2fe3
8 changed files with 1585 additions and 74 deletions
443
src/utils/adaptiveBackpressure.ts
Normal file
443
src/utils/adaptiveBackpressure.ts
Normal file
|
|
@ -0,0 +1,443 @@
|
|||
/**
|
||||
* Adaptive Backpressure System
|
||||
* Automatically manages request flow and prevents system overload
|
||||
* Self-healing with pattern learning for optimal throughput
|
||||
*/
|
||||
|
||||
import { createModuleLogger } from './logger.js'
|
||||
|
||||
interface BackpressureMetrics {
|
||||
queueDepth: number
|
||||
processingRate: number
|
||||
errorRate: number
|
||||
latency: number
|
||||
throughput: number
|
||||
}
|
||||
|
||||
interface BackpressureConfig {
|
||||
maxQueueDepth: number
|
||||
targetLatency: number
|
||||
minThroughput: number
|
||||
adaptationRate: number
|
||||
}
|
||||
|
||||
/**
|
||||
* Self-healing backpressure manager that learns from load patterns
|
||||
*/
|
||||
export class AdaptiveBackpressure {
|
||||
private logger = createModuleLogger('AdaptiveBackpressure')
|
||||
|
||||
// Queue management
|
||||
private queue: Array<{
|
||||
id: string
|
||||
priority: number
|
||||
timestamp: number
|
||||
resolve: () => void
|
||||
}> = []
|
||||
|
||||
// Active operations tracking
|
||||
private activeOperations = new Set<string>()
|
||||
private maxConcurrent = 100
|
||||
|
||||
// Metrics tracking
|
||||
private metrics: BackpressureMetrics = {
|
||||
queueDepth: 0,
|
||||
processingRate: 0,
|
||||
errorRate: 0,
|
||||
latency: 0,
|
||||
throughput: 0
|
||||
}
|
||||
|
||||
// Configuration that adapts over time
|
||||
private config: BackpressureConfig = {
|
||||
maxQueueDepth: 1000,
|
||||
targetLatency: 1000, // 1 second target
|
||||
minThroughput: 10, // Minimum 10 ops/sec
|
||||
adaptationRate: 0.1 // How quickly to adapt
|
||||
}
|
||||
|
||||
// Historical patterns for learning
|
||||
private patterns: Array<{
|
||||
timestamp: number
|
||||
load: number
|
||||
optimal: number
|
||||
}> = []
|
||||
|
||||
// Circuit breaker state
|
||||
private circuitState: 'closed' | 'open' | 'half-open' = 'closed'
|
||||
private circuitOpenTime = 0
|
||||
private circuitFailures = 0
|
||||
private circuitThreshold = 5
|
||||
private circuitTimeout = 30000 // 30 seconds
|
||||
|
||||
// Performance tracking
|
||||
private operationTimes = new Map<string, number>()
|
||||
private completedOps: number[] = []
|
||||
private errorOps = 0
|
||||
private lastAdaptation = Date.now()
|
||||
|
||||
/**
|
||||
* Request permission to proceed with an operation
|
||||
*/
|
||||
public async requestPermission(
|
||||
operationId: string,
|
||||
priority: number = 1
|
||||
): Promise<void> {
|
||||
// Check circuit breaker
|
||||
if (this.isCircuitOpen()) {
|
||||
throw new Error('Circuit breaker is open - system is recovering')
|
||||
}
|
||||
|
||||
// Fast path for low load
|
||||
if (this.activeOperations.size < this.maxConcurrent * 0.5 && this.queue.length === 0) {
|
||||
this.activeOperations.add(operationId)
|
||||
this.operationTimes.set(operationId, Date.now())
|
||||
return
|
||||
}
|
||||
|
||||
// Check if we need to queue
|
||||
if (this.activeOperations.size >= this.maxConcurrent) {
|
||||
// Check queue depth
|
||||
if (this.queue.length >= this.config.maxQueueDepth) {
|
||||
throw new Error('Backpressure queue is full - try again later')
|
||||
}
|
||||
|
||||
// Add to queue and wait
|
||||
return new Promise<void>((resolve) => {
|
||||
this.queue.push({
|
||||
id: operationId,
|
||||
priority,
|
||||
timestamp: Date.now(),
|
||||
resolve
|
||||
})
|
||||
|
||||
// Sort queue by priority (higher priority first)
|
||||
this.queue.sort((a, b) => b.priority - a.priority)
|
||||
|
||||
// Update metrics
|
||||
this.metrics.queueDepth = this.queue.length
|
||||
})
|
||||
}
|
||||
|
||||
// Add to active operations
|
||||
this.activeOperations.add(operationId)
|
||||
this.operationTimes.set(operationId, Date.now())
|
||||
}
|
||||
|
||||
/**
|
||||
* Release permission after operation completes
|
||||
*/
|
||||
public releasePermission(operationId: string, success: boolean = true): void {
|
||||
// Remove from active operations
|
||||
this.activeOperations.delete(operationId)
|
||||
|
||||
// Track completion time
|
||||
const startTime = this.operationTimes.get(operationId)
|
||||
if (startTime) {
|
||||
const duration = Date.now() - startTime
|
||||
this.completedOps.push(duration)
|
||||
this.operationTimes.delete(operationId)
|
||||
|
||||
// Keep array bounded
|
||||
if (this.completedOps.length > 1000) {
|
||||
this.completedOps = this.completedOps.slice(-500)
|
||||
}
|
||||
}
|
||||
|
||||
// Track errors for circuit breaker
|
||||
if (!success) {
|
||||
this.errorOps++
|
||||
this.circuitFailures++
|
||||
|
||||
// Check if we should open circuit
|
||||
if (this.circuitFailures >= this.circuitThreshold) {
|
||||
this.openCircuit()
|
||||
}
|
||||
} else {
|
||||
// Reset circuit failures on success
|
||||
if (this.circuitState === 'half-open') {
|
||||
this.closeCircuit()
|
||||
}
|
||||
}
|
||||
|
||||
// Process queue if there are waiting operations
|
||||
if (this.queue.length > 0 && this.activeOperations.size < this.maxConcurrent) {
|
||||
const next = this.queue.shift()
|
||||
if (next) {
|
||||
this.activeOperations.add(next.id)
|
||||
this.operationTimes.set(next.id, Date.now())
|
||||
next.resolve()
|
||||
|
||||
// Update metrics
|
||||
this.metrics.queueDepth = this.queue.length
|
||||
}
|
||||
}
|
||||
|
||||
// Adapt configuration periodically
|
||||
this.adaptIfNeeded()
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if circuit breaker is open
|
||||
*/
|
||||
private isCircuitOpen(): boolean {
|
||||
if (this.circuitState === 'open') {
|
||||
// Check if timeout has passed
|
||||
if (Date.now() - this.circuitOpenTime > this.circuitTimeout) {
|
||||
this.circuitState = 'half-open'
|
||||
this.logger.info('Circuit breaker entering half-open state')
|
||||
return false
|
||||
}
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
/**
|
||||
* Open the circuit breaker
|
||||
*/
|
||||
private openCircuit(): void {
|
||||
if (this.circuitState !== 'open') {
|
||||
this.circuitState = 'open'
|
||||
this.circuitOpenTime = Date.now()
|
||||
this.logger.warn('Circuit breaker opened due to high error rate')
|
||||
|
||||
// Reduce load immediately
|
||||
this.maxConcurrent = Math.max(10, Math.floor(this.maxConcurrent * 0.3))
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Close the circuit breaker
|
||||
*/
|
||||
private closeCircuit(): void {
|
||||
this.circuitState = 'closed'
|
||||
this.circuitFailures = 0
|
||||
this.logger.info('Circuit breaker closed - system recovered')
|
||||
|
||||
// Gradually increase capacity
|
||||
this.maxConcurrent = Math.min(500, Math.floor(this.maxConcurrent * 1.5))
|
||||
}
|
||||
|
||||
/**
|
||||
* Adapt configuration based on metrics
|
||||
*/
|
||||
private adaptIfNeeded(): void {
|
||||
const now = Date.now()
|
||||
if (now - this.lastAdaptation < 5000) { // Adapt every 5 seconds
|
||||
return
|
||||
}
|
||||
|
||||
this.lastAdaptation = now
|
||||
this.updateMetrics()
|
||||
|
||||
// Learn from current patterns
|
||||
this.learnPattern()
|
||||
|
||||
// Adapt based on metrics
|
||||
this.adaptConfiguration()
|
||||
}
|
||||
|
||||
/**
|
||||
* Update current metrics
|
||||
*/
|
||||
private updateMetrics(): void {
|
||||
// Calculate processing rate
|
||||
this.metrics.processingRate = this.completedOps.length > 0
|
||||
? 1000 / (this.completedOps.reduce((a, b) => a + b, 0) / this.completedOps.length)
|
||||
: 0
|
||||
|
||||
// Calculate error rate
|
||||
const totalOps = this.completedOps.length + this.errorOps
|
||||
this.metrics.errorRate = totalOps > 0 ? this.errorOps / totalOps : 0
|
||||
|
||||
// Calculate average latency
|
||||
this.metrics.latency = this.completedOps.length > 0
|
||||
? this.completedOps.reduce((a, b) => a + b, 0) / this.completedOps.length
|
||||
: 0
|
||||
|
||||
// Calculate throughput
|
||||
this.metrics.throughput = this.activeOperations.size + this.metrics.processingRate
|
||||
|
||||
// Reset error counter periodically
|
||||
if (this.completedOps.length > 100) {
|
||||
this.errorOps = Math.floor(this.errorOps * 0.9) // Decay error count
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Learn from current load patterns
|
||||
*/
|
||||
private learnPattern(): void {
|
||||
const currentLoad = this.activeOperations.size + this.queue.length
|
||||
const optimalConcurrency = this.calculateOptimalConcurrency()
|
||||
|
||||
this.patterns.push({
|
||||
timestamp: Date.now(),
|
||||
load: currentLoad,
|
||||
optimal: optimalConcurrency
|
||||
})
|
||||
|
||||
// Keep patterns bounded
|
||||
if (this.patterns.length > 1000) {
|
||||
this.patterns = this.patterns.slice(-500)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Calculate optimal concurrency based on Little's Law
|
||||
*/
|
||||
private calculateOptimalConcurrency(): number {
|
||||
// Little's Law: L = λ * W
|
||||
// L = number of requests in system
|
||||
// λ = arrival rate
|
||||
// W = average time in system
|
||||
|
||||
if (this.metrics.latency === 0 || this.metrics.processingRate === 0) {
|
||||
return this.maxConcurrent // Keep current if no data
|
||||
}
|
||||
|
||||
// Target: Keep latency under target while maximizing throughput
|
||||
const targetConcurrency = Math.ceil(
|
||||
this.metrics.processingRate * (this.config.targetLatency / 1000)
|
||||
)
|
||||
|
||||
// Adjust based on error rate
|
||||
const errorAdjustment = 1 - (this.metrics.errorRate * 2) // Reduce by up to 50% for errors
|
||||
|
||||
// Apply adjustment
|
||||
const adjusted = Math.floor(targetConcurrency * errorAdjustment)
|
||||
|
||||
// Apply bounds
|
||||
return Math.max(10, Math.min(500, adjusted))
|
||||
}
|
||||
|
||||
/**
|
||||
* Adapt configuration based on metrics and patterns
|
||||
*/
|
||||
private adaptConfiguration(): void {
|
||||
const optimal = this.calculateOptimalConcurrency()
|
||||
const current = this.maxConcurrent
|
||||
|
||||
// Smooth adaptation using exponential moving average
|
||||
const newConcurrency = Math.floor(
|
||||
current * (1 - this.config.adaptationRate) +
|
||||
optimal * this.config.adaptationRate
|
||||
)
|
||||
|
||||
// Check if adaptation is needed
|
||||
if (Math.abs(newConcurrency - current) > current * 0.1) { // 10% threshold
|
||||
const oldValue = this.maxConcurrent
|
||||
this.maxConcurrent = newConcurrency
|
||||
|
||||
this.logger.debug('Adapted concurrency', {
|
||||
from: oldValue,
|
||||
to: newConcurrency,
|
||||
metrics: this.metrics
|
||||
})
|
||||
}
|
||||
|
||||
// Adapt queue depth based on throughput
|
||||
if (this.metrics.throughput > 0) {
|
||||
// Allow queue depth to be 10 seconds worth of throughput
|
||||
this.config.maxQueueDepth = Math.max(
|
||||
100,
|
||||
Math.min(10000, Math.floor(this.metrics.throughput * 10))
|
||||
)
|
||||
}
|
||||
|
||||
// Adapt circuit breaker threshold based on error patterns
|
||||
if (this.metrics.errorRate < 0.01 && this.circuitThreshold > 5) {
|
||||
this.circuitThreshold = Math.max(5, this.circuitThreshold - 1)
|
||||
} else if (this.metrics.errorRate > 0.05 && this.circuitThreshold < 20) {
|
||||
this.circuitThreshold = Math.min(20, this.circuitThreshold + 1)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Predict future load based on patterns
|
||||
*/
|
||||
public predictLoad(futureSeconds: number = 60): number {
|
||||
if (this.patterns.length < 10) {
|
||||
return this.maxConcurrent // Not enough data
|
||||
}
|
||||
|
||||
// Simple linear regression on recent patterns
|
||||
const recentPatterns = this.patterns.slice(-50)
|
||||
const n = recentPatterns.length
|
||||
|
||||
// Calculate averages
|
||||
let sumX = 0, sumY = 0, sumXY = 0, sumX2 = 0
|
||||
const startTime = recentPatterns[0].timestamp
|
||||
|
||||
recentPatterns.forEach(p => {
|
||||
const x = (p.timestamp - startTime) / 1000 // Time in seconds
|
||||
const y = p.load
|
||||
sumX += x
|
||||
sumY += y
|
||||
sumXY += x * y
|
||||
sumX2 += x * x
|
||||
})
|
||||
|
||||
// Calculate slope and intercept
|
||||
const slope = (n * sumXY - sumX * sumY) / (n * sumX2 - sumX * sumX)
|
||||
const intercept = (sumY - slope * sumX) / n
|
||||
|
||||
// Predict future load
|
||||
const currentTime = (Date.now() - startTime) / 1000
|
||||
const predictedLoad = intercept + slope * (currentTime + futureSeconds)
|
||||
|
||||
return Math.max(0, Math.min(this.config.maxQueueDepth, Math.floor(predictedLoad)))
|
||||
}
|
||||
|
||||
/**
|
||||
* Get current configuration and metrics
|
||||
*/
|
||||
public getStatus(): {
|
||||
config: BackpressureConfig
|
||||
metrics: BackpressureMetrics
|
||||
circuit: string
|
||||
maxConcurrent: number
|
||||
activeOps: number
|
||||
queueLength: number
|
||||
} {
|
||||
return {
|
||||
config: { ...this.config },
|
||||
metrics: { ...this.metrics },
|
||||
circuit: this.circuitState,
|
||||
maxConcurrent: this.maxConcurrent,
|
||||
activeOps: this.activeOperations.size,
|
||||
queueLength: this.queue.length
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Reset to default state
|
||||
*/
|
||||
public reset(): void {
|
||||
this.queue = []
|
||||
this.activeOperations.clear()
|
||||
this.operationTimes.clear()
|
||||
this.completedOps = []
|
||||
this.errorOps = 0
|
||||
this.patterns = []
|
||||
this.circuitState = 'closed'
|
||||
this.circuitFailures = 0
|
||||
this.maxConcurrent = 100
|
||||
|
||||
this.logger.info('Backpressure system reset to defaults')
|
||||
}
|
||||
}
|
||||
|
||||
// Global singleton instance
|
||||
let globalBackpressure: AdaptiveBackpressure | null = null
|
||||
|
||||
/**
|
||||
* Get the global backpressure instance
|
||||
*/
|
||||
export function getGlobalBackpressure(): AdaptiveBackpressure {
|
||||
if (!globalBackpressure) {
|
||||
globalBackpressure = new AdaptiveBackpressure()
|
||||
}
|
||||
return globalBackpressure
|
||||
}
|
||||
474
src/utils/adaptiveSocketManager.ts
Normal file
474
src/utils/adaptiveSocketManager.ts
Normal file
|
|
@ -0,0 +1,474 @@
|
|||
/**
|
||||
* Adaptive Socket Manager
|
||||
* Automatically manages socket pools and connection settings based on load patterns
|
||||
* Zero-configuration approach that learns and adapts to workload characteristics
|
||||
*/
|
||||
|
||||
import { Agent as HttpsAgent } from 'https'
|
||||
import { NodeHttpHandler } from '@smithy/node-http-handler'
|
||||
import { createModuleLogger } from './logger.js'
|
||||
|
||||
interface LoadMetrics {
|
||||
requestsPerSecond: number
|
||||
pendingRequests: number
|
||||
socketUtilization: number
|
||||
errorRate: number
|
||||
latencyP50: number
|
||||
latencyP95: number
|
||||
memoryUsage: number
|
||||
}
|
||||
|
||||
interface AdaptiveConfig {
|
||||
maxSockets: number
|
||||
maxFreeSockets: number
|
||||
keepAliveTimeout: number
|
||||
connectionTimeout: number
|
||||
socketTimeout: number
|
||||
batchSize: number
|
||||
}
|
||||
|
||||
/**
|
||||
* Adaptive Socket Manager that automatically scales based on load patterns
|
||||
*/
|
||||
export class AdaptiveSocketManager {
|
||||
private logger = createModuleLogger('AdaptiveSocketManager')
|
||||
|
||||
// Current configuration
|
||||
private config: AdaptiveConfig = {
|
||||
maxSockets: 100, // Start conservative
|
||||
maxFreeSockets: 20,
|
||||
keepAliveTimeout: 60000,
|
||||
connectionTimeout: 10000,
|
||||
socketTimeout: 60000,
|
||||
batchSize: 10
|
||||
}
|
||||
|
||||
// Performance tracking
|
||||
private metrics: LoadMetrics = {
|
||||
requestsPerSecond: 0,
|
||||
pendingRequests: 0,
|
||||
socketUtilization: 0,
|
||||
errorRate: 0,
|
||||
latencyP50: 0,
|
||||
latencyP95: 0,
|
||||
memoryUsage: 0
|
||||
}
|
||||
|
||||
// Historical data for learning
|
||||
private history: LoadMetrics[] = []
|
||||
private maxHistorySize = 100
|
||||
|
||||
// Adaptation state
|
||||
private lastAdaptationTime = 0
|
||||
private adaptationInterval = 5000 // Check every 5 seconds
|
||||
private consecutiveHighLoad = 0
|
||||
private consecutiveLowLoad = 0
|
||||
|
||||
// Request tracking
|
||||
private requestStartTimes = new Map<string, number>()
|
||||
private requestLatencies: number[] = []
|
||||
private errorCount = 0
|
||||
private successCount = 0
|
||||
private lastMetricReset = Date.now()
|
||||
|
||||
// Socket pool instances
|
||||
private currentAgent: HttpsAgent | null = null
|
||||
private currentHandler: NodeHttpHandler | null = null
|
||||
|
||||
/**
|
||||
* Get or create an optimized HTTP handler
|
||||
*/
|
||||
public getHttpHandler(): NodeHttpHandler {
|
||||
// Adapt configuration if needed
|
||||
this.adaptIfNeeded()
|
||||
|
||||
// Create new handler if configuration changed
|
||||
if (!this.currentHandler || this.shouldRecreateHandler()) {
|
||||
this.currentAgent = new HttpsAgent({
|
||||
keepAlive: true,
|
||||
maxSockets: this.config.maxSockets,
|
||||
maxFreeSockets: this.config.maxFreeSockets,
|
||||
timeout: this.config.keepAliveTimeout,
|
||||
scheduling: 'fifo' // Fair scheduling for high-volume scenarios
|
||||
})
|
||||
|
||||
this.currentHandler = new NodeHttpHandler({
|
||||
httpsAgent: this.currentAgent,
|
||||
connectionTimeout: this.config.connectionTimeout,
|
||||
socketTimeout: this.config.socketTimeout
|
||||
})
|
||||
|
||||
this.logger.debug('Created new HTTP handler with config:', this.config)
|
||||
}
|
||||
|
||||
return this.currentHandler
|
||||
}
|
||||
|
||||
/**
|
||||
* Get current batch size recommendation
|
||||
*/
|
||||
public getBatchSize(): number {
|
||||
this.adaptIfNeeded()
|
||||
return this.config.batchSize
|
||||
}
|
||||
|
||||
/**
|
||||
* Track request start
|
||||
*/
|
||||
public trackRequestStart(requestId: string): void {
|
||||
this.requestStartTimes.set(requestId, Date.now())
|
||||
this.metrics.pendingRequests++
|
||||
}
|
||||
|
||||
/**
|
||||
* Track request completion
|
||||
*/
|
||||
public trackRequestComplete(requestId: string, success: boolean): void {
|
||||
const startTime = this.requestStartTimes.get(requestId)
|
||||
if (startTime) {
|
||||
const latency = Date.now() - startTime
|
||||
this.requestLatencies.push(latency)
|
||||
this.requestStartTimes.delete(requestId)
|
||||
|
||||
// Keep latency array bounded
|
||||
if (this.requestLatencies.length > 1000) {
|
||||
this.requestLatencies = this.requestLatencies.slice(-500)
|
||||
}
|
||||
}
|
||||
|
||||
if (success) {
|
||||
this.successCount++
|
||||
} else {
|
||||
this.errorCount++
|
||||
}
|
||||
|
||||
this.metrics.pendingRequests = Math.max(0, this.metrics.pendingRequests - 1)
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if we should adapt configuration
|
||||
*/
|
||||
private adaptIfNeeded(): void {
|
||||
const now = Date.now()
|
||||
if (now - this.lastAdaptationTime < this.adaptationInterval) {
|
||||
return
|
||||
}
|
||||
|
||||
this.lastAdaptationTime = now
|
||||
this.updateMetrics()
|
||||
this.analyzeAndAdapt()
|
||||
}
|
||||
|
||||
/**
|
||||
* Update current metrics
|
||||
*/
|
||||
private updateMetrics(): void {
|
||||
const now = Date.now()
|
||||
const timeSinceReset = (now - this.lastMetricReset) / 1000
|
||||
|
||||
// Calculate requests per second
|
||||
const totalRequests = this.successCount + this.errorCount
|
||||
this.metrics.requestsPerSecond = timeSinceReset > 0
|
||||
? totalRequests / timeSinceReset
|
||||
: 0
|
||||
|
||||
// Calculate error rate
|
||||
this.metrics.errorRate = totalRequests > 0
|
||||
? this.errorCount / totalRequests
|
||||
: 0
|
||||
|
||||
// Calculate latency percentiles
|
||||
if (this.requestLatencies.length > 0) {
|
||||
const sorted = [...this.requestLatencies].sort((a, b) => a - b)
|
||||
const p50Index = Math.floor(sorted.length * 0.5)
|
||||
const p95Index = Math.floor(sorted.length * 0.95)
|
||||
this.metrics.latencyP50 = sorted[p50Index] || 0
|
||||
this.metrics.latencyP95 = sorted[p95Index] || 0
|
||||
}
|
||||
|
||||
// Calculate socket utilization
|
||||
this.metrics.socketUtilization = this.metrics.pendingRequests / this.config.maxSockets
|
||||
|
||||
// Memory usage
|
||||
if (typeof process !== 'undefined' && process.memoryUsage) {
|
||||
const memUsage = process.memoryUsage()
|
||||
this.metrics.memoryUsage = memUsage.heapUsed / memUsage.heapTotal
|
||||
}
|
||||
|
||||
// Add to history
|
||||
this.history.push({ ...this.metrics })
|
||||
if (this.history.length > this.maxHistorySize) {
|
||||
this.history.shift()
|
||||
}
|
||||
|
||||
// Reset counters periodically
|
||||
if (timeSinceReset > 60) {
|
||||
this.lastMetricReset = now
|
||||
this.successCount = 0
|
||||
this.errorCount = 0
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Analyze metrics and adapt configuration
|
||||
*/
|
||||
private analyzeAndAdapt(): void {
|
||||
const wasConfig = { ...this.config }
|
||||
|
||||
// Detect high load conditions
|
||||
const isHighLoad = this.detectHighLoad()
|
||||
const isLowLoad = this.detectLowLoad()
|
||||
const hasErrors = this.metrics.errorRate > 0.01 // More than 1% errors
|
||||
|
||||
if (isHighLoad) {
|
||||
this.consecutiveHighLoad++
|
||||
this.consecutiveLowLoad = 0
|
||||
|
||||
if (this.consecutiveHighLoad >= 2) { // Wait for 2 consecutive high load readings
|
||||
this.scaleUp()
|
||||
}
|
||||
} else if (isLowLoad) {
|
||||
this.consecutiveLowLoad++
|
||||
this.consecutiveHighLoad = 0
|
||||
|
||||
if (this.consecutiveLowLoad >= 6) { // Wait longer before scaling down
|
||||
this.scaleDown()
|
||||
}
|
||||
} else {
|
||||
// Reset counters if load is normal
|
||||
this.consecutiveHighLoad = Math.max(0, this.consecutiveHighLoad - 1)
|
||||
this.consecutiveLowLoad = Math.max(0, this.consecutiveLowLoad - 1)
|
||||
}
|
||||
|
||||
// Handle error conditions
|
||||
if (hasErrors) {
|
||||
this.handleErrors()
|
||||
}
|
||||
|
||||
// Log significant changes
|
||||
if (JSON.stringify(wasConfig) !== JSON.stringify(this.config)) {
|
||||
this.logger.info('Adapted configuration', {
|
||||
from: wasConfig,
|
||||
to: this.config,
|
||||
metrics: this.metrics
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Detect high load conditions
|
||||
*/
|
||||
private detectHighLoad(): boolean {
|
||||
return (
|
||||
this.metrics.socketUtilization > 0.7 || // Sockets heavily used
|
||||
this.metrics.pendingRequests > this.config.maxSockets * 0.8 || // Many pending requests
|
||||
this.metrics.latencyP95 > 5000 || // High latency
|
||||
this.metrics.requestsPerSecond > 100 // High request rate
|
||||
)
|
||||
}
|
||||
|
||||
/**
|
||||
* Detect low load conditions
|
||||
*/
|
||||
private detectLowLoad(): boolean {
|
||||
return (
|
||||
this.metrics.socketUtilization < 0.2 && // Sockets barely used
|
||||
this.metrics.pendingRequests < 5 && // Few pending requests
|
||||
this.metrics.latencyP95 < 1000 && // Low latency
|
||||
this.metrics.requestsPerSecond < 10 && // Low request rate
|
||||
this.metrics.memoryUsage < 0.5 // Low memory usage
|
||||
)
|
||||
}
|
||||
|
||||
/**
|
||||
* Scale up resources for high load
|
||||
*/
|
||||
private scaleUp(): void {
|
||||
// Increase socket limits progressively
|
||||
const scaleFactor = this.metrics.errorRate > 0.05 ? 1.5 : 2.0 // Scale more aggressively if no errors
|
||||
|
||||
this.config.maxSockets = Math.min(
|
||||
2000, // Hard limit to prevent resource exhaustion
|
||||
Math.ceil(this.config.maxSockets * scaleFactor)
|
||||
)
|
||||
|
||||
this.config.maxFreeSockets = Math.min(
|
||||
200,
|
||||
Math.ceil(this.config.maxSockets * 0.1) // Keep 10% as free sockets
|
||||
)
|
||||
|
||||
// Increase batch size for better throughput
|
||||
this.config.batchSize = Math.min(
|
||||
100,
|
||||
Math.ceil(this.config.batchSize * 1.5)
|
||||
)
|
||||
|
||||
// Adjust timeouts for high load
|
||||
this.config.keepAliveTimeout = 120000 // Keep connections alive longer
|
||||
this.config.connectionTimeout = 15000 // Allow more time for connections
|
||||
this.config.socketTimeout = 90000 // Allow more time for responses
|
||||
|
||||
this.logger.debug('Scaled up for high load', {
|
||||
sockets: this.config.maxSockets,
|
||||
batchSize: this.config.batchSize
|
||||
})
|
||||
}
|
||||
|
||||
/**
|
||||
* Scale down resources for low load
|
||||
*/
|
||||
private scaleDown(): void {
|
||||
// Only scale down if memory pressure is low
|
||||
if (this.metrics.memoryUsage > 0.7) {
|
||||
return
|
||||
}
|
||||
|
||||
// Decrease socket limits conservatively
|
||||
this.config.maxSockets = Math.max(
|
||||
50, // Minimum sockets
|
||||
Math.floor(this.config.maxSockets * 0.7)
|
||||
)
|
||||
|
||||
this.config.maxFreeSockets = Math.max(
|
||||
10,
|
||||
Math.floor(this.config.maxSockets * 0.2) // Keep 20% as free sockets
|
||||
)
|
||||
|
||||
// Decrease batch size
|
||||
this.config.batchSize = Math.max(
|
||||
5,
|
||||
Math.floor(this.config.batchSize * 0.7)
|
||||
)
|
||||
|
||||
// Adjust timeouts for low load
|
||||
this.config.keepAliveTimeout = 60000
|
||||
this.config.connectionTimeout = 10000
|
||||
this.config.socketTimeout = 60000
|
||||
|
||||
this.logger.debug('Scaled down for low load', {
|
||||
sockets: this.config.maxSockets,
|
||||
batchSize: this.config.batchSize
|
||||
})
|
||||
}
|
||||
|
||||
/**
|
||||
* Handle error conditions by adjusting configuration
|
||||
*/
|
||||
private handleErrors(): void {
|
||||
const errorRate = this.metrics.errorRate
|
||||
|
||||
if (errorRate > 0.1) { // More than 10% errors
|
||||
// Severe errors - back off aggressively
|
||||
this.config.maxSockets = Math.max(50, Math.floor(this.config.maxSockets * 0.5))
|
||||
this.config.batchSize = Math.max(1, Math.floor(this.config.batchSize * 0.3))
|
||||
this.config.connectionTimeout = Math.min(30000, this.config.connectionTimeout * 2)
|
||||
this.config.socketTimeout = Math.min(120000, this.config.socketTimeout * 2)
|
||||
|
||||
this.logger.warn('High error rate detected, backing off', {
|
||||
errorRate,
|
||||
newConfig: this.config
|
||||
})
|
||||
} else if (errorRate > 0.05) { // More than 5% errors
|
||||
// Moderate errors - reduce load slightly
|
||||
this.config.batchSize = Math.max(1, Math.floor(this.config.batchSize * 0.7))
|
||||
this.config.connectionTimeout = Math.min(20000, this.config.connectionTimeout * 1.2)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if we should recreate the handler
|
||||
*/
|
||||
private shouldRecreateHandler(): boolean {
|
||||
if (!this.currentAgent) return true
|
||||
|
||||
// Recreate if socket configuration changed significantly
|
||||
const currentMaxSockets = (this.currentAgent as any).maxSockets
|
||||
const socketsDiff = Math.abs(currentMaxSockets - this.config.maxSockets)
|
||||
|
||||
return socketsDiff > currentMaxSockets * 0.5 // 50% change threshold
|
||||
}
|
||||
|
||||
/**
|
||||
* Get current configuration (for monitoring)
|
||||
*/
|
||||
public getConfig(): Readonly<AdaptiveConfig> {
|
||||
return { ...this.config }
|
||||
}
|
||||
|
||||
/**
|
||||
* Get current metrics (for monitoring)
|
||||
*/
|
||||
public getMetrics(): Readonly<LoadMetrics> {
|
||||
return { ...this.metrics }
|
||||
}
|
||||
|
||||
/**
|
||||
* Predict optimal configuration based on historical data
|
||||
*/
|
||||
public predictOptimalConfig(): AdaptiveConfig {
|
||||
if (this.history.length < 10) {
|
||||
return this.config // Not enough data to predict
|
||||
}
|
||||
|
||||
// Analyze recent history
|
||||
const recentHistory = this.history.slice(-20)
|
||||
const avgRPS = recentHistory.reduce((sum, m) => sum + m.requestsPerSecond, 0) / recentHistory.length
|
||||
const maxRPS = Math.max(...recentHistory.map(m => m.requestsPerSecond))
|
||||
const avgLatency = recentHistory.reduce((sum, m) => sum + m.latencyP95, 0) / recentHistory.length
|
||||
|
||||
// Predict optimal socket count based on request patterns
|
||||
const optimalSockets = Math.min(2000, Math.max(50, Math.ceil(maxRPS * 2)))
|
||||
|
||||
// Predict optimal batch size based on latency
|
||||
const optimalBatchSize = avgLatency < 1000 ? 50 : avgLatency < 3000 ? 20 : 10
|
||||
|
||||
return {
|
||||
maxSockets: optimalSockets,
|
||||
maxFreeSockets: Math.ceil(optimalSockets * 0.15),
|
||||
keepAliveTimeout: avgRPS > 50 ? 120000 : 60000,
|
||||
connectionTimeout: avgLatency > 3000 ? 20000 : 10000,
|
||||
socketTimeout: avgLatency > 3000 ? 90000 : 60000,
|
||||
batchSize: optimalBatchSize
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Reset to default configuration
|
||||
*/
|
||||
public reset(): void {
|
||||
this.config = {
|
||||
maxSockets: 100,
|
||||
maxFreeSockets: 20,
|
||||
keepAliveTimeout: 60000,
|
||||
connectionTimeout: 10000,
|
||||
socketTimeout: 60000,
|
||||
batchSize: 10
|
||||
}
|
||||
|
||||
this.consecutiveHighLoad = 0
|
||||
this.consecutiveLowLoad = 0
|
||||
this.history = []
|
||||
this.requestLatencies = []
|
||||
this.errorCount = 0
|
||||
this.successCount = 0
|
||||
|
||||
// Force recreation of handler
|
||||
this.currentAgent = null
|
||||
this.currentHandler = null
|
||||
|
||||
this.logger.info('Reset to default configuration')
|
||||
}
|
||||
}
|
||||
|
||||
// Global singleton instance
|
||||
let globalSocketManager: AdaptiveSocketManager | null = null
|
||||
|
||||
/**
|
||||
* Get the global socket manager instance
|
||||
*/
|
||||
export function getGlobalSocketManager(): AdaptiveSocketManager {
|
||||
if (!globalSocketManager) {
|
||||
globalSocketManager = new AdaptiveSocketManager()
|
||||
}
|
||||
return globalSocketManager
|
||||
}
|
||||
496
src/utils/performanceMonitor.ts
Normal file
496
src/utils/performanceMonitor.ts
Normal file
|
|
@ -0,0 +1,496 @@
|
|||
/**
|
||||
* Performance Monitor
|
||||
* Automatically tracks and optimizes system performance
|
||||
* Provides real-time insights and auto-tuning recommendations
|
||||
*/
|
||||
|
||||
import { createModuleLogger } from './logger.js'
|
||||
import { getGlobalSocketManager } from './adaptiveSocketManager.js'
|
||||
import { getGlobalBackpressure } from './adaptiveBackpressure.js'
|
||||
|
||||
interface PerformanceMetrics {
|
||||
// Operation metrics
|
||||
totalOperations: number
|
||||
successfulOperations: number
|
||||
failedOperations: number
|
||||
averageLatency: number
|
||||
p95Latency: number
|
||||
p99Latency: number
|
||||
|
||||
// Throughput metrics
|
||||
operationsPerSecond: number
|
||||
bytesPerSecond: number
|
||||
|
||||
// Resource metrics
|
||||
memoryUsage: number
|
||||
cpuUsage: number
|
||||
socketUtilization: number
|
||||
queueDepth: number
|
||||
|
||||
// Health indicators
|
||||
errorRate: number
|
||||
healthScore: number // 0-100
|
||||
}
|
||||
|
||||
interface PerformanceTrend {
|
||||
metric: string
|
||||
direction: 'improving' | 'degrading' | 'stable'
|
||||
changeRate: number
|
||||
prediction: number
|
||||
}
|
||||
|
||||
/**
|
||||
* Comprehensive performance monitoring and optimization
|
||||
*/
|
||||
export class PerformanceMonitor {
|
||||
private logger = createModuleLogger('PerformanceMonitor')
|
||||
|
||||
// Current metrics
|
||||
private metrics: PerformanceMetrics = {
|
||||
totalOperations: 0,
|
||||
successfulOperations: 0,
|
||||
failedOperations: 0,
|
||||
averageLatency: 0,
|
||||
p95Latency: 0,
|
||||
p99Latency: 0,
|
||||
operationsPerSecond: 0,
|
||||
bytesPerSecond: 0,
|
||||
memoryUsage: 0,
|
||||
cpuUsage: 0,
|
||||
socketUtilization: 0,
|
||||
queueDepth: 0,
|
||||
errorRate: 0,
|
||||
healthScore: 100
|
||||
}
|
||||
|
||||
// Historical data for trend analysis
|
||||
private history: PerformanceMetrics[] = []
|
||||
private maxHistorySize = 1000
|
||||
|
||||
// Operation tracking
|
||||
private operationLatencies: number[] = []
|
||||
private operationSizes: number[] = []
|
||||
private lastReset = Date.now()
|
||||
private resetInterval = 60000 // Reset counters every minute
|
||||
|
||||
// CPU tracking
|
||||
private lastCpuUsage = process.cpuUsage ? process.cpuUsage() : null
|
||||
private lastCpuCheck = Date.now()
|
||||
|
||||
// Alert thresholds
|
||||
private thresholds = {
|
||||
errorRate: 0.05, // 5% error rate
|
||||
latencyP95: 5000, // 5 second P95
|
||||
memoryUsage: 0.8, // 80% memory
|
||||
cpuUsage: 0.9, // 90% CPU
|
||||
healthScore: 70 // Health score below 70
|
||||
}
|
||||
|
||||
// Optimization recommendations
|
||||
private recommendations: string[] = []
|
||||
|
||||
// Auto-optimization state
|
||||
private autoOptimizeEnabled = true
|
||||
private lastOptimization = Date.now()
|
||||
private optimizationInterval = 30000 // Optimize every 30 seconds
|
||||
|
||||
/**
|
||||
* Track an operation completion
|
||||
*/
|
||||
public trackOperation(
|
||||
success: boolean,
|
||||
latency: number,
|
||||
bytes: number = 0
|
||||
): void {
|
||||
// Update counters
|
||||
this.metrics.totalOperations++
|
||||
if (success) {
|
||||
this.metrics.successfulOperations++
|
||||
} else {
|
||||
this.metrics.failedOperations++
|
||||
}
|
||||
|
||||
// Track latency
|
||||
this.operationLatencies.push(latency)
|
||||
if (this.operationLatencies.length > 10000) {
|
||||
this.operationLatencies = this.operationLatencies.slice(-5000)
|
||||
}
|
||||
|
||||
// Track size
|
||||
if (bytes > 0) {
|
||||
this.operationSizes.push(bytes)
|
||||
if (this.operationSizes.length > 10000) {
|
||||
this.operationSizes = this.operationSizes.slice(-5000)
|
||||
}
|
||||
}
|
||||
|
||||
// Update metrics periodically
|
||||
this.updateMetrics()
|
||||
}
|
||||
|
||||
/**
|
||||
* Update all metrics
|
||||
*/
|
||||
private updateMetrics(): void {
|
||||
const now = Date.now()
|
||||
const timeSinceReset = (now - this.lastReset) / 1000
|
||||
|
||||
// Calculate latency percentiles
|
||||
if (this.operationLatencies.length > 0) {
|
||||
const sorted = [...this.operationLatencies].sort((a, b) => a - b)
|
||||
const p95Index = Math.floor(sorted.length * 0.95)
|
||||
const p99Index = Math.floor(sorted.length * 0.99)
|
||||
|
||||
this.metrics.averageLatency = sorted.reduce((a, b) => a + b, 0) / sorted.length
|
||||
this.metrics.p95Latency = sorted[p95Index] || 0
|
||||
this.metrics.p99Latency = sorted[p99Index] || 0
|
||||
}
|
||||
|
||||
// Calculate throughput
|
||||
if (timeSinceReset > 0) {
|
||||
this.metrics.operationsPerSecond = this.metrics.totalOperations / timeSinceReset
|
||||
|
||||
const totalBytes = this.operationSizes.reduce((a, b) => a + b, 0)
|
||||
this.metrics.bytesPerSecond = totalBytes / timeSinceReset
|
||||
}
|
||||
|
||||
// Calculate error rate
|
||||
this.metrics.errorRate = this.metrics.totalOperations > 0
|
||||
? this.metrics.failedOperations / this.metrics.totalOperations
|
||||
: 0
|
||||
|
||||
// Update resource metrics
|
||||
this.updateResourceMetrics()
|
||||
|
||||
// Calculate health score
|
||||
this.calculateHealthScore()
|
||||
|
||||
// Store in history
|
||||
this.history.push({ ...this.metrics })
|
||||
if (this.history.length > this.maxHistorySize) {
|
||||
this.history.shift()
|
||||
}
|
||||
|
||||
// Check for alerts
|
||||
this.checkAlerts()
|
||||
|
||||
// Auto-optimize if enabled
|
||||
if (this.autoOptimizeEnabled && now - this.lastOptimization > this.optimizationInterval) {
|
||||
this.autoOptimize()
|
||||
this.lastOptimization = now
|
||||
}
|
||||
|
||||
// Reset counters periodically
|
||||
if (now - this.lastReset > this.resetInterval) {
|
||||
this.resetCounters()
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Update resource metrics
|
||||
*/
|
||||
private updateResourceMetrics(): void {
|
||||
// Memory usage
|
||||
if (typeof process !== 'undefined' && process.memoryUsage) {
|
||||
const memUsage = process.memoryUsage()
|
||||
this.metrics.memoryUsage = memUsage.heapUsed / memUsage.heapTotal
|
||||
}
|
||||
|
||||
// CPU usage (Node.js only)
|
||||
if (this.lastCpuUsage && process.cpuUsage) {
|
||||
const currentCpuUsage = process.cpuUsage()
|
||||
const now = Date.now()
|
||||
const timeDiff = now - this.lastCpuCheck
|
||||
|
||||
if (timeDiff > 1000) { // Update CPU every second
|
||||
const userDiff = currentCpuUsage.user - this.lastCpuUsage.user
|
||||
const systemDiff = currentCpuUsage.system - this.lastCpuUsage.system
|
||||
const totalDiff = userDiff + systemDiff
|
||||
|
||||
// CPU percentage (approximate)
|
||||
this.metrics.cpuUsage = totalDiff / (timeDiff * 1000)
|
||||
|
||||
this.lastCpuUsage = currentCpuUsage
|
||||
this.lastCpuCheck = now
|
||||
}
|
||||
}
|
||||
|
||||
// Get metrics from socket manager
|
||||
const socketMetrics = getGlobalSocketManager().getMetrics()
|
||||
this.metrics.socketUtilization = socketMetrics.socketUtilization
|
||||
|
||||
// Get metrics from backpressure system
|
||||
const backpressureStatus = getGlobalBackpressure().getStatus()
|
||||
this.metrics.queueDepth = backpressureStatus.queueLength
|
||||
}
|
||||
|
||||
/**
|
||||
* Calculate overall health score
|
||||
*/
|
||||
private calculateHealthScore(): void {
|
||||
let score = 100
|
||||
|
||||
// Deduct points for high error rate
|
||||
if (this.metrics.errorRate > 0.01) {
|
||||
score -= Math.min(30, this.metrics.errorRate * 300)
|
||||
}
|
||||
|
||||
// Deduct points for high latency
|
||||
if (this.metrics.p95Latency > 3000) {
|
||||
score -= Math.min(20, (this.metrics.p95Latency - 3000) / 100)
|
||||
}
|
||||
|
||||
// Deduct points for high memory usage
|
||||
if (this.metrics.memoryUsage > 0.7) {
|
||||
score -= Math.min(20, (this.metrics.memoryUsage - 0.7) * 66)
|
||||
}
|
||||
|
||||
// Deduct points for high CPU usage
|
||||
if (this.metrics.cpuUsage > 0.8) {
|
||||
score -= Math.min(15, (this.metrics.cpuUsage - 0.8) * 75)
|
||||
}
|
||||
|
||||
// Deduct points for low throughput
|
||||
if (this.metrics.operationsPerSecond < 1 && this.metrics.totalOperations > 10) {
|
||||
score -= 10
|
||||
}
|
||||
|
||||
// Deduct points for queue depth
|
||||
if (this.metrics.queueDepth > 100) {
|
||||
score -= Math.min(15, this.metrics.queueDepth / 20)
|
||||
}
|
||||
|
||||
this.metrics.healthScore = Math.max(0, Math.min(100, score))
|
||||
}
|
||||
|
||||
/**
|
||||
* Check for alert conditions
|
||||
*/
|
||||
private checkAlerts(): void {
|
||||
const alerts: string[] = []
|
||||
|
||||
if (this.metrics.errorRate > this.thresholds.errorRate) {
|
||||
alerts.push(`High error rate: ${(this.metrics.errorRate * 100).toFixed(1)}%`)
|
||||
}
|
||||
|
||||
if (this.metrics.p95Latency > this.thresholds.latencyP95) {
|
||||
alerts.push(`High P95 latency: ${this.metrics.p95Latency}ms`)
|
||||
}
|
||||
|
||||
if (this.metrics.memoryUsage > this.thresholds.memoryUsage) {
|
||||
alerts.push(`High memory usage: ${(this.metrics.memoryUsage * 100).toFixed(1)}%`)
|
||||
}
|
||||
|
||||
if (this.metrics.cpuUsage > this.thresholds.cpuUsage) {
|
||||
alerts.push(`High CPU usage: ${(this.metrics.cpuUsage * 100).toFixed(1)}%`)
|
||||
}
|
||||
|
||||
if (this.metrics.healthScore < this.thresholds.healthScore) {
|
||||
alerts.push(`Low health score: ${this.metrics.healthScore.toFixed(0)}`)
|
||||
}
|
||||
|
||||
if (alerts.length > 0) {
|
||||
this.logger.warn('Performance alerts', { alerts, metrics: this.metrics })
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Auto-optimize system based on metrics
|
||||
*/
|
||||
private autoOptimize(): void {
|
||||
this.recommendations = []
|
||||
|
||||
// Analyze trends
|
||||
const trends = this.analyzeTrends()
|
||||
|
||||
// Generate recommendations based on metrics and trends
|
||||
if (this.metrics.errorRate > 0.02) {
|
||||
this.recommendations.push('Reduce load or increase timeouts due to high error rate')
|
||||
}
|
||||
|
||||
if (this.metrics.p95Latency > 3000) {
|
||||
this.recommendations.push('Increase batch size or socket limits to improve latency')
|
||||
}
|
||||
|
||||
if (this.metrics.memoryUsage > 0.7) {
|
||||
this.recommendations.push('Reduce cache sizes or batch sizes to free memory')
|
||||
}
|
||||
|
||||
if (this.metrics.queueDepth > 50) {
|
||||
this.recommendations.push('Increase concurrency limits to reduce queue depth')
|
||||
}
|
||||
|
||||
// Check for degrading trends
|
||||
trends.forEach(trend => {
|
||||
if (trend.direction === 'degrading' && Math.abs(trend.changeRate) > 0.1) {
|
||||
this.recommendations.push(`${trend.metric} is degrading at ${(trend.changeRate * 100).toFixed(1)}% per minute`)
|
||||
}
|
||||
})
|
||||
|
||||
// Log recommendations if any
|
||||
if (this.recommendations.length > 0) {
|
||||
this.logger.info('Performance optimization recommendations', {
|
||||
recommendations: this.recommendations,
|
||||
metrics: this.metrics
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Analyze performance trends
|
||||
*/
|
||||
private analyzeTrends(): PerformanceTrend[] {
|
||||
const trends: PerformanceTrend[] = []
|
||||
|
||||
if (this.history.length < 10) {
|
||||
return trends // Not enough data
|
||||
}
|
||||
|
||||
// Get recent history
|
||||
const recent = this.history.slice(-20)
|
||||
const older = this.history.slice(-40, -20)
|
||||
|
||||
// Compare key metrics
|
||||
const metricsToAnalyze = [
|
||||
'errorRate',
|
||||
'averageLatency',
|
||||
'operationsPerSecond',
|
||||
'memoryUsage',
|
||||
'healthScore'
|
||||
] as const
|
||||
|
||||
metricsToAnalyze.forEach(metric => {
|
||||
const recentAvg = recent.reduce((sum, m) => sum + m[metric], 0) / recent.length
|
||||
const olderAvg = older.length > 0
|
||||
? older.reduce((sum, m) => sum + m[metric], 0) / older.length
|
||||
: recentAvg
|
||||
|
||||
const changeRate = olderAvg !== 0 ? (recentAvg - olderAvg) / olderAvg : 0
|
||||
|
||||
let direction: 'improving' | 'degrading' | 'stable' = 'stable'
|
||||
if (Math.abs(changeRate) > 0.05) { // 5% threshold
|
||||
// For error rate and latency, increase is bad
|
||||
if (metric === 'errorRate' || metric === 'averageLatency' || metric === 'memoryUsage') {
|
||||
direction = changeRate > 0 ? 'degrading' : 'improving'
|
||||
} else {
|
||||
// For throughput and health score, increase is good
|
||||
direction = changeRate > 0 ? 'improving' : 'degrading'
|
||||
}
|
||||
}
|
||||
|
||||
// Simple linear prediction
|
||||
const prediction = recentAvg + (recentAvg * changeRate)
|
||||
|
||||
trends.push({
|
||||
metric,
|
||||
direction,
|
||||
changeRate,
|
||||
prediction
|
||||
})
|
||||
})
|
||||
|
||||
return trends
|
||||
}
|
||||
|
||||
/**
|
||||
* Reset counters
|
||||
*/
|
||||
private resetCounters(): void {
|
||||
this.metrics.totalOperations = 0
|
||||
this.metrics.successfulOperations = 0
|
||||
this.metrics.failedOperations = 0
|
||||
this.operationSizes = []
|
||||
this.lastReset = Date.now()
|
||||
}
|
||||
|
||||
/**
|
||||
* Get current metrics
|
||||
*/
|
||||
public getMetrics(): Readonly<PerformanceMetrics> {
|
||||
return { ...this.metrics }
|
||||
}
|
||||
|
||||
/**
|
||||
* Get performance trends
|
||||
*/
|
||||
public getTrends(): PerformanceTrend[] {
|
||||
return this.analyzeTrends()
|
||||
}
|
||||
|
||||
/**
|
||||
* Get recommendations
|
||||
*/
|
||||
public getRecommendations(): string[] {
|
||||
return [...this.recommendations]
|
||||
}
|
||||
|
||||
/**
|
||||
* Get performance report
|
||||
*/
|
||||
public getReport(): {
|
||||
metrics: PerformanceMetrics
|
||||
trends: PerformanceTrend[]
|
||||
recommendations: string[]
|
||||
socketConfig: any
|
||||
backpressureStatus: any
|
||||
} {
|
||||
return {
|
||||
metrics: this.getMetrics(),
|
||||
trends: this.getTrends(),
|
||||
recommendations: this.getRecommendations(),
|
||||
socketConfig: getGlobalSocketManager().getConfig(),
|
||||
backpressureStatus: getGlobalBackpressure().getStatus()
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Enable/disable auto-optimization
|
||||
*/
|
||||
public setAutoOptimize(enabled: boolean): void {
|
||||
this.autoOptimizeEnabled = enabled
|
||||
this.logger.info(`Auto-optimization ${enabled ? 'enabled' : 'disabled'}`)
|
||||
}
|
||||
|
||||
/**
|
||||
* Reset all metrics and history
|
||||
*/
|
||||
public reset(): void {
|
||||
this.metrics = {
|
||||
totalOperations: 0,
|
||||
successfulOperations: 0,
|
||||
failedOperations: 0,
|
||||
averageLatency: 0,
|
||||
p95Latency: 0,
|
||||
p99Latency: 0,
|
||||
operationsPerSecond: 0,
|
||||
bytesPerSecond: 0,
|
||||
memoryUsage: 0,
|
||||
cpuUsage: 0,
|
||||
socketUtilization: 0,
|
||||
queueDepth: 0,
|
||||
errorRate: 0,
|
||||
healthScore: 100
|
||||
}
|
||||
|
||||
this.history = []
|
||||
this.operationLatencies = []
|
||||
this.operationSizes = []
|
||||
this.recommendations = []
|
||||
this.lastReset = Date.now()
|
||||
|
||||
this.logger.info('Performance monitor reset')
|
||||
}
|
||||
}
|
||||
|
||||
// Global singleton instance
|
||||
let globalMonitor: PerformanceMonitor | null = null
|
||||
|
||||
/**
|
||||
* Get the global performance monitor instance
|
||||
*/
|
||||
export function getGlobalPerformanceMonitor(): PerformanceMonitor {
|
||||
if (!globalMonitor) {
|
||||
globalMonitor = new PerformanceMonitor()
|
||||
}
|
||||
return globalMonitor
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue