feat: implement always-adaptive caching with getCacheStats monitoring
Replaces lazy mode concept with always-adaptive caching strategy:
- Rename getLazyModeStats() → getCacheStats() with enhanced metrics
- Change lazyModeEnabled boolean → cachingStrategy enum ('preloaded' | 'on-demand')
- Update preloading threshold from 30% to 80% for better cache utilization
- Add comprehensive production monitoring and diagnostics
- Add memory detection for containers (Docker/K8s cgroups v1/v2)
- Add adaptive memory sizing from 2GB to 128GB+ systems
Breaking changes: None (backward compatible, deprecated lazy option ignored)
New APIs:
- getCacheStats(): Comprehensive cache performance statistics
- cachingStrategy field: Transparent strategy reporting
- Enhanced fairness metrics and memory pressure monitoring
Documentation:
- Add migration guide for v3.36.0
- Add operations/capacity-planning.md for enterprise deployments
- Update all examples and troubleshooting guides
- Rename monitor-lazy-mode.ts → monitor-cache-performance.ts
This commit is contained in:
parent
6037db3d85
commit
46c6af3f21
17 changed files with 2737 additions and 127 deletions
476
src/utils/memoryDetection.ts
Normal file
476
src/utils/memoryDetection.ts
Normal file
|
|
@ -0,0 +1,476 @@
|
|||
/**
|
||||
* Memory Detection Utilities
|
||||
* Detects available system memory across different environments:
|
||||
* - Docker/Kubernetes (cgroups v1 and v2)
|
||||
* - Bare metal servers
|
||||
* - Cloud instances
|
||||
* - Development environments
|
||||
*
|
||||
* Scales from 2GB to 128GB+ with intelligent allocation
|
||||
*/
|
||||
|
||||
import * as os from 'os'
|
||||
import * as fs from 'fs'
|
||||
import { prodLog } from './logger.js'
|
||||
|
||||
export interface MemoryInfo {
|
||||
/** Total memory available to this process (bytes) */
|
||||
available: number
|
||||
|
||||
/** Source of memory information */
|
||||
source: 'cgroup-v2' | 'cgroup-v1' | 'system' | 'fallback'
|
||||
|
||||
/** Whether running in a container */
|
||||
isContainer: boolean
|
||||
|
||||
/** System total memory (may differ from available in containers) */
|
||||
systemTotal: number
|
||||
|
||||
/** Currently free memory (best-effort estimate) */
|
||||
free: number
|
||||
|
||||
/** Detection warnings (if any) */
|
||||
warnings: string[]
|
||||
}
|
||||
|
||||
export interface CacheAllocationStrategy {
|
||||
/** Recommended cache size (bytes) */
|
||||
cacheSize: number
|
||||
|
||||
/** Allocation ratio used (0-1) */
|
||||
ratio: number
|
||||
|
||||
/** Minimum guaranteed size (bytes) */
|
||||
minSize: number
|
||||
|
||||
/** Maximum allowed size (bytes) */
|
||||
maxSize: number | null
|
||||
|
||||
/** Environment type detected */
|
||||
environment: 'production' | 'development' | 'container' | 'unknown'
|
||||
|
||||
/** Model memory reserved (bytes) - v3.36.0+ */
|
||||
modelMemory: number
|
||||
|
||||
/** Model precision (q8 or fp32) */
|
||||
modelPrecision: 'q8' | 'fp32'
|
||||
|
||||
/** Available memory after model reservation (bytes) */
|
||||
availableForCache: number
|
||||
|
||||
/** Reasoning for allocation */
|
||||
reasoning: string
|
||||
}
|
||||
|
||||
/**
|
||||
* Detect available memory across all environments
|
||||
*/
|
||||
export function detectAvailableMemory(): MemoryInfo {
|
||||
const warnings: string[] = []
|
||||
|
||||
// Try cgroups v2 first (modern Docker/K8s)
|
||||
const cgroupV2 = detectCgroupV2Memory()
|
||||
if (cgroupV2 !== null) {
|
||||
const systemTotal = os.totalmem()
|
||||
const free = os.freemem()
|
||||
|
||||
return {
|
||||
available: cgroupV2,
|
||||
source: 'cgroup-v2',
|
||||
isContainer: true,
|
||||
systemTotal,
|
||||
free,
|
||||
warnings: cgroupV2 < systemTotal
|
||||
? [`Container limited to ${formatBytes(cgroupV2)} (host has ${formatBytes(systemTotal)})`]
|
||||
: []
|
||||
}
|
||||
}
|
||||
|
||||
// Try cgroups v1 (older Docker/K8s)
|
||||
const cgroupV1 = detectCgroupV1Memory()
|
||||
if (cgroupV1 !== null) {
|
||||
const systemTotal = os.totalmem()
|
||||
const free = os.freemem()
|
||||
|
||||
return {
|
||||
available: cgroupV1,
|
||||
source: 'cgroup-v1',
|
||||
isContainer: true,
|
||||
systemTotal,
|
||||
free,
|
||||
warnings: cgroupV1 < systemTotal
|
||||
? [`Container limited to ${formatBytes(cgroupV1)} (host has ${formatBytes(systemTotal)})`]
|
||||
: []
|
||||
}
|
||||
}
|
||||
|
||||
// Use system memory (bare metal, VM, or unlimited container)
|
||||
const systemTotal = os.totalmem()
|
||||
const free = os.freemem()
|
||||
|
||||
// Check if we might be in an unlimited container
|
||||
if (process.env.KUBERNETES_SERVICE_HOST || process.env.DOCKER_CONTAINER) {
|
||||
warnings.push('Container detected but no memory limit set - using host memory')
|
||||
}
|
||||
|
||||
return {
|
||||
available: systemTotal,
|
||||
source: 'system',
|
||||
isContainer: false,
|
||||
systemTotal,
|
||||
free,
|
||||
warnings
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Detect memory limit from cgroups v2 (modern containers)
|
||||
* Path: /sys/fs/cgroup/memory.max
|
||||
*/
|
||||
function detectCgroupV2Memory(): number | null {
|
||||
try {
|
||||
const memoryMaxPath = '/sys/fs/cgroup/memory.max'
|
||||
|
||||
if (!fs.existsSync(memoryMaxPath)) {
|
||||
return null
|
||||
}
|
||||
|
||||
const content = fs.readFileSync(memoryMaxPath, 'utf8').trim()
|
||||
|
||||
// 'max' means unlimited
|
||||
if (content === 'max') {
|
||||
return null
|
||||
}
|
||||
|
||||
const bytes = parseInt(content, 10)
|
||||
|
||||
// Sanity check: Must be reasonable number (between 64MB and 1TB)
|
||||
if (bytes < 64 * 1024 * 1024 || bytes > 1024 * 1024 * 1024 * 1024) {
|
||||
prodLog.warn(`Suspicious cgroup v2 memory limit: ${formatBytes(bytes)}`)
|
||||
return null
|
||||
}
|
||||
|
||||
return bytes
|
||||
} catch (error) {
|
||||
// Not in a cgroup v2 environment
|
||||
return null
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Detect memory limit from cgroups v1 (older containers)
|
||||
* Path: /sys/fs/cgroup/memory/memory.limit_in_bytes
|
||||
*/
|
||||
function detectCgroupV1Memory(): number | null {
|
||||
try {
|
||||
const limitPath = '/sys/fs/cgroup/memory/memory.limit_in_bytes'
|
||||
|
||||
if (!fs.existsSync(limitPath)) {
|
||||
return null
|
||||
}
|
||||
|
||||
const content = fs.readFileSync(limitPath, 'utf8').trim()
|
||||
const bytes = parseInt(content, 10)
|
||||
|
||||
// cgroup v1 uses very large number (2^63-1) to indicate unlimited
|
||||
// If limit is > 1TB, consider it unlimited
|
||||
if (bytes > 1024 * 1024 * 1024 * 1024) {
|
||||
return null
|
||||
}
|
||||
|
||||
// Sanity check: Must be reasonable number (between 64MB and 1TB)
|
||||
if (bytes < 64 * 1024 * 1024) {
|
||||
prodLog.warn(`Suspicious cgroup v1 memory limit: ${formatBytes(bytes)}`)
|
||||
return null
|
||||
}
|
||||
|
||||
return bytes
|
||||
} catch (error) {
|
||||
// Not in a cgroup v1 environment
|
||||
return null
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Calculate optimal cache size based on available memory
|
||||
* Scales intelligently from 2GB to 128GB+
|
||||
*
|
||||
* v3.36.0+: Accounts for embedding model memory (150MB Q8, 250MB FP32)
|
||||
*/
|
||||
export function calculateOptimalCacheSize(
|
||||
memoryInfo: MemoryInfo,
|
||||
options: {
|
||||
/** Manual override (bytes) - takes precedence */
|
||||
manualSize?: number
|
||||
|
||||
/** Minimum cache size (bytes) - default 256MB */
|
||||
minSize?: number
|
||||
|
||||
/** Maximum cache size (bytes) - default unlimited */
|
||||
maxSize?: number
|
||||
|
||||
/** Force development mode allocation (more conservative) */
|
||||
developmentMode?: boolean
|
||||
|
||||
/** Model precision for memory calculation - default 'q8' */
|
||||
modelPrecision?: 'q8' | 'fp32'
|
||||
} = {}
|
||||
): CacheAllocationStrategy {
|
||||
const minSize = options.minSize || 256 * 1024 * 1024 // 256MB minimum
|
||||
const maxSize = options.maxSize || null
|
||||
|
||||
// Detect model memory usage (v3.36.0+)
|
||||
const modelInfo = detectModelMemory({ precision: options.modelPrecision || 'q8' })
|
||||
const modelMemory = modelInfo.bytes
|
||||
|
||||
// Reserve model memory from available RAM BEFORE calculating cache
|
||||
// This ensures we don't over-allocate and cause OOM
|
||||
const availableForCache = Math.max(0, memoryInfo.available - modelMemory)
|
||||
|
||||
// Manual override takes precedence
|
||||
if (options.manualSize !== undefined) {
|
||||
const clamped = Math.max(minSize, options.manualSize)
|
||||
return {
|
||||
cacheSize: clamped,
|
||||
ratio: clamped / availableForCache,
|
||||
minSize,
|
||||
maxSize,
|
||||
environment: 'unknown',
|
||||
modelMemory,
|
||||
modelPrecision: modelInfo.precision,
|
||||
availableForCache,
|
||||
reasoning: 'Manual override specified'
|
||||
}
|
||||
}
|
||||
|
||||
// Determine environment and allocation ratio
|
||||
let ratio: number
|
||||
let environment: CacheAllocationStrategy['environment']
|
||||
let reasoning: string
|
||||
|
||||
if (options.developmentMode || process.env.NODE_ENV === 'development') {
|
||||
// Development: More conservative (25%)
|
||||
ratio = 0.25
|
||||
environment = 'development'
|
||||
reasoning = `Development mode - conservative allocation (25% of ${formatBytes(availableForCache)} after ${formatBytes(modelMemory)} model)`
|
||||
} else if (memoryInfo.isContainer) {
|
||||
// Container: Moderate allocation (40%)
|
||||
// Containers often have tight limits, leave room for heap growth
|
||||
ratio = 0.40
|
||||
environment = 'container'
|
||||
reasoning = `Container environment - moderate allocation (40% of ${formatBytes(availableForCache)} after ${formatBytes(modelMemory)} model)`
|
||||
} else {
|
||||
// Production bare metal/VM: Aggressive allocation (50%)
|
||||
// More memory available, can be more aggressive
|
||||
ratio = 0.50
|
||||
environment = 'production'
|
||||
reasoning = `Production environment - aggressive allocation (50% of ${formatBytes(availableForCache)} after ${formatBytes(modelMemory)} model)`
|
||||
}
|
||||
|
||||
// Calculate base cache size from AVAILABLE memory (after model reservation)
|
||||
let cacheSize = Math.floor(availableForCache * ratio)
|
||||
|
||||
// Apply minimum constraint
|
||||
if (cacheSize < minSize) {
|
||||
const originalSize = cacheSize
|
||||
cacheSize = minSize
|
||||
reasoning += ` (increased from ${formatBytes(originalSize)} to meet minimum)`
|
||||
|
||||
// Warn if available memory is very low
|
||||
if (availableForCache < minSize * 2) {
|
||||
prodLog.warn(
|
||||
`⚠️ Low available memory for cache (${formatBytes(availableForCache)} after ${formatBytes(modelMemory)} model). ` +
|
||||
`Cache size ${formatBytes(cacheSize)} may cause memory pressure.`
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
// Apply maximum constraint
|
||||
if (maxSize !== null && cacheSize > maxSize) {
|
||||
const originalSize = cacheSize
|
||||
cacheSize = maxSize
|
||||
reasoning += ` (capped from ${formatBytes(originalSize)} to maximum)`
|
||||
}
|
||||
|
||||
// Intelligent scaling for large memory systems
|
||||
// For systems with >64GB available for cache, use logarithmic scaling to avoid over-allocation
|
||||
if (availableForCache > 64 * 1024 * 1024 * 1024) {
|
||||
// Above 64GB, scale more conservatively
|
||||
// Formula: base + log2(availableForCache/64GB) * 8GB
|
||||
const base = 32 * 1024 * 1024 * 1024 // 32GB base
|
||||
const scaleFactor = Math.log2(availableForCache / (64 * 1024 * 1024 * 1024))
|
||||
const scaled = base + scaleFactor * 8 * 1024 * 1024 * 1024 // +8GB per doubling
|
||||
|
||||
if (scaled < cacheSize) {
|
||||
const originalSize = cacheSize
|
||||
cacheSize = Math.floor(scaled)
|
||||
reasoning += ` (scaled down from ${formatBytes(originalSize)} for large memory system)`
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
cacheSize,
|
||||
ratio,
|
||||
minSize,
|
||||
maxSize,
|
||||
environment,
|
||||
modelMemory,
|
||||
modelPrecision: modelInfo.precision,
|
||||
availableForCache,
|
||||
reasoning
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get recommended cache configuration for current environment
|
||||
*/
|
||||
export function getRecommendedCacheConfig(options: {
|
||||
/** Manual cache size override (bytes) */
|
||||
manualSize?: number
|
||||
|
||||
/** Minimum cache size (bytes) */
|
||||
minSize?: number
|
||||
|
||||
/** Maximum cache size (bytes) */
|
||||
maxSize?: number
|
||||
|
||||
/** Force development mode */
|
||||
developmentMode?: boolean
|
||||
} = {}): {
|
||||
memoryInfo: MemoryInfo
|
||||
allocation: CacheAllocationStrategy
|
||||
warnings: string[]
|
||||
} {
|
||||
const memoryInfo = detectAvailableMemory()
|
||||
const allocation = calculateOptimalCacheSize(memoryInfo, options)
|
||||
|
||||
const warnings: string[] = [...memoryInfo.warnings]
|
||||
|
||||
// Add allocation warnings
|
||||
if (allocation.cacheSize === allocation.minSize) {
|
||||
warnings.push(
|
||||
`Cache size at minimum (${formatBytes(allocation.minSize)}). ` +
|
||||
`Consider increasing available memory for better performance.`
|
||||
)
|
||||
}
|
||||
|
||||
if (allocation.ratio > 0.6) {
|
||||
warnings.push(
|
||||
`Cache using ${(allocation.ratio * 100).toFixed(0)}% of available memory. ` +
|
||||
`Monitor for memory pressure.`
|
||||
)
|
||||
}
|
||||
|
||||
return {
|
||||
memoryInfo,
|
||||
allocation,
|
||||
warnings
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Detect embedding model memory usage
|
||||
*
|
||||
* Returns estimated runtime memory for the embedding model:
|
||||
* - Q8 (quantized, default): ~150MB runtime (22MB on disk)
|
||||
* - FP32 (full precision): ~250MB runtime (86MB on disk)
|
||||
*
|
||||
* Breakdown for Q8:
|
||||
* - Model weights: 22MB
|
||||
* - ONNX Runtime: 15-30MB
|
||||
* - Session workspace: 50-100MB (peak during inference)
|
||||
* - Total: ~100-150MB (we use 150MB conservative)
|
||||
*/
|
||||
export function detectModelMemory(options: {
|
||||
/** Model precision (default: 'q8') */
|
||||
precision?: 'q8' | 'fp32'
|
||||
} = {}): {
|
||||
bytes: number
|
||||
precision: 'q8' | 'fp32'
|
||||
breakdown: {
|
||||
modelWeights: number
|
||||
onnxRuntime: number
|
||||
sessionWorkspace: number
|
||||
}
|
||||
} {
|
||||
const precision = options.precision || 'q8'
|
||||
|
||||
if (precision === 'q8') {
|
||||
// Q8 quantized model (default)
|
||||
return {
|
||||
bytes: 150 * 1024 * 1024, // 150MB
|
||||
precision: 'q8',
|
||||
breakdown: {
|
||||
modelWeights: 22 * 1024 * 1024, // 22MB
|
||||
onnxRuntime: 30 * 1024 * 1024, // 30MB (conservative)
|
||||
sessionWorkspace: 98 * 1024 * 1024 // 98MB (peak during inference)
|
||||
}
|
||||
}
|
||||
} else {
|
||||
// FP32 full precision model
|
||||
return {
|
||||
bytes: 250 * 1024 * 1024, // 250MB
|
||||
precision: 'fp32',
|
||||
breakdown: {
|
||||
modelWeights: 86 * 1024 * 1024, // 86MB
|
||||
onnxRuntime: 30 * 1024 * 1024, // 30MB
|
||||
sessionWorkspace: 134 * 1024 * 1024 // 134MB (peak during inference)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Format bytes to human-readable string
|
||||
*/
|
||||
export function formatBytes(bytes: number): string {
|
||||
if (bytes === 0) return '0 B'
|
||||
|
||||
const k = 1024
|
||||
const sizes = ['B', 'KB', 'MB', 'GB', 'TB']
|
||||
const i = Math.floor(Math.log(bytes) / Math.log(k))
|
||||
|
||||
return `${(bytes / Math.pow(k, i)).toFixed(2)} ${sizes[i]}`
|
||||
}
|
||||
|
||||
/**
|
||||
* Monitor memory usage and warn if approaching limits
|
||||
*/
|
||||
export function checkMemoryPressure(
|
||||
cacheSize: number,
|
||||
memoryInfo: MemoryInfo
|
||||
): {
|
||||
pressure: 'none' | 'moderate' | 'high' | 'critical'
|
||||
warnings: string[]
|
||||
} {
|
||||
const warnings: string[] = []
|
||||
const heapUsed = process.memoryUsage().heapUsed
|
||||
const totalUsed = heapUsed + cacheSize
|
||||
const utilization = totalUsed / memoryInfo.available
|
||||
|
||||
if (utilization > 0.95) {
|
||||
warnings.push(
|
||||
`🔴 CRITICAL: Memory utilization at ${(utilization * 100).toFixed(1)}%. ` +
|
||||
`Reduce cache size or increase available memory.`
|
||||
)
|
||||
return { pressure: 'critical', warnings }
|
||||
}
|
||||
|
||||
if (utilization > 0.85) {
|
||||
warnings.push(
|
||||
`🟠 HIGH: Memory utilization at ${(utilization * 100).toFixed(1)}%. ` +
|
||||
`Consider increasing available memory.`
|
||||
)
|
||||
return { pressure: 'high', warnings }
|
||||
}
|
||||
|
||||
if (utilization > 0.70) {
|
||||
warnings.push(
|
||||
`🟡 MODERATE: Memory utilization at ${(utilization * 100).toFixed(1)}%. ` +
|
||||
`Monitor for memory pressure.`
|
||||
)
|
||||
return { pressure: 'moderate', warnings }
|
||||
}
|
||||
|
||||
return { pressure: 'none', warnings: [] }
|
||||
}
|
||||
|
|
@ -1,9 +1,22 @@
|
|||
/**
|
||||
* UnifiedCache - Single cache for both HNSW and MetadataIndex
|
||||
* Prevents resource competition with cost-aware eviction
|
||||
*
|
||||
* Features (v3.36.0+):
|
||||
* - Adaptive sizing: Automatically scales from 2GB to 128GB+ based on available memory
|
||||
* - Container-aware: Detects Docker/K8s limits (cgroups v1/v2)
|
||||
* - Environment detection: Production vs development allocation strategies
|
||||
* - Memory pressure monitoring: Warns when approaching limits
|
||||
*/
|
||||
|
||||
import { prodLog } from './logger.js'
|
||||
import {
|
||||
getRecommendedCacheConfig,
|
||||
formatBytes,
|
||||
checkMemoryPressure,
|
||||
type MemoryInfo,
|
||||
type CacheAllocationStrategy
|
||||
} from './memoryDetection.js'
|
||||
|
||||
export interface CacheItem {
|
||||
key: string
|
||||
|
|
@ -16,11 +29,32 @@ export interface CacheItem {
|
|||
}
|
||||
|
||||
export interface UnifiedCacheConfig {
|
||||
maxSize?: number // bytes
|
||||
/** Maximum cache size in bytes (auto-detected if not specified) */
|
||||
maxSize?: number
|
||||
|
||||
/** Minimum cache size in bytes (default 256MB) */
|
||||
minSize?: number
|
||||
|
||||
/** Force development mode allocation (25% instead of 40-50%) */
|
||||
developmentMode?: boolean
|
||||
|
||||
/** Enable request coalescing to prevent duplicate loads */
|
||||
enableRequestCoalescing?: boolean
|
||||
|
||||
/** Enable fairness monitoring to prevent cache starvation */
|
||||
enableFairnessCheck?: boolean
|
||||
fairnessCheckInterval?: number // ms
|
||||
|
||||
/** Fairness check interval in milliseconds */
|
||||
fairnessCheckInterval?: number
|
||||
|
||||
/** Enable access pattern persistence for warm starts */
|
||||
persistPatterns?: boolean
|
||||
|
||||
/** Enable memory pressure monitoring (default true) */
|
||||
enableMemoryMonitoring?: boolean
|
||||
|
||||
/** Memory pressure check interval in milliseconds (default 30s) */
|
||||
memoryCheckInterval?: number
|
||||
}
|
||||
|
||||
export class UnifiedCache {
|
||||
|
|
@ -33,19 +67,67 @@ export class UnifiedCache {
|
|||
private readonly maxSize: number
|
||||
private readonly config: UnifiedCacheConfig
|
||||
|
||||
// Memory management (v3.36.0+)
|
||||
private readonly memoryInfo: MemoryInfo
|
||||
private readonly allocationStrategy: CacheAllocationStrategy
|
||||
private memoryPressureCheckTimer: NodeJS.Timeout | null = null
|
||||
private lastMemoryWarning = 0
|
||||
|
||||
constructor(config: UnifiedCacheConfig = {}) {
|
||||
this.maxSize = config.maxSize || 2 * 1024 * 1024 * 1024 // 2GB default
|
||||
// Adaptive cache sizing (v3.36.0+)
|
||||
const recommendation = getRecommendedCacheConfig({
|
||||
manualSize: config.maxSize,
|
||||
minSize: config.minSize,
|
||||
developmentMode: config.developmentMode
|
||||
})
|
||||
|
||||
this.memoryInfo = recommendation.memoryInfo
|
||||
this.allocationStrategy = recommendation.allocation
|
||||
this.maxSize = recommendation.allocation.cacheSize
|
||||
|
||||
// Log allocation decision (v3.36.0+: includes model memory)
|
||||
prodLog.info(
|
||||
`UnifiedCache initialized: ${formatBytes(this.maxSize)} ` +
|
||||
`(${this.allocationStrategy.environment} mode, ` +
|
||||
`${(this.allocationStrategy.ratio * 100).toFixed(0)}% of ${formatBytes(this.allocationStrategy.availableForCache)} ` +
|
||||
`after ${formatBytes(this.allocationStrategy.modelMemory)} ${this.allocationStrategy.modelPrecision.toUpperCase()} model)`
|
||||
)
|
||||
|
||||
// Log memory detection details
|
||||
prodLog.debug(
|
||||
`Memory detection: source=${this.memoryInfo.source}, ` +
|
||||
`container=${this.memoryInfo.isContainer}, ` +
|
||||
`system=${formatBytes(this.memoryInfo.systemTotal)}, ` +
|
||||
`free=${formatBytes(this.memoryInfo.free)}, ` +
|
||||
`totalAvailable=${formatBytes(this.memoryInfo.available)}, ` +
|
||||
`modelReserved=${formatBytes(this.allocationStrategy.modelMemory)}, ` +
|
||||
`availableForCache=${formatBytes(this.allocationStrategy.availableForCache)}`
|
||||
)
|
||||
|
||||
// Log warnings if any
|
||||
for (const warning of recommendation.warnings) {
|
||||
prodLog.warn(`UnifiedCache: ${warning}`)
|
||||
}
|
||||
|
||||
// Finalize configuration
|
||||
this.config = {
|
||||
enableRequestCoalescing: true,
|
||||
enableFairnessCheck: true,
|
||||
fairnessCheckInterval: 60000, // Check fairness every minute
|
||||
persistPatterns: true,
|
||||
enableMemoryMonitoring: true,
|
||||
memoryCheckInterval: 30000, // Check memory every 30s
|
||||
...config
|
||||
}
|
||||
|
||||
// Start monitoring
|
||||
if (this.config.enableFairnessCheck) {
|
||||
this.startFairnessMonitor()
|
||||
}
|
||||
|
||||
if (this.config.enableMemoryMonitoring) {
|
||||
this.startMemoryPressureMonitor()
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -92,6 +174,27 @@ export class UnifiedCache {
|
|||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Synchronous cache lookup (v3.36.0+)
|
||||
* Returns cached data immediately or undefined if not cached
|
||||
* Use for sync fast path optimization - zero async overhead
|
||||
*/
|
||||
getSync(key: string): any | undefined {
|
||||
// Check if in cache
|
||||
const item = this.cache.get(key)
|
||||
if (item) {
|
||||
// Update access tracking synchronously
|
||||
this.access.set(key, (this.access.get(key) || 0) + 1)
|
||||
this.totalAccessCount++
|
||||
item.lastAccess = Date.now()
|
||||
item.accessCount++
|
||||
this.typeAccessCounts[item.type]++
|
||||
return item.data
|
||||
}
|
||||
|
||||
return undefined
|
||||
}
|
||||
|
||||
/**
|
||||
* Set item in cache with cost-aware eviction
|
||||
*/
|
||||
|
|
@ -300,7 +403,55 @@ export class UnifiedCache {
|
|||
}
|
||||
|
||||
/**
|
||||
* Get cache statistics
|
||||
* Start memory pressure monitoring
|
||||
* Periodically checks if we're approaching memory limits
|
||||
*/
|
||||
private startMemoryPressureMonitor(): void {
|
||||
const checkInterval = this.config.memoryCheckInterval || 30000
|
||||
|
||||
this.memoryPressureCheckTimer = setInterval(() => {
|
||||
this.checkMemoryPressure()
|
||||
}, checkInterval)
|
||||
|
||||
// Unref so it doesn't keep process alive
|
||||
if (this.memoryPressureCheckTimer.unref) {
|
||||
this.memoryPressureCheckTimer.unref()
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Check current memory pressure and warn if needed
|
||||
*/
|
||||
private checkMemoryPressure(): void {
|
||||
const pressure = checkMemoryPressure(this.currentSize, this.memoryInfo)
|
||||
|
||||
// Only log warnings every 5 minutes to avoid spam
|
||||
const now = Date.now()
|
||||
const fiveMinutes = 5 * 60 * 1000
|
||||
|
||||
if (pressure.warnings.length > 0 && now - this.lastMemoryWarning > fiveMinutes) {
|
||||
for (const warning of pressure.warnings) {
|
||||
prodLog.warn(`UnifiedCache: ${warning}`)
|
||||
}
|
||||
this.lastMemoryWarning = now
|
||||
}
|
||||
|
||||
// If critical, force aggressive eviction
|
||||
if (pressure.pressure === 'critical') {
|
||||
const targetSize = Math.floor(this.maxSize * 0.7) // Evict to 70%
|
||||
const bytesToFree = this.currentSize - targetSize
|
||||
|
||||
if (bytesToFree > 0) {
|
||||
prodLog.warn(
|
||||
`UnifiedCache: Critical memory pressure - forcing eviction of ${formatBytes(bytesToFree)}`
|
||||
)
|
||||
this.evictForSize(bytesToFree)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get cache statistics with memory information
|
||||
*/
|
||||
getStats() {
|
||||
const typeSizes = { hnsw: 0, metadata: 0, embedding: 0, other: 0 }
|
||||
|
|
@ -311,7 +462,11 @@ export class UnifiedCache {
|
|||
typeCounts[item.type]++
|
||||
}
|
||||
|
||||
const hitRate = this.cache.size > 0 ?
|
||||
Array.from(this.cache.values()).reduce((sum, item) => sum + item.accessCount, 0) / this.totalAccessCount : 0
|
||||
|
||||
return {
|
||||
// Cache statistics
|
||||
totalSize: this.currentSize,
|
||||
maxSize: this.maxSize,
|
||||
utilization: this.currentSize / this.maxSize,
|
||||
|
|
@ -320,8 +475,28 @@ export class UnifiedCache {
|
|||
typeCounts,
|
||||
typeAccessCounts: this.typeAccessCounts,
|
||||
totalAccessCount: this.totalAccessCount,
|
||||
hitRate: this.cache.size > 0 ?
|
||||
Array.from(this.cache.values()).reduce((sum, item) => sum + item.accessCount, 0) / this.totalAccessCount : 0
|
||||
hitRate,
|
||||
|
||||
// Memory management (v3.36.0+)
|
||||
memory: {
|
||||
available: this.memoryInfo.available,
|
||||
source: this.memoryInfo.source,
|
||||
isContainer: this.memoryInfo.isContainer,
|
||||
systemTotal: this.memoryInfo.systemTotal,
|
||||
allocationRatio: this.allocationStrategy.ratio,
|
||||
environment: this.allocationStrategy.environment
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get detailed memory information
|
||||
*/
|
||||
getMemoryInfo() {
|
||||
return {
|
||||
memoryInfo: { ...this.memoryInfo },
|
||||
allocationStrategy: { ...this.allocationStrategy },
|
||||
currentPressure: checkMemoryPressure(this.currentSize, this.memoryInfo)
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue