feat: implement always-adaptive caching with getCacheStats monitoring

Replaces lazy mode concept with always-adaptive caching strategy:

- Rename getLazyModeStats() → getCacheStats() with enhanced metrics
- Change lazyModeEnabled boolean → cachingStrategy enum ('preloaded' | 'on-demand')
- Update preloading threshold from 30% to 80% for better cache utilization
- Add comprehensive production monitoring and diagnostics
- Add memory detection for containers (Docker/K8s cgroups v1/v2)
- Add adaptive memory sizing from 2GB to 128GB+ systems

Breaking changes: None (backward compatible, deprecated lazy option ignored)

New APIs:
- getCacheStats(): Comprehensive cache performance statistics
- cachingStrategy field: Transparent strategy reporting
- Enhanced fairness metrics and memory pressure monitoring

Documentation:
- Add migration guide for v3.36.0
- Add operations/capacity-planning.md for enterprise deployments
- Update all examples and troubleshooting guides
- Rename monitor-lazy-mode.ts → monitor-cache-performance.ts
This commit is contained in:
David Snelling 2025-10-10 14:09:30 -07:00
parent 6037db3d85
commit 46c6af3f21
17 changed files with 2737 additions and 127 deletions

View file

@ -0,0 +1,476 @@
/**
* Memory Detection Utilities
* Detects available system memory across different environments:
* - Docker/Kubernetes (cgroups v1 and v2)
* - Bare metal servers
* - Cloud instances
* - Development environments
*
* Scales from 2GB to 128GB+ with intelligent allocation
*/
import * as os from 'os'
import * as fs from 'fs'
import { prodLog } from './logger.js'
export interface MemoryInfo {
/** Total memory available to this process (bytes) */
available: number
/** Source of memory information */
source: 'cgroup-v2' | 'cgroup-v1' | 'system' | 'fallback'
/** Whether running in a container */
isContainer: boolean
/** System total memory (may differ from available in containers) */
systemTotal: number
/** Currently free memory (best-effort estimate) */
free: number
/** Detection warnings (if any) */
warnings: string[]
}
export interface CacheAllocationStrategy {
/** Recommended cache size (bytes) */
cacheSize: number
/** Allocation ratio used (0-1) */
ratio: number
/** Minimum guaranteed size (bytes) */
minSize: number
/** Maximum allowed size (bytes) */
maxSize: number | null
/** Environment type detected */
environment: 'production' | 'development' | 'container' | 'unknown'
/** Model memory reserved (bytes) - v3.36.0+ */
modelMemory: number
/** Model precision (q8 or fp32) */
modelPrecision: 'q8' | 'fp32'
/** Available memory after model reservation (bytes) */
availableForCache: number
/** Reasoning for allocation */
reasoning: string
}
/**
* Detect available memory across all environments
*/
export function detectAvailableMemory(): MemoryInfo {
const warnings: string[] = []
// Try cgroups v2 first (modern Docker/K8s)
const cgroupV2 = detectCgroupV2Memory()
if (cgroupV2 !== null) {
const systemTotal = os.totalmem()
const free = os.freemem()
return {
available: cgroupV2,
source: 'cgroup-v2',
isContainer: true,
systemTotal,
free,
warnings: cgroupV2 < systemTotal
? [`Container limited to ${formatBytes(cgroupV2)} (host has ${formatBytes(systemTotal)})`]
: []
}
}
// Try cgroups v1 (older Docker/K8s)
const cgroupV1 = detectCgroupV1Memory()
if (cgroupV1 !== null) {
const systemTotal = os.totalmem()
const free = os.freemem()
return {
available: cgroupV1,
source: 'cgroup-v1',
isContainer: true,
systemTotal,
free,
warnings: cgroupV1 < systemTotal
? [`Container limited to ${formatBytes(cgroupV1)} (host has ${formatBytes(systemTotal)})`]
: []
}
}
// Use system memory (bare metal, VM, or unlimited container)
const systemTotal = os.totalmem()
const free = os.freemem()
// Check if we might be in an unlimited container
if (process.env.KUBERNETES_SERVICE_HOST || process.env.DOCKER_CONTAINER) {
warnings.push('Container detected but no memory limit set - using host memory')
}
return {
available: systemTotal,
source: 'system',
isContainer: false,
systemTotal,
free,
warnings
}
}
/**
* Detect memory limit from cgroups v2 (modern containers)
* Path: /sys/fs/cgroup/memory.max
*/
function detectCgroupV2Memory(): number | null {
try {
const memoryMaxPath = '/sys/fs/cgroup/memory.max'
if (!fs.existsSync(memoryMaxPath)) {
return null
}
const content = fs.readFileSync(memoryMaxPath, 'utf8').trim()
// 'max' means unlimited
if (content === 'max') {
return null
}
const bytes = parseInt(content, 10)
// Sanity check: Must be reasonable number (between 64MB and 1TB)
if (bytes < 64 * 1024 * 1024 || bytes > 1024 * 1024 * 1024 * 1024) {
prodLog.warn(`Suspicious cgroup v2 memory limit: ${formatBytes(bytes)}`)
return null
}
return bytes
} catch (error) {
// Not in a cgroup v2 environment
return null
}
}
/**
* Detect memory limit from cgroups v1 (older containers)
* Path: /sys/fs/cgroup/memory/memory.limit_in_bytes
*/
function detectCgroupV1Memory(): number | null {
try {
const limitPath = '/sys/fs/cgroup/memory/memory.limit_in_bytes'
if (!fs.existsSync(limitPath)) {
return null
}
const content = fs.readFileSync(limitPath, 'utf8').trim()
const bytes = parseInt(content, 10)
// cgroup v1 uses very large number (2^63-1) to indicate unlimited
// If limit is > 1TB, consider it unlimited
if (bytes > 1024 * 1024 * 1024 * 1024) {
return null
}
// Sanity check: Must be reasonable number (between 64MB and 1TB)
if (bytes < 64 * 1024 * 1024) {
prodLog.warn(`Suspicious cgroup v1 memory limit: ${formatBytes(bytes)}`)
return null
}
return bytes
} catch (error) {
// Not in a cgroup v1 environment
return null
}
}
/**
* Calculate optimal cache size based on available memory
* Scales intelligently from 2GB to 128GB+
*
* v3.36.0+: Accounts for embedding model memory (150MB Q8, 250MB FP32)
*/
export function calculateOptimalCacheSize(
memoryInfo: MemoryInfo,
options: {
/** Manual override (bytes) - takes precedence */
manualSize?: number
/** Minimum cache size (bytes) - default 256MB */
minSize?: number
/** Maximum cache size (bytes) - default unlimited */
maxSize?: number
/** Force development mode allocation (more conservative) */
developmentMode?: boolean
/** Model precision for memory calculation - default 'q8' */
modelPrecision?: 'q8' | 'fp32'
} = {}
): CacheAllocationStrategy {
const minSize = options.minSize || 256 * 1024 * 1024 // 256MB minimum
const maxSize = options.maxSize || null
// Detect model memory usage (v3.36.0+)
const modelInfo = detectModelMemory({ precision: options.modelPrecision || 'q8' })
const modelMemory = modelInfo.bytes
// Reserve model memory from available RAM BEFORE calculating cache
// This ensures we don't over-allocate and cause OOM
const availableForCache = Math.max(0, memoryInfo.available - modelMemory)
// Manual override takes precedence
if (options.manualSize !== undefined) {
const clamped = Math.max(minSize, options.manualSize)
return {
cacheSize: clamped,
ratio: clamped / availableForCache,
minSize,
maxSize,
environment: 'unknown',
modelMemory,
modelPrecision: modelInfo.precision,
availableForCache,
reasoning: 'Manual override specified'
}
}
// Determine environment and allocation ratio
let ratio: number
let environment: CacheAllocationStrategy['environment']
let reasoning: string
if (options.developmentMode || process.env.NODE_ENV === 'development') {
// Development: More conservative (25%)
ratio = 0.25
environment = 'development'
reasoning = `Development mode - conservative allocation (25% of ${formatBytes(availableForCache)} after ${formatBytes(modelMemory)} model)`
} else if (memoryInfo.isContainer) {
// Container: Moderate allocation (40%)
// Containers often have tight limits, leave room for heap growth
ratio = 0.40
environment = 'container'
reasoning = `Container environment - moderate allocation (40% of ${formatBytes(availableForCache)} after ${formatBytes(modelMemory)} model)`
} else {
// Production bare metal/VM: Aggressive allocation (50%)
// More memory available, can be more aggressive
ratio = 0.50
environment = 'production'
reasoning = `Production environment - aggressive allocation (50% of ${formatBytes(availableForCache)} after ${formatBytes(modelMemory)} model)`
}
// Calculate base cache size from AVAILABLE memory (after model reservation)
let cacheSize = Math.floor(availableForCache * ratio)
// Apply minimum constraint
if (cacheSize < minSize) {
const originalSize = cacheSize
cacheSize = minSize
reasoning += ` (increased from ${formatBytes(originalSize)} to meet minimum)`
// Warn if available memory is very low
if (availableForCache < minSize * 2) {
prodLog.warn(
`⚠️ Low available memory for cache (${formatBytes(availableForCache)} after ${formatBytes(modelMemory)} model). ` +
`Cache size ${formatBytes(cacheSize)} may cause memory pressure.`
)
}
}
// Apply maximum constraint
if (maxSize !== null && cacheSize > maxSize) {
const originalSize = cacheSize
cacheSize = maxSize
reasoning += ` (capped from ${formatBytes(originalSize)} to maximum)`
}
// Intelligent scaling for large memory systems
// For systems with >64GB available for cache, use logarithmic scaling to avoid over-allocation
if (availableForCache > 64 * 1024 * 1024 * 1024) {
// Above 64GB, scale more conservatively
// Formula: base + log2(availableForCache/64GB) * 8GB
const base = 32 * 1024 * 1024 * 1024 // 32GB base
const scaleFactor = Math.log2(availableForCache / (64 * 1024 * 1024 * 1024))
const scaled = base + scaleFactor * 8 * 1024 * 1024 * 1024 // +8GB per doubling
if (scaled < cacheSize) {
const originalSize = cacheSize
cacheSize = Math.floor(scaled)
reasoning += ` (scaled down from ${formatBytes(originalSize)} for large memory system)`
}
}
return {
cacheSize,
ratio,
minSize,
maxSize,
environment,
modelMemory,
modelPrecision: modelInfo.precision,
availableForCache,
reasoning
}
}
/**
* Get recommended cache configuration for current environment
*/
export function getRecommendedCacheConfig(options: {
/** Manual cache size override (bytes) */
manualSize?: number
/** Minimum cache size (bytes) */
minSize?: number
/** Maximum cache size (bytes) */
maxSize?: number
/** Force development mode */
developmentMode?: boolean
} = {}): {
memoryInfo: MemoryInfo
allocation: CacheAllocationStrategy
warnings: string[]
} {
const memoryInfo = detectAvailableMemory()
const allocation = calculateOptimalCacheSize(memoryInfo, options)
const warnings: string[] = [...memoryInfo.warnings]
// Add allocation warnings
if (allocation.cacheSize === allocation.minSize) {
warnings.push(
`Cache size at minimum (${formatBytes(allocation.minSize)}). ` +
`Consider increasing available memory for better performance.`
)
}
if (allocation.ratio > 0.6) {
warnings.push(
`Cache using ${(allocation.ratio * 100).toFixed(0)}% of available memory. ` +
`Monitor for memory pressure.`
)
}
return {
memoryInfo,
allocation,
warnings
}
}
/**
* Detect embedding model memory usage
*
* Returns estimated runtime memory for the embedding model:
* - Q8 (quantized, default): ~150MB runtime (22MB on disk)
* - FP32 (full precision): ~250MB runtime (86MB on disk)
*
* Breakdown for Q8:
* - Model weights: 22MB
* - ONNX Runtime: 15-30MB
* - Session workspace: 50-100MB (peak during inference)
* - Total: ~100-150MB (we use 150MB conservative)
*/
export function detectModelMemory(options: {
/** Model precision (default: 'q8') */
precision?: 'q8' | 'fp32'
} = {}): {
bytes: number
precision: 'q8' | 'fp32'
breakdown: {
modelWeights: number
onnxRuntime: number
sessionWorkspace: number
}
} {
const precision = options.precision || 'q8'
if (precision === 'q8') {
// Q8 quantized model (default)
return {
bytes: 150 * 1024 * 1024, // 150MB
precision: 'q8',
breakdown: {
modelWeights: 22 * 1024 * 1024, // 22MB
onnxRuntime: 30 * 1024 * 1024, // 30MB (conservative)
sessionWorkspace: 98 * 1024 * 1024 // 98MB (peak during inference)
}
}
} else {
// FP32 full precision model
return {
bytes: 250 * 1024 * 1024, // 250MB
precision: 'fp32',
breakdown: {
modelWeights: 86 * 1024 * 1024, // 86MB
onnxRuntime: 30 * 1024 * 1024, // 30MB
sessionWorkspace: 134 * 1024 * 1024 // 134MB (peak during inference)
}
}
}
}
/**
* Format bytes to human-readable string
*/
export function formatBytes(bytes: number): string {
if (bytes === 0) return '0 B'
const k = 1024
const sizes = ['B', 'KB', 'MB', 'GB', 'TB']
const i = Math.floor(Math.log(bytes) / Math.log(k))
return `${(bytes / Math.pow(k, i)).toFixed(2)} ${sizes[i]}`
}
/**
* Monitor memory usage and warn if approaching limits
*/
export function checkMemoryPressure(
cacheSize: number,
memoryInfo: MemoryInfo
): {
pressure: 'none' | 'moderate' | 'high' | 'critical'
warnings: string[]
} {
const warnings: string[] = []
const heapUsed = process.memoryUsage().heapUsed
const totalUsed = heapUsed + cacheSize
const utilization = totalUsed / memoryInfo.available
if (utilization > 0.95) {
warnings.push(
`🔴 CRITICAL: Memory utilization at ${(utilization * 100).toFixed(1)}%. ` +
`Reduce cache size or increase available memory.`
)
return { pressure: 'critical', warnings }
}
if (utilization > 0.85) {
warnings.push(
`🟠 HIGH: Memory utilization at ${(utilization * 100).toFixed(1)}%. ` +
`Consider increasing available memory.`
)
return { pressure: 'high', warnings }
}
if (utilization > 0.70) {
warnings.push(
`🟡 MODERATE: Memory utilization at ${(utilization * 100).toFixed(1)}%. ` +
`Monitor for memory pressure.`
)
return { pressure: 'moderate', warnings }
}
return { pressure: 'none', warnings: [] }
}

View file

@ -1,9 +1,22 @@
/**
* UnifiedCache - Single cache for both HNSW and MetadataIndex
* Prevents resource competition with cost-aware eviction
*
* Features (v3.36.0+):
* - Adaptive sizing: Automatically scales from 2GB to 128GB+ based on available memory
* - Container-aware: Detects Docker/K8s limits (cgroups v1/v2)
* - Environment detection: Production vs development allocation strategies
* - Memory pressure monitoring: Warns when approaching limits
*/
import { prodLog } from './logger.js'
import {
getRecommendedCacheConfig,
formatBytes,
checkMemoryPressure,
type MemoryInfo,
type CacheAllocationStrategy
} from './memoryDetection.js'
export interface CacheItem {
key: string
@ -16,11 +29,32 @@ export interface CacheItem {
}
export interface UnifiedCacheConfig {
maxSize?: number // bytes
/** Maximum cache size in bytes (auto-detected if not specified) */
maxSize?: number
/** Minimum cache size in bytes (default 256MB) */
minSize?: number
/** Force development mode allocation (25% instead of 40-50%) */
developmentMode?: boolean
/** Enable request coalescing to prevent duplicate loads */
enableRequestCoalescing?: boolean
/** Enable fairness monitoring to prevent cache starvation */
enableFairnessCheck?: boolean
fairnessCheckInterval?: number // ms
/** Fairness check interval in milliseconds */
fairnessCheckInterval?: number
/** Enable access pattern persistence for warm starts */
persistPatterns?: boolean
/** Enable memory pressure monitoring (default true) */
enableMemoryMonitoring?: boolean
/** Memory pressure check interval in milliseconds (default 30s) */
memoryCheckInterval?: number
}
export class UnifiedCache {
@ -33,19 +67,67 @@ export class UnifiedCache {
private readonly maxSize: number
private readonly config: UnifiedCacheConfig
// Memory management (v3.36.0+)
private readonly memoryInfo: MemoryInfo
private readonly allocationStrategy: CacheAllocationStrategy
private memoryPressureCheckTimer: NodeJS.Timeout | null = null
private lastMemoryWarning = 0
constructor(config: UnifiedCacheConfig = {}) {
this.maxSize = config.maxSize || 2 * 1024 * 1024 * 1024 // 2GB default
// Adaptive cache sizing (v3.36.0+)
const recommendation = getRecommendedCacheConfig({
manualSize: config.maxSize,
minSize: config.minSize,
developmentMode: config.developmentMode
})
this.memoryInfo = recommendation.memoryInfo
this.allocationStrategy = recommendation.allocation
this.maxSize = recommendation.allocation.cacheSize
// Log allocation decision (v3.36.0+: includes model memory)
prodLog.info(
`UnifiedCache initialized: ${formatBytes(this.maxSize)} ` +
`(${this.allocationStrategy.environment} mode, ` +
`${(this.allocationStrategy.ratio * 100).toFixed(0)}% of ${formatBytes(this.allocationStrategy.availableForCache)} ` +
`after ${formatBytes(this.allocationStrategy.modelMemory)} ${this.allocationStrategy.modelPrecision.toUpperCase()} model)`
)
// Log memory detection details
prodLog.debug(
`Memory detection: source=${this.memoryInfo.source}, ` +
`container=${this.memoryInfo.isContainer}, ` +
`system=${formatBytes(this.memoryInfo.systemTotal)}, ` +
`free=${formatBytes(this.memoryInfo.free)}, ` +
`totalAvailable=${formatBytes(this.memoryInfo.available)}, ` +
`modelReserved=${formatBytes(this.allocationStrategy.modelMemory)}, ` +
`availableForCache=${formatBytes(this.allocationStrategy.availableForCache)}`
)
// Log warnings if any
for (const warning of recommendation.warnings) {
prodLog.warn(`UnifiedCache: ${warning}`)
}
// Finalize configuration
this.config = {
enableRequestCoalescing: true,
enableFairnessCheck: true,
fairnessCheckInterval: 60000, // Check fairness every minute
persistPatterns: true,
enableMemoryMonitoring: true,
memoryCheckInterval: 30000, // Check memory every 30s
...config
}
// Start monitoring
if (this.config.enableFairnessCheck) {
this.startFairnessMonitor()
}
if (this.config.enableMemoryMonitoring) {
this.startMemoryPressureMonitor()
}
}
/**
@ -92,6 +174,27 @@ export class UnifiedCache {
}
}
/**
* Synchronous cache lookup (v3.36.0+)
* Returns cached data immediately or undefined if not cached
* Use for sync fast path optimization - zero async overhead
*/
getSync(key: string): any | undefined {
// Check if in cache
const item = this.cache.get(key)
if (item) {
// Update access tracking synchronously
this.access.set(key, (this.access.get(key) || 0) + 1)
this.totalAccessCount++
item.lastAccess = Date.now()
item.accessCount++
this.typeAccessCounts[item.type]++
return item.data
}
return undefined
}
/**
* Set item in cache with cost-aware eviction
*/
@ -300,7 +403,55 @@ export class UnifiedCache {
}
/**
* Get cache statistics
* Start memory pressure monitoring
* Periodically checks if we're approaching memory limits
*/
private startMemoryPressureMonitor(): void {
const checkInterval = this.config.memoryCheckInterval || 30000
this.memoryPressureCheckTimer = setInterval(() => {
this.checkMemoryPressure()
}, checkInterval)
// Unref so it doesn't keep process alive
if (this.memoryPressureCheckTimer.unref) {
this.memoryPressureCheckTimer.unref()
}
}
/**
* Check current memory pressure and warn if needed
*/
private checkMemoryPressure(): void {
const pressure = checkMemoryPressure(this.currentSize, this.memoryInfo)
// Only log warnings every 5 minutes to avoid spam
const now = Date.now()
const fiveMinutes = 5 * 60 * 1000
if (pressure.warnings.length > 0 && now - this.lastMemoryWarning > fiveMinutes) {
for (const warning of pressure.warnings) {
prodLog.warn(`UnifiedCache: ${warning}`)
}
this.lastMemoryWarning = now
}
// If critical, force aggressive eviction
if (pressure.pressure === 'critical') {
const targetSize = Math.floor(this.maxSize * 0.7) // Evict to 70%
const bytesToFree = this.currentSize - targetSize
if (bytesToFree > 0) {
prodLog.warn(
`UnifiedCache: Critical memory pressure - forcing eviction of ${formatBytes(bytesToFree)}`
)
this.evictForSize(bytesToFree)
}
}
}
/**
* Get cache statistics with memory information
*/
getStats() {
const typeSizes = { hnsw: 0, metadata: 0, embedding: 0, other: 0 }
@ -311,7 +462,11 @@ export class UnifiedCache {
typeCounts[item.type]++
}
const hitRate = this.cache.size > 0 ?
Array.from(this.cache.values()).reduce((sum, item) => sum + item.accessCount, 0) / this.totalAccessCount : 0
return {
// Cache statistics
totalSize: this.currentSize,
maxSize: this.maxSize,
utilization: this.currentSize / this.maxSize,
@ -320,8 +475,28 @@ export class UnifiedCache {
typeCounts,
typeAccessCounts: this.typeAccessCounts,
totalAccessCount: this.totalAccessCount,
hitRate: this.cache.size > 0 ?
Array.from(this.cache.values()).reduce((sum, item) => sum + item.accessCount, 0) / this.totalAccessCount : 0
hitRate,
// Memory management (v3.36.0+)
memory: {
available: this.memoryInfo.available,
source: this.memoryInfo.source,
isContainer: this.memoryInfo.isContainer,
systemTotal: this.memoryInfo.systemTotal,
allocationRatio: this.allocationStrategy.ratio,
environment: this.allocationStrategy.environment
}
}
}
/**
* Get detailed memory information
*/
getMemoryInfo() {
return {
memoryInfo: { ...this.memoryInfo },
allocationStrategy: { ...this.allocationStrategy },
currentPressure: checkMemoryPressure(this.currentSize, this.memoryInfo)
}
}