feat: implement always-adaptive caching with getCacheStats monitoring
Replaces lazy mode concept with always-adaptive caching strategy:
- Rename getLazyModeStats() → getCacheStats() with enhanced metrics
- Change lazyModeEnabled boolean → cachingStrategy enum ('preloaded' | 'on-demand')
- Update preloading threshold from 30% to 80% for better cache utilization
- Add comprehensive production monitoring and diagnostics
- Add memory detection for containers (Docker/K8s cgroups v1/v2)
- Add adaptive memory sizing from 2GB to 128GB+ systems
Breaking changes: None (backward compatible, deprecated lazy option ignored)
New APIs:
- getCacheStats(): Comprehensive cache performance statistics
- cachingStrategy field: Transparent strategy reporting
- Enhanced fairness metrics and memory pressure monitoring
Documentation:
- Add migration guide for v3.36.0
- Add operations/capacity-planning.md for enterprise deployments
- Update all examples and troubleshooting guides
- Rename monitor-lazy-mode.ts → monitor-cache-performance.ts
This commit is contained in:
parent
6037db3d85
commit
46c6af3f21
17 changed files with 2737 additions and 127 deletions
476
src/utils/memoryDetection.ts
Normal file
476
src/utils/memoryDetection.ts
Normal file
|
|
@ -0,0 +1,476 @@
|
|||
/**
|
||||
* Memory Detection Utilities
|
||||
* Detects available system memory across different environments:
|
||||
* - Docker/Kubernetes (cgroups v1 and v2)
|
||||
* - Bare metal servers
|
||||
* - Cloud instances
|
||||
* - Development environments
|
||||
*
|
||||
* Scales from 2GB to 128GB+ with intelligent allocation
|
||||
*/
|
||||
|
||||
import * as os from 'os'
|
||||
import * as fs from 'fs'
|
||||
import { prodLog } from './logger.js'
|
||||
|
||||
export interface MemoryInfo {
|
||||
/** Total memory available to this process (bytes) */
|
||||
available: number
|
||||
|
||||
/** Source of memory information */
|
||||
source: 'cgroup-v2' | 'cgroup-v1' | 'system' | 'fallback'
|
||||
|
||||
/** Whether running in a container */
|
||||
isContainer: boolean
|
||||
|
||||
/** System total memory (may differ from available in containers) */
|
||||
systemTotal: number
|
||||
|
||||
/** Currently free memory (best-effort estimate) */
|
||||
free: number
|
||||
|
||||
/** Detection warnings (if any) */
|
||||
warnings: string[]
|
||||
}
|
||||
|
||||
export interface CacheAllocationStrategy {
|
||||
/** Recommended cache size (bytes) */
|
||||
cacheSize: number
|
||||
|
||||
/** Allocation ratio used (0-1) */
|
||||
ratio: number
|
||||
|
||||
/** Minimum guaranteed size (bytes) */
|
||||
minSize: number
|
||||
|
||||
/** Maximum allowed size (bytes) */
|
||||
maxSize: number | null
|
||||
|
||||
/** Environment type detected */
|
||||
environment: 'production' | 'development' | 'container' | 'unknown'
|
||||
|
||||
/** Model memory reserved (bytes) - v3.36.0+ */
|
||||
modelMemory: number
|
||||
|
||||
/** Model precision (q8 or fp32) */
|
||||
modelPrecision: 'q8' | 'fp32'
|
||||
|
||||
/** Available memory after model reservation (bytes) */
|
||||
availableForCache: number
|
||||
|
||||
/** Reasoning for allocation */
|
||||
reasoning: string
|
||||
}
|
||||
|
||||
/**
|
||||
* Detect available memory across all environments
|
||||
*/
|
||||
export function detectAvailableMemory(): MemoryInfo {
|
||||
const warnings: string[] = []
|
||||
|
||||
// Try cgroups v2 first (modern Docker/K8s)
|
||||
const cgroupV2 = detectCgroupV2Memory()
|
||||
if (cgroupV2 !== null) {
|
||||
const systemTotal = os.totalmem()
|
||||
const free = os.freemem()
|
||||
|
||||
return {
|
||||
available: cgroupV2,
|
||||
source: 'cgroup-v2',
|
||||
isContainer: true,
|
||||
systemTotal,
|
||||
free,
|
||||
warnings: cgroupV2 < systemTotal
|
||||
? [`Container limited to ${formatBytes(cgroupV2)} (host has ${formatBytes(systemTotal)})`]
|
||||
: []
|
||||
}
|
||||
}
|
||||
|
||||
// Try cgroups v1 (older Docker/K8s)
|
||||
const cgroupV1 = detectCgroupV1Memory()
|
||||
if (cgroupV1 !== null) {
|
||||
const systemTotal = os.totalmem()
|
||||
const free = os.freemem()
|
||||
|
||||
return {
|
||||
available: cgroupV1,
|
||||
source: 'cgroup-v1',
|
||||
isContainer: true,
|
||||
systemTotal,
|
||||
free,
|
||||
warnings: cgroupV1 < systemTotal
|
||||
? [`Container limited to ${formatBytes(cgroupV1)} (host has ${formatBytes(systemTotal)})`]
|
||||
: []
|
||||
}
|
||||
}
|
||||
|
||||
// Use system memory (bare metal, VM, or unlimited container)
|
||||
const systemTotal = os.totalmem()
|
||||
const free = os.freemem()
|
||||
|
||||
// Check if we might be in an unlimited container
|
||||
if (process.env.KUBERNETES_SERVICE_HOST || process.env.DOCKER_CONTAINER) {
|
||||
warnings.push('Container detected but no memory limit set - using host memory')
|
||||
}
|
||||
|
||||
return {
|
||||
available: systemTotal,
|
||||
source: 'system',
|
||||
isContainer: false,
|
||||
systemTotal,
|
||||
free,
|
||||
warnings
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Detect memory limit from cgroups v2 (modern containers)
|
||||
* Path: /sys/fs/cgroup/memory.max
|
||||
*/
|
||||
function detectCgroupV2Memory(): number | null {
|
||||
try {
|
||||
const memoryMaxPath = '/sys/fs/cgroup/memory.max'
|
||||
|
||||
if (!fs.existsSync(memoryMaxPath)) {
|
||||
return null
|
||||
}
|
||||
|
||||
const content = fs.readFileSync(memoryMaxPath, 'utf8').trim()
|
||||
|
||||
// 'max' means unlimited
|
||||
if (content === 'max') {
|
||||
return null
|
||||
}
|
||||
|
||||
const bytes = parseInt(content, 10)
|
||||
|
||||
// Sanity check: Must be reasonable number (between 64MB and 1TB)
|
||||
if (bytes < 64 * 1024 * 1024 || bytes > 1024 * 1024 * 1024 * 1024) {
|
||||
prodLog.warn(`Suspicious cgroup v2 memory limit: ${formatBytes(bytes)}`)
|
||||
return null
|
||||
}
|
||||
|
||||
return bytes
|
||||
} catch (error) {
|
||||
// Not in a cgroup v2 environment
|
||||
return null
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Detect memory limit from cgroups v1 (older containers)
|
||||
* Path: /sys/fs/cgroup/memory/memory.limit_in_bytes
|
||||
*/
|
||||
function detectCgroupV1Memory(): number | null {
|
||||
try {
|
||||
const limitPath = '/sys/fs/cgroup/memory/memory.limit_in_bytes'
|
||||
|
||||
if (!fs.existsSync(limitPath)) {
|
||||
return null
|
||||
}
|
||||
|
||||
const content = fs.readFileSync(limitPath, 'utf8').trim()
|
||||
const bytes = parseInt(content, 10)
|
||||
|
||||
// cgroup v1 uses very large number (2^63-1) to indicate unlimited
|
||||
// If limit is > 1TB, consider it unlimited
|
||||
if (bytes > 1024 * 1024 * 1024 * 1024) {
|
||||
return null
|
||||
}
|
||||
|
||||
// Sanity check: Must be reasonable number (between 64MB and 1TB)
|
||||
if (bytes < 64 * 1024 * 1024) {
|
||||
prodLog.warn(`Suspicious cgroup v1 memory limit: ${formatBytes(bytes)}`)
|
||||
return null
|
||||
}
|
||||
|
||||
return bytes
|
||||
} catch (error) {
|
||||
// Not in a cgroup v1 environment
|
||||
return null
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Calculate optimal cache size based on available memory
|
||||
* Scales intelligently from 2GB to 128GB+
|
||||
*
|
||||
* v3.36.0+: Accounts for embedding model memory (150MB Q8, 250MB FP32)
|
||||
*/
|
||||
export function calculateOptimalCacheSize(
|
||||
memoryInfo: MemoryInfo,
|
||||
options: {
|
||||
/** Manual override (bytes) - takes precedence */
|
||||
manualSize?: number
|
||||
|
||||
/** Minimum cache size (bytes) - default 256MB */
|
||||
minSize?: number
|
||||
|
||||
/** Maximum cache size (bytes) - default unlimited */
|
||||
maxSize?: number
|
||||
|
||||
/** Force development mode allocation (more conservative) */
|
||||
developmentMode?: boolean
|
||||
|
||||
/** Model precision for memory calculation - default 'q8' */
|
||||
modelPrecision?: 'q8' | 'fp32'
|
||||
} = {}
|
||||
): CacheAllocationStrategy {
|
||||
const minSize = options.minSize || 256 * 1024 * 1024 // 256MB minimum
|
||||
const maxSize = options.maxSize || null
|
||||
|
||||
// Detect model memory usage (v3.36.0+)
|
||||
const modelInfo = detectModelMemory({ precision: options.modelPrecision || 'q8' })
|
||||
const modelMemory = modelInfo.bytes
|
||||
|
||||
// Reserve model memory from available RAM BEFORE calculating cache
|
||||
// This ensures we don't over-allocate and cause OOM
|
||||
const availableForCache = Math.max(0, memoryInfo.available - modelMemory)
|
||||
|
||||
// Manual override takes precedence
|
||||
if (options.manualSize !== undefined) {
|
||||
const clamped = Math.max(minSize, options.manualSize)
|
||||
return {
|
||||
cacheSize: clamped,
|
||||
ratio: clamped / availableForCache,
|
||||
minSize,
|
||||
maxSize,
|
||||
environment: 'unknown',
|
||||
modelMemory,
|
||||
modelPrecision: modelInfo.precision,
|
||||
availableForCache,
|
||||
reasoning: 'Manual override specified'
|
||||
}
|
||||
}
|
||||
|
||||
// Determine environment and allocation ratio
|
||||
let ratio: number
|
||||
let environment: CacheAllocationStrategy['environment']
|
||||
let reasoning: string
|
||||
|
||||
if (options.developmentMode || process.env.NODE_ENV === 'development') {
|
||||
// Development: More conservative (25%)
|
||||
ratio = 0.25
|
||||
environment = 'development'
|
||||
reasoning = `Development mode - conservative allocation (25% of ${formatBytes(availableForCache)} after ${formatBytes(modelMemory)} model)`
|
||||
} else if (memoryInfo.isContainer) {
|
||||
// Container: Moderate allocation (40%)
|
||||
// Containers often have tight limits, leave room for heap growth
|
||||
ratio = 0.40
|
||||
environment = 'container'
|
||||
reasoning = `Container environment - moderate allocation (40% of ${formatBytes(availableForCache)} after ${formatBytes(modelMemory)} model)`
|
||||
} else {
|
||||
// Production bare metal/VM: Aggressive allocation (50%)
|
||||
// More memory available, can be more aggressive
|
||||
ratio = 0.50
|
||||
environment = 'production'
|
||||
reasoning = `Production environment - aggressive allocation (50% of ${formatBytes(availableForCache)} after ${formatBytes(modelMemory)} model)`
|
||||
}
|
||||
|
||||
// Calculate base cache size from AVAILABLE memory (after model reservation)
|
||||
let cacheSize = Math.floor(availableForCache * ratio)
|
||||
|
||||
// Apply minimum constraint
|
||||
if (cacheSize < minSize) {
|
||||
const originalSize = cacheSize
|
||||
cacheSize = minSize
|
||||
reasoning += ` (increased from ${formatBytes(originalSize)} to meet minimum)`
|
||||
|
||||
// Warn if available memory is very low
|
||||
if (availableForCache < minSize * 2) {
|
||||
prodLog.warn(
|
||||
`⚠️ Low available memory for cache (${formatBytes(availableForCache)} after ${formatBytes(modelMemory)} model). ` +
|
||||
`Cache size ${formatBytes(cacheSize)} may cause memory pressure.`
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
// Apply maximum constraint
|
||||
if (maxSize !== null && cacheSize > maxSize) {
|
||||
const originalSize = cacheSize
|
||||
cacheSize = maxSize
|
||||
reasoning += ` (capped from ${formatBytes(originalSize)} to maximum)`
|
||||
}
|
||||
|
||||
// Intelligent scaling for large memory systems
|
||||
// For systems with >64GB available for cache, use logarithmic scaling to avoid over-allocation
|
||||
if (availableForCache > 64 * 1024 * 1024 * 1024) {
|
||||
// Above 64GB, scale more conservatively
|
||||
// Formula: base + log2(availableForCache/64GB) * 8GB
|
||||
const base = 32 * 1024 * 1024 * 1024 // 32GB base
|
||||
const scaleFactor = Math.log2(availableForCache / (64 * 1024 * 1024 * 1024))
|
||||
const scaled = base + scaleFactor * 8 * 1024 * 1024 * 1024 // +8GB per doubling
|
||||
|
||||
if (scaled < cacheSize) {
|
||||
const originalSize = cacheSize
|
||||
cacheSize = Math.floor(scaled)
|
||||
reasoning += ` (scaled down from ${formatBytes(originalSize)} for large memory system)`
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
cacheSize,
|
||||
ratio,
|
||||
minSize,
|
||||
maxSize,
|
||||
environment,
|
||||
modelMemory,
|
||||
modelPrecision: modelInfo.precision,
|
||||
availableForCache,
|
||||
reasoning
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get recommended cache configuration for current environment
|
||||
*/
|
||||
export function getRecommendedCacheConfig(options: {
|
||||
/** Manual cache size override (bytes) */
|
||||
manualSize?: number
|
||||
|
||||
/** Minimum cache size (bytes) */
|
||||
minSize?: number
|
||||
|
||||
/** Maximum cache size (bytes) */
|
||||
maxSize?: number
|
||||
|
||||
/** Force development mode */
|
||||
developmentMode?: boolean
|
||||
} = {}): {
|
||||
memoryInfo: MemoryInfo
|
||||
allocation: CacheAllocationStrategy
|
||||
warnings: string[]
|
||||
} {
|
||||
const memoryInfo = detectAvailableMemory()
|
||||
const allocation = calculateOptimalCacheSize(memoryInfo, options)
|
||||
|
||||
const warnings: string[] = [...memoryInfo.warnings]
|
||||
|
||||
// Add allocation warnings
|
||||
if (allocation.cacheSize === allocation.minSize) {
|
||||
warnings.push(
|
||||
`Cache size at minimum (${formatBytes(allocation.minSize)}). ` +
|
||||
`Consider increasing available memory for better performance.`
|
||||
)
|
||||
}
|
||||
|
||||
if (allocation.ratio > 0.6) {
|
||||
warnings.push(
|
||||
`Cache using ${(allocation.ratio * 100).toFixed(0)}% of available memory. ` +
|
||||
`Monitor for memory pressure.`
|
||||
)
|
||||
}
|
||||
|
||||
return {
|
||||
memoryInfo,
|
||||
allocation,
|
||||
warnings
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Detect embedding model memory usage
|
||||
*
|
||||
* Returns estimated runtime memory for the embedding model:
|
||||
* - Q8 (quantized, default): ~150MB runtime (22MB on disk)
|
||||
* - FP32 (full precision): ~250MB runtime (86MB on disk)
|
||||
*
|
||||
* Breakdown for Q8:
|
||||
* - Model weights: 22MB
|
||||
* - ONNX Runtime: 15-30MB
|
||||
* - Session workspace: 50-100MB (peak during inference)
|
||||
* - Total: ~100-150MB (we use 150MB conservative)
|
||||
*/
|
||||
export function detectModelMemory(options: {
|
||||
/** Model precision (default: 'q8') */
|
||||
precision?: 'q8' | 'fp32'
|
||||
} = {}): {
|
||||
bytes: number
|
||||
precision: 'q8' | 'fp32'
|
||||
breakdown: {
|
||||
modelWeights: number
|
||||
onnxRuntime: number
|
||||
sessionWorkspace: number
|
||||
}
|
||||
} {
|
||||
const precision = options.precision || 'q8'
|
||||
|
||||
if (precision === 'q8') {
|
||||
// Q8 quantized model (default)
|
||||
return {
|
||||
bytes: 150 * 1024 * 1024, // 150MB
|
||||
precision: 'q8',
|
||||
breakdown: {
|
||||
modelWeights: 22 * 1024 * 1024, // 22MB
|
||||
onnxRuntime: 30 * 1024 * 1024, // 30MB (conservative)
|
||||
sessionWorkspace: 98 * 1024 * 1024 // 98MB (peak during inference)
|
||||
}
|
||||
}
|
||||
} else {
|
||||
// FP32 full precision model
|
||||
return {
|
||||
bytes: 250 * 1024 * 1024, // 250MB
|
||||
precision: 'fp32',
|
||||
breakdown: {
|
||||
modelWeights: 86 * 1024 * 1024, // 86MB
|
||||
onnxRuntime: 30 * 1024 * 1024, // 30MB
|
||||
sessionWorkspace: 134 * 1024 * 1024 // 134MB (peak during inference)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Format bytes to human-readable string
|
||||
*/
|
||||
export function formatBytes(bytes: number): string {
|
||||
if (bytes === 0) return '0 B'
|
||||
|
||||
const k = 1024
|
||||
const sizes = ['B', 'KB', 'MB', 'GB', 'TB']
|
||||
const i = Math.floor(Math.log(bytes) / Math.log(k))
|
||||
|
||||
return `${(bytes / Math.pow(k, i)).toFixed(2)} ${sizes[i]}`
|
||||
}
|
||||
|
||||
/**
|
||||
* Monitor memory usage and warn if approaching limits
|
||||
*/
|
||||
export function checkMemoryPressure(
|
||||
cacheSize: number,
|
||||
memoryInfo: MemoryInfo
|
||||
): {
|
||||
pressure: 'none' | 'moderate' | 'high' | 'critical'
|
||||
warnings: string[]
|
||||
} {
|
||||
const warnings: string[] = []
|
||||
const heapUsed = process.memoryUsage().heapUsed
|
||||
const totalUsed = heapUsed + cacheSize
|
||||
const utilization = totalUsed / memoryInfo.available
|
||||
|
||||
if (utilization > 0.95) {
|
||||
warnings.push(
|
||||
`🔴 CRITICAL: Memory utilization at ${(utilization * 100).toFixed(1)}%. ` +
|
||||
`Reduce cache size or increase available memory.`
|
||||
)
|
||||
return { pressure: 'critical', warnings }
|
||||
}
|
||||
|
||||
if (utilization > 0.85) {
|
||||
warnings.push(
|
||||
`🟠 HIGH: Memory utilization at ${(utilization * 100).toFixed(1)}%. ` +
|
||||
`Consider increasing available memory.`
|
||||
)
|
||||
return { pressure: 'high', warnings }
|
||||
}
|
||||
|
||||
if (utilization > 0.70) {
|
||||
warnings.push(
|
||||
`🟡 MODERATE: Memory utilization at ${(utilization * 100).toFixed(1)}%. ` +
|
||||
`Monitor for memory pressure.`
|
||||
)
|
||||
return { pressure: 'moderate', warnings }
|
||||
}
|
||||
|
||||
return { pressure: 'none', warnings: [] }
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue