fix: exclude __words__ keyword index from corruption detection and getStats()

The __words__ keyword index stores 50-5000 entries per entity (one per
word), which inflated avg entries/entity well above the corruption
threshold of 100. This caused:

1. validateConsistency() to falsely detect corruption on every startup,
   triggering unnecessary clearAllIndexData() + rebuild() cycles
2. getStats() to log false "Metadata index may be corrupted" warnings
   and report inflated totalEntries/totalIds stats

Both methods now skip __words__ when counting, so stats and health
checks reflect metadata fields only (noun, type, createdAt, etc.).
Keyword search is unaffected since the __words__ field index itself
is not modified.
This commit is contained in:
David Snelling 2026-01-27 15:38:21 -08:00
parent 32dbdcec61
commit 364360d447
128 changed files with 5637 additions and 5682 deletions

View file

@ -2,7 +2,7 @@
* UnifiedCache - Single cache for both HNSW and MetadataIndex
* Prevents resource competition with cost-aware eviction
*
* Features (v3.36.0+):
* Features:
* - Adaptive sizing: Automatically scales from 2GB to 128GB+ based on available memory
* - Container-aware: Detects Docker/K8s limits (cgroups v1/v2)
* - Environment detection: Production vs development allocation strategies
@ -67,14 +67,14 @@ export class UnifiedCache {
private readonly maxSize: number
private readonly config: UnifiedCacheConfig
// Memory management (v3.36.0+)
// Memory management
private readonly memoryInfo: MemoryInfo
private readonly allocationStrategy: CacheAllocationStrategy
private memoryPressureCheckTimer: NodeJS.Timeout | null = null
private lastMemoryWarning = 0
constructor(config: UnifiedCacheConfig = {}) {
// Adaptive cache sizing (v3.36.0+)
// Adaptive cache sizing
const recommendation = getRecommendedCacheConfig({
manualSize: config.maxSize,
minSize: config.minSize,
@ -85,7 +85,7 @@ export class UnifiedCache {
this.allocationStrategy = recommendation.allocation
this.maxSize = recommendation.allocation.cacheSize
// Log allocation decision (v3.36.0+: includes model memory)
// Log allocation decision (includes model memory)
prodLog.info(
`UnifiedCache initialized: ${formatBytes(this.maxSize)} ` +
`(${this.allocationStrategy.environment} mode, ` +
@ -113,7 +113,7 @@ export class UnifiedCache {
this.config = {
enableRequestCoalescing: true,
enableFairnessCheck: true,
fairnessCheckInterval: 30000, // Check fairness every 30 seconds (v3.40.2: was 60s)
fairnessCheckInterval: 30000, // Check fairness every 30 seconds (was 60s)
persistPatterns: true,
enableMemoryMonitoring: true,
memoryCheckInterval: 30000, // Check memory every 30s
@ -175,7 +175,7 @@ export class UnifiedCache {
}
/**
* Synchronous cache lookup (v3.36.0+)
* Synchronous cache lookup
* Returns cached data immediately or undefined if not cached
* Use for sync fast path optimization - zero async overhead
*/
@ -232,7 +232,7 @@ export class UnifiedCache {
this.typeAccessCounts[type]++
this.totalAccessCount++
// Proactive fairness check (v3.40.2): Check immediately if adding to a dominant type
// Proactive fairness check: Check immediately if adding to a dominant type
// This prevents imbalance formation instead of reacting to it
if (this.config.enableFairnessCheck && this.cache.size > 10) {
this.checkProactiveFairness(type)
@ -341,7 +341,7 @@ export class UnifiedCache {
other: typeSizes.other / totalSize
}
// Check for starvation (v3.40.2: more aggressive - 70% cache with <15% accesses)
// Check for starvation (more aggressive - 70% cache with <15% accesses)
// Previous: 90% cache, <10% access (too lenient, caused thrashing)
for (const type of ['hnsw', 'metadata', 'embedding', 'other'] as const) {
if (sizeRatios[type] > 0.7 && accessRatios[type] < 0.15) {
@ -352,7 +352,7 @@ export class UnifiedCache {
}
/**
* Proactive fairness check (v3.40.2)
* Proactive fairness check
* Called immediately when adding items to prevent imbalance formation
* Uses same thresholds as periodic check but runs on-demand
*/
@ -391,7 +391,7 @@ export class UnifiedCache {
// Sort by score (lower is worse)
candidates.sort((a, b) => a[1] - b[1])
// Evict bottom 50% of this type (v3.40.2: was 20%, too slow to prevent thrashing)
// Evict bottom 50% of this type (was 20%, too slow to prevent thrashing)
const evictCount = Math.max(1, Math.floor(candidates.length * 0.5))
for (let i = 0; i < evictCount && i < candidates.length; i++) {
@ -417,7 +417,7 @@ export class UnifiedCache {
/**
* Delete all items with keys starting with the given prefix
* v6.2.9: Added for VFS cache invalidation (fixes stale parent ID bug)
* Added for VFS cache invalidation (fixes stale parent ID bug)
* @param prefix - The key prefix to match
* @returns Number of items deleted
*/
@ -526,7 +526,7 @@ export class UnifiedCache {
totalAccessCount: this.totalAccessCount,
hitRate,
// Memory management (v3.36.0+)
// Memory management
memory: {
available: this.memoryInfo.available,
source: this.memoryInfo.source,