From 9d035aceeebbe7ad0ac9aab11b186260a770507c Mon Sep 17 00:00:00 2001 From: David Snelling Date: Tue, 24 Mar 2026 12:57:11 -0700 Subject: [PATCH] =?UTF-8?q?fix:=20metadata=20index=20corruption=20after=20?= =?UTF-8?q?restart=20=E2=80=94=206-point=20integrity=20fix?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Root cause: metadata index failed to reconstruct after deploy restart because the field registry file (__metadata_field_registry__) was lost during an interrupted flush. init() silently assumed the workspace was empty even though 5000+ entities existed on disk. Fix 1 (root cause): init() now probes storage for entities when field registry is missing. If entities exist, triggers rebuild instead of silently skipping. Never trusts a missing registry as "empty." Fix 2 (safety net): after rebuildIndexesIfNeeded(), verifies metadata index entry count matches storage entity count. Forces second rebuild if mismatch detected. Fix 3 (prevention): flush() now always saves field registry and EntityIdMapper, even when no dirty fields exist. These tiny files are the critical link that init() needs to discover persisted indices. Fix 5 (safe rebuild): rebuild() no longer deletes the field registry file before rewriting. If rebuild fails partway, the registry survives for the next init() to discover and re-trigger rebuild. Fix 6 (collision guard): EntityIdMapper init() warns when mapper file is missing but entities exist on disk, preventing silent ID collisions from nextId starting at 1. --- src/brainy.ts | 21 +++++++++-- src/utils/entityIdMapper.ts | 16 ++++++++ src/utils/metadataIndex.ts | 74 +++++++++++++++++++++++-------------- 3 files changed, 80 insertions(+), 31 deletions(-) diff --git a/src/brainy.ts b/src/brainy.ts index ddb69f90..68aa17ae 100644 --- a/src/brainy.ts +++ b/src/brainy.ts @@ -7170,16 +7170,29 @@ export class Brainy implements BrainyInterface { ]) const rebuildDuration = Date.now() - rebuildStartTime + const metadataCountAfter = (await this.metadataIndex.getStats()).totalEntries + if (!this.config.silent) { console.log( - `✅ All indexes rebuilt in ${rebuildDuration}ms:\n` + - ` - Metadata: ${await this.metadataIndex.getStats().then(s => s.totalEntries)} entries\n` + + `All indexes rebuilt in ${rebuildDuration}ms:\n` + + ` - Metadata: ${metadataCountAfter} entries\n` + ` - HNSW Vector: ${this.index.size()} nodes\n` + - ` - Graph Adjacency: ${await this.graphIndex.size()} relationships\n` + - ` 💡 Indexes loaded from persisted storage (no recomputation)` + ` - Graph Adjacency: ${await this.graphIndex.size()} relationships` ) } + // Consistency verification: metadata index must match storage entity count. + // If mismatch, the rebuild missed entities — force a second attempt. + if (metadataCountAfter === 0 && totalCount > 0) { + console.error( + `[Brainy] CRITICAL: Metadata index has 0 entries but storage has ${totalCount} entities. ` + + `Forcing second rebuild.` + ) + await this.metadataIndex.rebuild() + const secondAttempt = (await this.metadataIndex.getStats()).totalEntries + console.log(`[Brainy] Second rebuild result: ${secondAttempt} entries`) + } + } catch (error) { console.warn('Warning: Could not rebuild indexes:', error) // Don't throw - allow system to start even if rebuild fails diff --git a/src/utils/entityIdMapper.ts b/src/utils/entityIdMapper.ts index 0a10788b..78eadea8 100644 --- a/src/utils/entityIdMapper.ts +++ b/src/utils/entityIdMapper.ts @@ -62,6 +62,22 @@ export class EntityIdMapper { // Rebuild maps from serialized data this.uuidToInt = new Map(Object.entries(data.uuidToInt).map(([k, v]) => [k, Number(v)])) this.intToUuid = new Map(Object.entries(data.intToUuid).map(([k, v]) => [Number(k), v])) + } else { + // Guard: mapper file missing but entities may exist on disk. + // If we start from nextId=1 with existing entities, roaring bitmap + // queries will use wrong integer IDs → silent data corruption. + // Probe storage to detect this case and log a warning. + try { + const probe = await this.storage.getNouns({ pagination: { limit: 1, offset: 0 } }) + if ((probe.totalCount ?? 0) > 0 || probe.items.length > 0) { + console.warn( + `[EntityIdMapper] Mapper file missing but entities exist on disk. ` + + `IDs will be rebuilt during metadata index reconstruction.` + ) + } + } catch { + // Storage not ready + } } } catch (error) { // First time initialization - maps are empty, nextId = 1 diff --git a/src/utils/metadataIndex.ts b/src/utils/metadataIndex.ts index 142189b3..f408a999 100644 --- a/src/utils/metadataIndex.ts +++ b/src/utils/metadataIndex.ts @@ -223,23 +223,38 @@ export class MetadataIndexManager { // Initialize EntityIdMapper (loads UUID ↔ integer mappings from storage) await this.idMapper.init() - // Short-circuit: skip warmCache, lazyLoadCounts, syncTypeCountsToFixed for empty workspace - // On empty workspace, there are no fields to warm — avoids 4-6 wasted storage reads (404s) + // Check if field registry was loaded successfully const hasFields = this.fieldIndexes.size > 0 - if (hasFields) { - // Warm the cache with common fields (lazy loading optimization) - // This loads the 'noun' sparse index which is needed for type counts - await this.warmCache() + if (!hasFields) { + // Don't trust "empty" — field registry may be missing due to interrupted flush. + // Probe storage for actual entities before concluding the workspace is empty. + try { + const probe = await this.storage.getNouns({ pagination: { limit: 1, offset: 0 } }) + const hasEntities = (probe.totalCount ?? 0) > 0 || probe.items.length > 0 - // Load type counts AFTER warmCache (sparse index is now cached) - // Previously called in constructor without await and read from wrong source - await this.lazyLoadCounts() - - // Phase 1b: Sync loaded counts to fixed-size arrays - // Now correctly happens AFTER lazyLoadCounts() finishes - this.syncTypeCountsToFixed() + if (hasEntities) { + console.warn( + `[MetadataIndex] Field registry missing but ${probe.totalCount ?? 'unknown'} entities exist on disk — rebuilding index` + ) + await this.rebuild() + return // rebuild handles warmCache + lazyLoadCounts internally + } + } catch { + // Storage probe failed — genuinely empty or storage not ready + } + return // Truly empty workspace — nothing to warm } + + // Warm the cache with common fields (lazy loading optimization) + // This loads the 'noun' sparse index which is needed for type counts + await this.warmCache() + + // Load type counts AFTER warmCache (sparse index is now cached) + await this.lazyLoadCounts() + + // Phase 1b: Sync loaded counts to fixed-size arrays + this.syncTypeCountsToFixed() } /** @@ -2409,9 +2424,19 @@ export class MetadataIndexManager { // Flush any deferred chunk/sparse writes first await this.flushDirtyMetadata() - // Check if we have anything to flush + // Always save field registry — even with no dirty fields. This tiny file + // (list of field names) is the critical link that init() needs to discover + // persisted indices. Without it, the index appears empty after restart. + if (this.fieldIndexes.size > 0) { + await this.saveFieldRegistry() + } + + // Also always flush the EntityIdMapper — prevents ID collisions on restart + await this.idMapper.flush() + + // Check if we have anything else to flush if (this.dirtyFields.size === 0) { - return // Nothing to flush + return // No dirty field indexes to flush } // Process in smaller batches to avoid blocking @@ -3072,10 +3097,12 @@ export class MetadataIndexManager { // This ensures rebuild starts fresh this.unifiedCache.clear('metadata') - // CRITICAL FIX - Delete existing chunk files from storage - // Without this, old chunk data accumulates with each rebuild causing 77x overcounting! - // Previous fix cleared type counts but missed chunk file accumulation. - prodLog.info('🗑️ Clearing existing metadata index chunks from storage...') + // Clear existing chunk files from storage to prevent overcounting. + // Chunks are deleted first, then rebuilt. The field registry is NOT deleted + // here — it's always saved at the end of rebuild via flush(). This ensures + // that if rebuild fails partway, the next init() can still discover fields + // and trigger another rebuild attempt. + prodLog.info('Clearing existing metadata index chunks from storage...') const existingFields = await this.getPersistedFieldList() if (existingFields.length > 0) { @@ -3083,14 +3110,7 @@ export class MetadataIndexManager { await this.deleteFieldChunks(field) } - // Delete field registry (will be recreated on flush) - try { - await this.storage.saveMetadata('__metadata_field_registry__', null as any) - } catch (error) { - prodLog.debug('Could not delete field registry:', error) - } - - prodLog.info(`✅ Cleared ${existingFields.length} field indexes from storage`) + prodLog.info(`Cleared ${existingFields.length} field indexes from storage`) } // Clear EntityIdMapper to start fresh