fix: metadata index corruption after restart — 6-point integrity fix

Root cause: metadata index failed to reconstruct after deploy restart
because the field registry file (__metadata_field_registry__) was lost
during an interrupted flush. init() silently assumed the workspace was
empty even though 5000+ entities existed on disk.

Fix 1 (root cause): init() now probes storage for entities when field
registry is missing. If entities exist, triggers rebuild instead of
silently skipping. Never trusts a missing registry as "empty."

Fix 2 (safety net): after rebuildIndexesIfNeeded(), verifies metadata
index entry count matches storage entity count. Forces second rebuild
if mismatch detected.

Fix 3 (prevention): flush() now always saves field registry and
EntityIdMapper, even when no dirty fields exist. These tiny files are
the critical link that init() needs to discover persisted indices.

Fix 5 (safe rebuild): rebuild() no longer deletes the field registry
file before rewriting. If rebuild fails partway, the registry survives
for the next init() to discover and re-trigger rebuild.

Fix 6 (collision guard): EntityIdMapper init() warns when mapper file
is missing but entities exist on disk, preventing silent ID collisions
from nextId starting at 1.

src/brainy.ts

@@ -7170,16 +7170,29 @@ export class Brainy<T = any> implements BrainyInterface<T> {
       ])
 
       const rebuildDuration = Date.now() - rebuildStartTime
+      const metadataCountAfter = (await this.metadataIndex.getStats()).totalEntries
+
       if (!this.config.silent) {
         console.log(
-          `✅ All indexes rebuilt in ${rebuildDuration}ms:\n` +
-          `   - Metadata: ${await this.metadataIndex.getStats().then(s => s.totalEntries)} entries\n` +
+          `All indexes rebuilt in ${rebuildDuration}ms:\n` +
+          `   - Metadata: ${metadataCountAfter} entries\n` +
           `   - HNSW Vector: ${this.index.size()} nodes\n` +
-          `   - Graph Adjacency: ${await this.graphIndex.size()} relationships\n` +
-          `   💡 Indexes loaded from persisted storage (no recomputation)`
+          `   - Graph Adjacency: ${await this.graphIndex.size()} relationships`
         )
       }
 
+      // Consistency verification: metadata index must match storage entity count.
+      // If mismatch, the rebuild missed entities — force a second attempt.
+      if (metadataCountAfter === 0 && totalCount > 0) {
+        console.error(
+          `[Brainy] CRITICAL: Metadata index has 0 entries but storage has ${totalCount} entities. ` +
+          `Forcing second rebuild.`
+        )
+        await this.metadataIndex.rebuild()
+        const secondAttempt = (await this.metadataIndex.getStats()).totalEntries
+        console.log(`[Brainy] Second rebuild result: ${secondAttempt} entries`)
+      }
+
     } catch (error) {
       console.warn('Warning: Could not rebuild indexes:', error)
       // Don't throw - allow system to start even if rebuild fails

src/utils/entityIdMapper.ts

@@ -62,6 +62,22 @@ export class EntityIdMapper {
         // Rebuild maps from serialized data
         this.uuidToInt = new Map(Object.entries(data.uuidToInt).map(([k, v]) => [k, Number(v)]))
         this.intToUuid = new Map(Object.entries(data.intToUuid).map(([k, v]) => [Number(k), v]))
+      } else {
+        // Guard: mapper file missing but entities may exist on disk.
+        // If we start from nextId=1 with existing entities, roaring bitmap
+        // queries will use wrong integer IDs → silent data corruption.
+        // Probe storage to detect this case and log a warning.
+        try {
+          const probe = await this.storage.getNouns({ pagination: { limit: 1, offset: 0 } })
+          if ((probe.totalCount ?? 0) > 0 || probe.items.length > 0) {
+            console.warn(
+              `[EntityIdMapper] Mapper file missing but entities exist on disk. ` +
+              `IDs will be rebuilt during metadata index reconstruction.`
+            )
+          }
+        } catch {
+          // Storage not ready
+        }
       }
     } catch (error) {
       // First time initialization - maps are empty, nextId = 1

src/utils/metadataIndex.ts

@@ -223,23 +223,38 @@ export class MetadataIndexManager {
     // Initialize EntityIdMapper (loads UUID ↔ integer mappings from storage)
     await this.idMapper.init()
 
-    // Short-circuit: skip warmCache, lazyLoadCounts, syncTypeCountsToFixed for empty workspace
-    // On empty workspace, there are no fields to warm — avoids 4-6 wasted storage reads (404s)
+    // Check if field registry was loaded successfully
     const hasFields = this.fieldIndexes.size > 0
 
-    if (hasFields) {
-      // Warm the cache with common fields (lazy loading optimization)
-      // This loads the 'noun' sparse index which is needed for type counts
-      await this.warmCache()
+    if (!hasFields) {
+      // Don't trust "empty" — field registry may be missing due to interrupted flush.
+      // Probe storage for actual entities before concluding the workspace is empty.
+      try {
+        const probe = await this.storage.getNouns({ pagination: { limit: 1, offset: 0 } })
+        const hasEntities = (probe.totalCount ?? 0) > 0 || probe.items.length > 0
 
-      // Load type counts AFTER warmCache (sparse index is now cached)
-      // Previously called in constructor without await and read from wrong source
-      await this.lazyLoadCounts()
-
-      // Phase 1b: Sync loaded counts to fixed-size arrays
-      // Now correctly happens AFTER lazyLoadCounts() finishes
-      this.syncTypeCountsToFixed()
+        if (hasEntities) {
+          console.warn(
+            `[MetadataIndex] Field registry missing but ${probe.totalCount ?? 'unknown'} entities exist on disk — rebuilding index`
+          )
+          await this.rebuild()
+          return // rebuild handles warmCache + lazyLoadCounts internally
+        }
+      } catch {
+        // Storage probe failed — genuinely empty or storage not ready
+      }
+      return // Truly empty workspace — nothing to warm
     }
+
+    // Warm the cache with common fields (lazy loading optimization)
+    // This loads the 'noun' sparse index which is needed for type counts
+    await this.warmCache()
+
+    // Load type counts AFTER warmCache (sparse index is now cached)
+    await this.lazyLoadCounts()
+
+    // Phase 1b: Sync loaded counts to fixed-size arrays
+    this.syncTypeCountsToFixed()
   }
 
   /**
@@ -2409,9 +2424,19 @@ export class MetadataIndexManager {
     // Flush any deferred chunk/sparse writes first
     await this.flushDirtyMetadata()
 
-    // Check if we have anything to flush
+    // Always save field registry — even with no dirty fields. This tiny file
+    // (list of field names) is the critical link that init() needs to discover
+    // persisted indices. Without it, the index appears empty after restart.
+    if (this.fieldIndexes.size > 0) {
+      await this.saveFieldRegistry()
+    }
+
+    // Also always flush the EntityIdMapper — prevents ID collisions on restart
+    await this.idMapper.flush()
+
+    // Check if we have anything else to flush
     if (this.dirtyFields.size === 0) {
-      return // Nothing to flush
+      return // No dirty field indexes to flush
     }
 
     // Process in smaller batches to avoid blocking
@@ -3072,10 +3097,12 @@ export class MetadataIndexManager {
       // This ensures rebuild starts fresh
       this.unifiedCache.clear('metadata')
 
-      // CRITICAL FIX - Delete existing chunk files from storage
-      // Without this, old chunk data accumulates with each rebuild causing 77x overcounting!
-      // Previous fix cleared type counts but missed chunk file accumulation.
-      prodLog.info('🗑️ Clearing existing metadata index chunks from storage...')
+      // Clear existing chunk files from storage to prevent overcounting.
+      // Chunks are deleted first, then rebuilt. The field registry is NOT deleted
+      // here — it's always saved at the end of rebuild via flush(). This ensures
+      // that if rebuild fails partway, the next init() can still discover fields
+      // and trigger another rebuild attempt.
+      prodLog.info('Clearing existing metadata index chunks from storage...')
       const existingFields = await this.getPersistedFieldList()
 
       if (existingFields.length > 0) {
@@ -3083,14 +3110,7 @@ export class MetadataIndexManager {
           await this.deleteFieldChunks(field)
         }
 
-        // Delete field registry (will be recreated on flush)
-        try {
-          await this.storage.saveMetadata('__metadata_field_registry__', null as any)
-        } catch (error) {
-          prodLog.debug('Could not delete field registry:', error)
-        }
-
-        prodLog.info(`✅ Cleared ${existingFields.length} field indexes from storage`)
+        prodLog.info(`Cleared ${existingFields.length} field indexes from storage`)
       }
 
       // Clear EntityIdMapper to start fresh