fix: metadata index not cleaned up after delete/deleteMany

Three bugs caused deleted entities to persist in the metadata index:

1. idMapper never cleaned up — EntityIdMapper accumulated UUID→int mappings
   permanently. idMapper.getAllIntIds() is used as the universe for ne and
   exists:false operators, so deleted entities returned in those queries
   indefinitely. Fix: removeFromIndex() now calls idMapper.remove(id) and
   idMapper.flush() after all bitmap operations complete (must be last because
   removeFromChunk() reads idMapper.getInt(id) internally).

2. Optional fields indexed as __NULL__ but never unindexed — entityForIndexing
   in add() included confidence, weight, and createdBy as explicit keys even
   when undefined. Object.entries() preserves undefined-valued keys so
   extractIndexableFields() indexed them as '__NULL__' bitmap entries.
   storageMetadata omitted those keys via conditional spreading, so
   removeFromIndex() passed a structure without those keys and never cleaned
   them up. Fix: entityForIndexing now uses conditional spreading for
   confidence, weight, and createdBy matching storageMetadata exactly.

3. result.successful updated before transaction commits — deleteMany() pushed
   ids to result.successful inside the transaction builder, before
   transaction.execute() ran. A rollback would leave result.successful
   containing ids that were never actually deleted. Fix: queued ids are held
   in a local chunkQueued array and moved to result.successful only after
   executeTransaction() resolves without throwing.

Adds regression test suite (14 tests) covering delete() and deleteMany() for
type-index cleanup, ne operator, exists:false operator, optional-field indexing,
and partial deletion correctness.

Reported by wickworks team.

src/brainy.ts

@@ -791,19 +791,23 @@ export class Brainy<T = any> implements BrainyInterface<T> {
       }
 
       // Build entity structure for indexing (NEW - with top-level fields)
+      // Optional fields must use conditional spreading to match storageMetadata exactly.
+      // If undefined values are included as explicit keys, extractIndexableFields indexes
+      // them as '__NULL__' entries that removeFromIndex can never clean up (storageMetadata
+      // omits those keys entirely via conditional spreading, so the fields don't match).
       const entityForIndexing = {
         id,
         vector,
         connections: new Map(),
         level: 0,
         type: params.type,
-        confidence: params.confidence,
-        weight: params.weight,
+        ...(params.confidence !== undefined && { confidence: params.confidence }),
+        ...(params.weight !== undefined && { weight: params.weight }),
         createdAt: Date.now(),
         updatedAt: Date.now(),
         service: params.service,
         data: params.data,
-        createdBy: params.createdBy,
+        ...(params.createdBy && { createdBy: params.createdBy }),
         // Only custom fields in metadata
         metadata: params.metadata || {}
       }
@@ -2780,6 +2784,13 @@ export class Brainy<T = any> implements BrainyInterface<T> {
     for (let i = 0; i < idsToDelete.length; i += chunkSize) {
       const chunk = idsToDelete.slice(i, i + chunkSize)
 
+      // Track IDs queued during builder phase separately from confirmed deletions.
+      // result.successful must only be updated AFTER the transaction commits — pushing
+      // inside the builder runs before transaction.execute(), so a rollback would leave
+      // successfully-queued IDs incorrectly listed as deleted.
+      const chunkQueued: string[] = []
+      const chunkBuilderFailed: Array<{ item: string; error: string }> = []
+
       try {
         // Process chunk in single transaction for atomic deletion
         await this.transactionManager.executeTransaction(async (tx) => {
@@ -2818,9 +2829,9 @@ export class Brainy<T = any> implements BrainyInterface<T> {
                 )
               }
 
-              result.successful.push(id)
+              chunkQueued.push(id)
             } catch (error) {
-              result.failed.push({
+              chunkBuilderFailed.push({
                 item: id,
                 error: (error as Error).message
               })
@@ -2830,15 +2841,18 @@ export class Brainy<T = any> implements BrainyInterface<T> {
             }
           }
         })
+
+        // Transaction committed — queued IDs were actually deleted
+        result.successful.push(...chunkQueued)
+        result.failed.push(...chunkBuilderFailed)
       } catch (error) {
-        // Transaction failed - mark remaining entities in chunk as failed if not already recorded
-        for (const id of chunk) {
-          if (!result.successful.includes(id) && !result.failed.find(f => f.item === id)) {
-            result.failed.push({
-              item: id,
-              error: (error as Error).message
-            })
-          }
+        // Transaction failed/rolled back — queued IDs were NOT deleted
+        result.failed.push(...chunkBuilderFailed)
+        for (const id of chunkQueued) {
+          result.failed.push({
+            item: id,
+            error: (error as Error).message
+          })
         }
 
         // Stop processing if continueOnError is false

src/utils/metadataIndex.ts

@@ -1667,6 +1667,12 @@ export class MetadataIndexManager {
 
       // Flush all dirty chunks and sparse indices accumulated during remove
       await this.flushDirtyMetadata()
+
+      // Clean up ID mapper — must happen AFTER bitmap removal since removeFromChunk
+      // calls idMapper.getInt(id) internally. Skipping this leaves deleted IDs in the
+      // idMapper universe, causing ne/exists:false queries to return deleted entities.
+      this.idMapper.remove(id)
+      await this.idMapper.flush()
     } else {
       // Remove from all indexes (slower, requires scanning all field indexes)
       // This should be rare - prefer providing metadata when removing
@@ -1697,6 +1703,10 @@ export class MetadataIndexManager {
 
       // Flush all dirty chunks and sparse indices accumulated during scan-remove
       await this.flushDirtyMetadata()
+
+      // Clean up ID mapper — must happen AFTER bitmap removal (same reason as fast path above)
+      this.idMapper.remove(id)
+      await this.idMapper.flush()
     }
   }