From b0f72ef36f6c48bd6abe282525e044bfa2175acb Mon Sep 17 00:00:00 2001
From: David Snelling <david@soulcraft.com>
Date: Thu, 13 Nov 2025 11:06:59 -0800
Subject: [PATCH] fix: implement exists: false and missing operators in
 MetadataIndexManager

Fixes critical bug where excludeVFS: true excluded ALL entities including
non-VFS entities created with brain.add(). The MetadataIndexManager's
getIdsForFilter() only implemented exists: true, missing the else clause
for exists: false.

Changes:
- Added exists: false implementation (returns all IDs minus field IDs)
- Added missing operator (for API consistency with metadataFilter.ts)
- Both operators now match in-memory metadataFilter.ts behavior

Root Cause:
- brainy.ts sets filter.vfsType = { exists: false } for excludeVFS
- metadataIndex.ts case 'exists' only checked if (operand) with no else
- Missing else clause caused empty fieldResults, returning []

Impact:
- Fixes excludeVFS feature (used by Workshop team)
- Fixes any queries using exists: false operator
- Adds missing operator support for completeness

Testing:
- Build: passing
- Tests: passing
- Manual test: verified excludeVFS correctly excludes VFS entities only

Reported by: Workshop Team (Soulcraft)
Issue: BRAINY_V5_7_7_EXCLUDEVFS_BUG.md
---
 src/utils/metadataIndex.ts | 78 +++++++++++++++++++++++++++++++++++++-
 1 file changed, 77 insertions(+), 1 deletion(-)

diff --git a/src/utils/metadataIndex.ts b/src/utils/metadataIndex.ts
index 8586b2f9..e4404f78 100644
--- a/src/utils/metadataIndex.ts
+++ b/src/utils/metadataIndex.ts
@@ -1618,7 +1618,8 @@ export class MetadataIndexManager {
             // exists: boolean - check if field exists (any value)
             case 'exists':
               if (operand) {
-                // Get all IDs that have this field (any value) from chunked sparse index with roaring bitmaps (v3.43.0)
+                // exists: true - Get all IDs that have this field (any value)
+                // v3.43.0: From chunked sparse index with roaring bitmaps
                 // v3.44.1: Now fully lazy-loaded via UnifiedCache (no local sparseIndices Map)
                 const allIntIds = new Set<number>()
 
@@ -1640,6 +1641,81 @@ export class MetadataIndexManager {
                 }
 
                 // Convert integer IDs back to UUIDs
+                fieldResults = this.idMapper.intsIterableToUuids(allIntIds)
+              } else {
+                // exists: false - Get all IDs that DON'T have this field
+                // v5.7.9: Fixed excludeVFS bug (was returning empty array)
+                const allItemIds = await this.getAllIds()
+                const existsIntIds = new Set<number>()
+
+                // Get IDs that HAVE this field
+                const sparseIndex = await this.loadSparseIndex(field)
+                if (sparseIndex) {
+                  for (const chunkId of sparseIndex.getAllChunkIds()) {
+                    const chunk = await this.chunkManager.loadChunk(field, chunkId)
+                    if (chunk) {
+                      for (const bitmap of chunk.entries.values()) {
+                        for (const intId of bitmap) {
+                          existsIntIds.add(intId)
+                        }
+                      }
+                    }
+                  }
+                }
+
+                // Convert to UUIDs and subtract from all IDs
+                const existsUuids = this.idMapper.intsIterableToUuids(existsIntIds)
+                const existsSet = new Set(existsUuids)
+                fieldResults = allItemIds.filter(id => !existsSet.has(id))
+              }
+              break
+
+            // ===== MISSING OPERATOR =====
+            // missing: boolean - equivalent to exists: !boolean (for compatibility with metadataFilter.ts)
+            case 'missing':
+              // missing: true is equivalent to exists: false
+              // missing: false is equivalent to exists: true
+              // v5.7.9: Added for API consistency with in-memory metadataFilter
+              if (operand) {
+                // missing: true - field does NOT exist (same as exists: false)
+                const allItemIds = await this.getAllIds()
+                const existsIntIds = new Set<number>()
+
+                const sparseIndex = await this.loadSparseIndex(field)
+                if (sparseIndex) {
+                  for (const chunkId of sparseIndex.getAllChunkIds()) {
+                    const chunk = await this.chunkManager.loadChunk(field, chunkId)
+                    if (chunk) {
+                      for (const bitmap of chunk.entries.values()) {
+                        for (const intId of bitmap) {
+                          existsIntIds.add(intId)
+                        }
+                      }
+                    }
+                  }
+                }
+
+                const existsUuids = this.idMapper.intsIterableToUuids(existsIntIds)
+                const existsSet = new Set(existsUuids)
+                fieldResults = allItemIds.filter(id => !existsSet.has(id))
+              } else {
+                // missing: false - field DOES exist (same as exists: true)
+                const allIntIds = new Set<number>()
+
+                const sparseIndex = await this.loadSparseIndex(field)
+                if (sparseIndex) {
+                  for (const chunkId of sparseIndex.getAllChunkIds()) {
+                    const chunk = await this.chunkManager.loadChunk(field, chunkId)
+                    if (chunk) {
+                      for (const bitmap of chunk.entries.values()) {
+                        for (const intId of bitmap) {
+                          allIntIds.add(intId)
+                        }
+                      }
+                    }
+                  }
+                }
+
                 fieldResults = this.idMapper.intsIterableToUuids(allIntIds)
               }
               break