perf(8.0): negation/absence where-operators via roaring-bitmap difference

The JS metadata index served `ne`, `exists:false`, and `missing:true` by
materializing the ENTIRE id universe as an array of UUID strings, building a Set
of the excluded ids, and filtering — O(N) heap and two full passes on the
open-core path, on the most common shape (`deleted !== true`).

Compute the complement as a roaring-bitmap difference over the int-id universe
instead (`complementIds(excludeInts)` = universe \ exclude), converting only the
result back to UUIDs. The exclude set is small (the matching value), so this
avoids the full-corpus string materialization entirely. Behaviour is unchanged,
including the load-bearing soft-delete semantic that `field !== value` includes
entities with no such field. New focused tests pin `ne`/`exists:false`/
`missing:true`/`exists:true` result sets and that the complement reflects deletes.

Full gate green: unit 1520, integration 607.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

src/utils/metadataIndex.ts

@@ -1888,19 +1888,17 @@ export class MetadataIndexManager implements MetadataIndexProvider {
             // Canonical: 'ne' | Alias: 'notEquals'
             case 'notEquals':  // Alias for 'ne'
             case 'ne': {
-              // For notEquals, we need all IDs EXCEPT those matching the value
+              // All ids EXCEPT those matching the value. Important for soft delete:
-              // This is especially important for soft delete: deleted !== true
+              // `deleted !== true` must include items WITHOUT a deleted field. The
-              // should include items without a deleted field
+              // excluded set is typically small (the matching value); compute the
-
+              // complement as a bitmap difference over the int-id universe rather
-              // Use EntityIdMapper universe (in-memory) instead of getAllIds() storage scan
+              // than materializing the whole corpus as UUID strings to filter it.
-              const allKnownIds = this.idMapper.intsIterableToUuids(this.idMapper.getAllIntIds())
+              const excludeInts: number[] = []
-
+              for (const uuid of await this.getIds(field, operand)) {
-              // Then get IDs that match the value we want to exclude
+                const intId = this.idMapper.getInt(uuid)
-              const excludeIds = await this.getIds(field, operand)
+                if (intId !== undefined) excludeInts.push(intId)
-              const excludeSet = new Set(excludeIds)
+              }
-
+              fieldResults = this.complementIds(excludeInts)
-              // Return all IDs except those to exclude
-              fieldResults = allKnownIds.filter(id => !excludeSet.has(id))
               break
             }
 
@@ -1972,11 +1970,8 @@ export class MetadataIndexManager implements MetadataIndexProvider {
                 // exists: true — entities that HAVE this field
                 fieldResults = this.idMapper.intsIterableToUuids(existsBitmap)
               } else {
-                // exists: false — entities that DON'T have this field
+                // exists: false — entities that DON'T have this field (universe \ has-field)
-                const allKnownIds = this.idMapper.intsIterableToUuids(this.idMapper.getAllIntIds())
+                fieldResults = this.complementIds(existsBitmap)
-                const existsUuids = this.idMapper.intsIterableToUuids(existsBitmap)
-                const existsSet = new Set(existsUuids)
-                fieldResults = allKnownIds.filter(id => !existsSet.has(id))
               }
               break
             }
@@ -1989,11 +1984,8 @@ export class MetadataIndexManager implements MetadataIndexProvider {
                 : await this.getExistsBitmapLegacy(field)
 
               if (operand) {
-                // missing: true — entities that DON'T have this field (same as exists: false)
+                // missing: true — entities that DON'T have this field (universe \ has-field)
-                const allKnownIds = this.idMapper.intsIterableToUuids(this.idMapper.getAllIntIds())
+                fieldResults = this.complementIds(missingBitmap)
-                const existsUuids = this.idMapper.intsIterableToUuids(missingBitmap)
-                const existsSet = new Set(existsUuids)
-                fieldResults = allKnownIds.filter(id => !existsSet.has(id))
               } else {
                 // missing: false — entities that HAVE this field (same as exists: true)
                 fieldResults = this.idMapper.intsIterableToUuids(missingBitmap)
@@ -2066,6 +2058,21 @@ export class MetadataIndexManager implements MetadataIndexProvider {
    * @returns Roaring bitmap of entity int IDs (or iterable for compatibility)
    * @private
    */
+  /**
+   * All ids EXCEPT the excluded int-id set, computed as a roaring-bitmap difference
+   * over the int-id universe. Used by the negation/absence operators (`ne`,
+   * `exists:false`, `missing:true`) so they don't first materialize the ENTIRE
+   * corpus as an array of UUID strings (plus a Set, plus an O(N) filter pass) just
+   * to remove a small subset — only the final result is converted back to UUIDs.
+   */
+  private complementIds(excludeInts: Iterable<number>): string[] {
+    const universe = new RoaringBitmap32()
+    for (const intId of this.idMapper.getAllIntIds()) universe.add(intId)
+    const exclude = new RoaringBitmap32()
+    for (const intId of excludeInts) exclude.add(intId)
+    return this.idMapper.intsIterableToUuids(RoaringBitmap32.andNot(universe, exclude))
+  }
+
   private async getExistsBitmapLegacy(field: string): Promise<Iterable<number>> {
     const allIntIds = new Set<number>()
     const sparseIndex = await this.loadSparseIndex(field)

tests/unit/brainy/find-complement-operators.test.ts

@@ -0,0 +1,60 @@
+/**
+ * @module tests/unit/brainy/find-complement-operators
+ * @description The negation/absence where-operators (`ne`, `exists:false`,
+ * `missing:true`) are served as a roaring-bitmap difference over the int-id
+ * universe (rather than materializing the whole corpus as UUID strings to
+ * filter). These tests pin the exact result set — including the load-bearing
+ * soft-delete semantic: `field !== value` MUST include entities that have no
+ * such field at all.
+ */
+import { describe, it, expect, beforeEach } from 'vitest'
+import { Brainy } from '../../../src/brainy'
+import { NounType } from '../../../src/types/graphTypes'
+
+describe('find() complement operators (ne / exists:false / missing:true)', () => {
+  let brain: Brainy<any>
+  // Three entities WITH a `status` field, two WITHOUT it.
+  const ids: Record<string, string> = {}
+
+  beforeEach(async () => {
+    brain = new Brainy({ requireSubtype: false, storage: { type: 'memory' } })
+    await brain.init()
+    ids.active1 = await brain.add({ data: 'a1', type: NounType.Thing, metadata: { status: 'active' } })
+    ids.active2 = await brain.add({ data: 'a2', type: NounType.Thing, metadata: { status: 'active' } })
+    ids.closed = await brain.add({ data: 'c', type: NounType.Thing, metadata: { status: 'closed' } })
+    ids.noField1 = await brain.add({ data: 'n1', type: NounType.Thing, metadata: { other: 1 } })
+    ids.noField2 = await brain.add({ data: 'n2', type: NounType.Thing, metadata: { other: 2 } })
+  })
+
+  it('ne returns everything except the matching value — INCLUDING entities without the field', async () => {
+    const rows = await brain.find({ where: { status: { ne: 'active' } }, limit: 100 })
+    const got = new Set(rows.map((r) => r.id))
+    // closed (status≠active) + the two without a status field at all.
+    expect(got).toEqual(new Set([ids.closed, ids.noField1, ids.noField2]))
+    expect(got.has(ids.active1)).toBe(false)
+    expect(got.has(ids.active2)).toBe(false)
+  })
+
+  it('exists:false returns exactly the entities without the field', async () => {
+    const rows = await brain.find({ where: { status: { exists: false } }, limit: 100 })
+    expect(new Set(rows.map((r) => r.id))).toEqual(new Set([ids.noField1, ids.noField2]))
+  })
+
+  it('missing:true matches exists:false (entities without the field)', async () => {
+    const rows = await brain.find({ where: { status: { missing: true } }, limit: 100 })
+    expect(new Set(rows.map((r) => r.id))).toEqual(new Set([ids.noField1, ids.noField2]))
+  })
+
+  it('exists:true returns exactly the entities with the field', async () => {
+    const rows = await brain.find({ where: { status: { exists: true } }, limit: 100 })
+    expect(new Set(rows.map((r) => r.id))).toEqual(new Set([ids.active1, ids.active2, ids.closed]))
+  })
+
+  it('the complement reflects deletes (a removed entity drops out of ne / exists:false)', async () => {
+    await brain.remove(ids.noField1)
+    const ne = await brain.find({ where: { status: { ne: 'active' } }, limit: 100 })
+    expect(new Set(ne.map((r) => r.id))).toEqual(new Set([ids.closed, ids.noField2]))
+    const absent = await brain.find({ where: { status: { exists: false } }, limit: 100 })
+    expect(new Set(absent.map((r) => r.id))).toEqual(new Set([ids.noField2]))
+  })
+})