From 5f974abc8a7f02a1b2d63f3f3d91c6b0533642c0 Mon Sep 17 00:00:00 2001 From: David Snelling Date: Mon, 29 Jun 2026 11:19:18 -0700 Subject: [PATCH] perf(8.0): negation/absence where-operators via roaring-bitmap difference MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The JS metadata index served `ne`, `exists:false`, and `missing:true` by materializing the ENTIRE id universe as an array of UUID strings, building a Set of the excluded ids, and filtering — O(N) heap and two full passes on the open-core path, on the most common shape (`deleted !== true`). Compute the complement as a roaring-bitmap difference over the int-id universe instead (`complementIds(excludeInts)` = universe \ exclude), converting only the result back to UUIDs. The exclude set is small (the matching value), so this avoids the full-corpus string materialization entirely. Behaviour is unchanged, including the load-bearing soft-delete semantic that `field !== value` includes entities with no such field. New focused tests pin `ne`/`exists:false`/ `missing:true`/`exists:true` result sets and that the complement reflects deletes. Full gate green: unit 1520, integration 607. Co-Authored-By: Claude Opus 4.8 --- src/utils/metadataIndex.ts | 53 +++++++++------- .../brainy/find-complement-operators.test.ts | 60 +++++++++++++++++++ 2 files changed, 90 insertions(+), 23 deletions(-) create mode 100644 tests/unit/brainy/find-complement-operators.test.ts diff --git a/src/utils/metadataIndex.ts b/src/utils/metadataIndex.ts index 676c3b2e..1bae541a 100644 --- a/src/utils/metadataIndex.ts +++ b/src/utils/metadataIndex.ts @@ -1888,19 +1888,17 @@ export class MetadataIndexManager implements MetadataIndexProvider { // Canonical: 'ne' | Alias: 'notEquals' case 'notEquals': // Alias for 'ne' case 'ne': { - // For notEquals, we need all IDs EXCEPT those matching the value - // This is especially important for soft delete: deleted !== true - // should include items without a deleted field - - // Use EntityIdMapper universe (in-memory) instead of getAllIds() storage scan - const allKnownIds = this.idMapper.intsIterableToUuids(this.idMapper.getAllIntIds()) - - // Then get IDs that match the value we want to exclude - const excludeIds = await this.getIds(field, operand) - const excludeSet = new Set(excludeIds) - - // Return all IDs except those to exclude - fieldResults = allKnownIds.filter(id => !excludeSet.has(id)) + // All ids EXCEPT those matching the value. Important for soft delete: + // `deleted !== true` must include items WITHOUT a deleted field. The + // excluded set is typically small (the matching value); compute the + // complement as a bitmap difference over the int-id universe rather + // than materializing the whole corpus as UUID strings to filter it. + const excludeInts: number[] = [] + for (const uuid of await this.getIds(field, operand)) { + const intId = this.idMapper.getInt(uuid) + if (intId !== undefined) excludeInts.push(intId) + } + fieldResults = this.complementIds(excludeInts) break } @@ -1972,11 +1970,8 @@ export class MetadataIndexManager implements MetadataIndexProvider { // exists: true — entities that HAVE this field fieldResults = this.idMapper.intsIterableToUuids(existsBitmap) } else { - // exists: false — entities that DON'T have this field - const allKnownIds = this.idMapper.intsIterableToUuids(this.idMapper.getAllIntIds()) - const existsUuids = this.idMapper.intsIterableToUuids(existsBitmap) - const existsSet = new Set(existsUuids) - fieldResults = allKnownIds.filter(id => !existsSet.has(id)) + // exists: false — entities that DON'T have this field (universe \ has-field) + fieldResults = this.complementIds(existsBitmap) } break } @@ -1989,11 +1984,8 @@ export class MetadataIndexManager implements MetadataIndexProvider { : await this.getExistsBitmapLegacy(field) if (operand) { - // missing: true — entities that DON'T have this field (same as exists: false) - const allKnownIds = this.idMapper.intsIterableToUuids(this.idMapper.getAllIntIds()) - const existsUuids = this.idMapper.intsIterableToUuids(missingBitmap) - const existsSet = new Set(existsUuids) - fieldResults = allKnownIds.filter(id => !existsSet.has(id)) + // missing: true — entities that DON'T have this field (universe \ has-field) + fieldResults = this.complementIds(missingBitmap) } else { // missing: false — entities that HAVE this field (same as exists: true) fieldResults = this.idMapper.intsIterableToUuids(missingBitmap) @@ -2066,6 +2058,21 @@ export class MetadataIndexManager implements MetadataIndexProvider { * @returns Roaring bitmap of entity int IDs (or iterable for compatibility) * @private */ + /** + * All ids EXCEPT the excluded int-id set, computed as a roaring-bitmap difference + * over the int-id universe. Used by the negation/absence operators (`ne`, + * `exists:false`, `missing:true`) so they don't first materialize the ENTIRE + * corpus as an array of UUID strings (plus a Set, plus an O(N) filter pass) just + * to remove a small subset — only the final result is converted back to UUIDs. + */ + private complementIds(excludeInts: Iterable): string[] { + const universe = new RoaringBitmap32() + for (const intId of this.idMapper.getAllIntIds()) universe.add(intId) + const exclude = new RoaringBitmap32() + for (const intId of excludeInts) exclude.add(intId) + return this.idMapper.intsIterableToUuids(RoaringBitmap32.andNot(universe, exclude)) + } + private async getExistsBitmapLegacy(field: string): Promise> { const allIntIds = new Set() const sparseIndex = await this.loadSparseIndex(field) diff --git a/tests/unit/brainy/find-complement-operators.test.ts b/tests/unit/brainy/find-complement-operators.test.ts new file mode 100644 index 00000000..76fbb017 --- /dev/null +++ b/tests/unit/brainy/find-complement-operators.test.ts @@ -0,0 +1,60 @@ +/** + * @module tests/unit/brainy/find-complement-operators + * @description The negation/absence where-operators (`ne`, `exists:false`, + * `missing:true`) are served as a roaring-bitmap difference over the int-id + * universe (rather than materializing the whole corpus as UUID strings to + * filter). These tests pin the exact result set — including the load-bearing + * soft-delete semantic: `field !== value` MUST include entities that have no + * such field at all. + */ +import { describe, it, expect, beforeEach } from 'vitest' +import { Brainy } from '../../../src/brainy' +import { NounType } from '../../../src/types/graphTypes' + +describe('find() complement operators (ne / exists:false / missing:true)', () => { + let brain: Brainy + // Three entities WITH a `status` field, two WITHOUT it. + const ids: Record = {} + + beforeEach(async () => { + brain = new Brainy({ requireSubtype: false, storage: { type: 'memory' } }) + await brain.init() + ids.active1 = await brain.add({ data: 'a1', type: NounType.Thing, metadata: { status: 'active' } }) + ids.active2 = await brain.add({ data: 'a2', type: NounType.Thing, metadata: { status: 'active' } }) + ids.closed = await brain.add({ data: 'c', type: NounType.Thing, metadata: { status: 'closed' } }) + ids.noField1 = await brain.add({ data: 'n1', type: NounType.Thing, metadata: { other: 1 } }) + ids.noField2 = await brain.add({ data: 'n2', type: NounType.Thing, metadata: { other: 2 } }) + }) + + it('ne returns everything except the matching value — INCLUDING entities without the field', async () => { + const rows = await brain.find({ where: { status: { ne: 'active' } }, limit: 100 }) + const got = new Set(rows.map((r) => r.id)) + // closed (status≠active) + the two without a status field at all. + expect(got).toEqual(new Set([ids.closed, ids.noField1, ids.noField2])) + expect(got.has(ids.active1)).toBe(false) + expect(got.has(ids.active2)).toBe(false) + }) + + it('exists:false returns exactly the entities without the field', async () => { + const rows = await brain.find({ where: { status: { exists: false } }, limit: 100 }) + expect(new Set(rows.map((r) => r.id))).toEqual(new Set([ids.noField1, ids.noField2])) + }) + + it('missing:true matches exists:false (entities without the field)', async () => { + const rows = await brain.find({ where: { status: { missing: true } }, limit: 100 }) + expect(new Set(rows.map((r) => r.id))).toEqual(new Set([ids.noField1, ids.noField2])) + }) + + it('exists:true returns exactly the entities with the field', async () => { + const rows = await brain.find({ where: { status: { exists: true } }, limit: 100 }) + expect(new Set(rows.map((r) => r.id))).toEqual(new Set([ids.active1, ids.active2, ids.closed])) + }) + + it('the complement reflects deletes (a removed entity drops out of ne / exists:false)', async () => { + await brain.remove(ids.noField1) + const ne = await brain.find({ where: { status: { ne: 'active' } }, limit: 100 }) + expect(new Set(ne.map((r) => r.id))).toEqual(new Set([ids.closed, ids.noField2])) + const absent = await brain.find({ where: { status: { exists: false } }, limit: 100 }) + expect(new Set(absent.map((r) => r.id))).toEqual(new Set([ids.noField2])) + }) +})