soulcraft/brainy
2
0
Fork 0
Code Issues 1 Pull requests Projects Releases 335 Packages Wiki Activity Actions

perf(8.0): negation/absence where-operators via roaring-bitmap difference

Browse source 
The JS metadata index served `ne`, `exists:false`, and `missing:true` by
materializing the ENTIRE id universe as an array of UUID strings, building a Set
of the excluded ids, and filtering — O(N) heap and two full passes on the
open-core path, on the most common shape (`deleted !== true`).

Compute the complement as a roaring-bitmap difference over the int-id universe
instead (`complementIds(excludeInts)` = universe \ exclude), converting only the
result back to UUIDs. The exclude set is small (the matching value), so this
avoids the full-corpus string materialization entirely. Behaviour is unchanged,
including the load-bearing soft-delete semantic that `field !== value` includes
entities with no such field. New focused tests pin `ne`/`exists:false`/
`missing:true`/`exists:true` result sets and that the complement reflects deletes.

Full gate green: unit 1520, integration 607.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
David Snelling 2026-06-29 11:19:18 -07:00
parent 72df5572b7
commit 5f974abc8a
2 changed files with 90 additions and 23 deletions
Download patch file Download diff file Expand all files Collapse all files

53
src/utils/metadataIndex.ts
View file

@ -1888,19 +1888,17 @@ export class MetadataIndexManager implements MetadataIndexProvider {
// Canonical: 'ne' | Alias: 'notEquals'
case 'notEquals': // Alias for 'ne'
case 'ne': {
// For notEquals, we need all IDs EXCEPT those matching the value
// This is especially important for soft delete: deleted !== true
// should include items without a deleted field
// Use EntityIdMapper universe (in-memory) instead of getAllIds() storage scan
const allKnownIds = this.idMapper.intsIterableToUuids(this.idMapper.getAllIntIds())
// Then get IDs that match the value we want to exclude
const excludeIds = await this.getIds(field, operand)
const excludeSet = new Set(excludeIds)
// Return all IDs except those to exclude
fieldResults = allKnownIds.filter(id => !excludeSet.has(id))
// All ids EXCEPT those matching the value. Important for soft delete:
// `deleted !== true` must include items WITHOUT a deleted field. The
// excluded set is typically small (the matching value); compute the
// complement as a bitmap difference over the int-id universe rather
// than materializing the whole corpus as UUID strings to filter it.
const excludeInts: number[] = []
for (const uuid of await this.getIds(field, operand)) {
const intId = this.idMapper.getInt(uuid)
if (intId !== undefined) excludeInts.push(intId)
}
fieldResults = this.complementIds(excludeInts)
break
}
@ -1972,11 +1970,8 @@ export class MetadataIndexManager implements MetadataIndexProvider {
// exists: true — entities that HAVE this field
fieldResults = this.idMapper.intsIterableToUuids(existsBitmap)
} else {
// exists: false — entities that DON'T have this field
const allKnownIds = this.idMapper.intsIterableToUuids(this.idMapper.getAllIntIds())
const existsUuids = this.idMapper.intsIterableToUuids(existsBitmap)
const existsSet = new Set(existsUuids)
fieldResults = allKnownIds.filter(id => !existsSet.has(id))
// exists: false — entities that DON'T have this field (universe \ has-field)
fieldResults = this.complementIds(existsBitmap)
}
break
}
@ -1989,11 +1984,8 @@ export class MetadataIndexManager implements MetadataIndexProvider {
: await this.getExistsBitmapLegacy(field)
if (operand) {
// missing: true — entities that DON'T have this field (same as exists: false)
const allKnownIds = this.idMapper.intsIterableToUuids(this.idMapper.getAllIntIds())
const existsUuids = this.idMapper.intsIterableToUuids(missingBitmap)
const existsSet = new Set(existsUuids)
fieldResults = allKnownIds.filter(id => !existsSet.has(id))
// missing: true — entities that DON'T have this field (universe \ has-field)
fieldResults = this.complementIds(missingBitmap)
} else {
// missing: false — entities that HAVE this field (same as exists: true)
fieldResults = this.idMapper.intsIterableToUuids(missingBitmap)
@ -2066,6 +2058,21 @@ export class MetadataIndexManager implements MetadataIndexProvider {
* @returns Roaring bitmap of entity int IDs (or iterable for compatibility)
* @private
*/
/**
* All ids EXCEPT the excluded int-id set, computed as a roaring-bitmap difference
* over the int-id universe. Used by the negation/absence operators (`ne`,
* `exists:false`, `missing:true`) so they don't first materialize the ENTIRE
* corpus as an array of UUID strings (plus a Set, plus an O(N) filter pass) just
* to remove a small subset — only the final result is converted back to UUIDs.
*/
private complementIds(excludeInts: Iterable<number>): string[] {
const universe = new RoaringBitmap32()
for (const intId of this.idMapper.getAllIntIds()) universe.add(intId)
const exclude = new RoaringBitmap32()
for (const intId of excludeInts) exclude.add(intId)
return this.idMapper.intsIterableToUuids(RoaringBitmap32.andNot(universe, exclude))
}
private async getExistsBitmapLegacy(field: string): Promise<Iterable<number>> {
const allIntIds = new Set<number>()
const sparseIndex = await this.loadSparseIndex(field)