From dd325f2f94d725d8fe085df34fd2febbd756b113 Mon Sep 17 00:00:00 2001 From: David Snelling Date: Tue, 23 Jun 2026 13:18:34 -0700 Subject: [PATCH] feat(8.0): vector allowedIds predicate-pushdown into find() (#46) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Filtered semantic search now keeps its recall. A find({ query, where }) that pairs vector search with a metadata filter restricts the HNSW beam walk to the matching candidates INSIDE the traversal (walk-all, collect-allowed) rather than filtering the top-k afterward — so a query whose nearest vectors are all filtered out still returns the best matches that DO pass the filter, instead of empty. - JS HNSW search() honors the 8.0 `allowedIds: OpaqueIdSet | ReadonlySet` contract param (ANDs with candidateIds; ignores the opaque Buffer form it can't decode — only a native provider consumes that). - MetadataIndexProvider gains an OPTIONAL `getIdSetForFilter(filter): OpaqueIdSet` producer — the native (cor) metadata index returns its roaring filter result as a serialized buffer; the JS index does not implement it. - find() forwards the matched universe to the vector walk: the opaque buffer (zero id materialization) when the native producer is present, alongside the materialized visibility-precise candidateIds for the JS index. Visibility stays correct via the existing post-search hard filter. Tested: JS recall/restriction/AND/opaque-ignore on the index directly, plus find() forwarding the opaque universe through to the beam walk via a stubbed native producer. Co-Authored-By: Claude Opus 4.8 --- RELEASES.md | 7 + src/brainy.ts | 50 +++++- src/hnsw/hnswIndex.ts | 45 ++++- src/plugin.ts | 16 ++ tests/unit/hnsw/allowed-ids-pushdown.test.ts | 176 +++++++++++++++++++ 5 files changed, 279 insertions(+), 15 deletions(-) create mode 100644 tests/unit/hnsw/allowed-ids-pushdown.test.ts diff --git a/RELEASES.md b/RELEASES.md index 983da445..63293704 100644 --- a/RELEASES.md +++ b/RELEASES.md @@ -66,6 +66,13 @@ no separate migration doc. **`8.0.0-rc.2` is live** — install with > engine when present, and fall back to pure-TS kernels (PageRank, connected components / Tarjan > SCC, BFS / Dijkstra) otherwise. Both paths return identical shapes and respect the default > visibility filter (opt in with `includeInternal` / `includeSystem`). +> - **Filtered vector search keeps its recall (`allowedIds` pushdown).** A `find({ query, where })` +> that combines semantic search with a metadata filter now restricts the vector walk to the +> matching candidates *inside* the search (walk-all, collect-allowed) instead of filtering the +> top-k afterward — so a query whose nearest vectors are all filtered out still returns the best +> matches that DO pass the filter, rather than coming back empty. No API change. With the native +> `@soulcraft/cor` 3.0 stack the matched universe is forwarded as an opaque roaring buffer with +> zero id materialization in TypeScript; the pure-JS path restricts the beam walk with a string set. ### Headline: Database as a Value diff --git a/src/brainy.ts b/src/brainy.ts index 1d1bf0e9..87fa5dc5 100644 --- a/src/brainy.ts +++ b/src/brainy.ts @@ -49,7 +49,9 @@ import type { Subgraph, RankOptions, CommunitiesOptions, - PathOptions + PathOptions, + MetadataIndexProvider, + OpaqueIdSet } from './plugin.js' import type { BrainyPlugin, @@ -4807,6 +4809,11 @@ export class Brainy implements BrainyInterface { // bitmap), JS HNSW converts to a Set-based filter function. let preResolvedMetadataIds: string[] | null = null let preResolvedFilter: any = null + // 8.0 #46 (CTX-PUSHDOWN-ALLOWEDIDS): the matched universe as an opaque roaring + // Buffer, forwarded straight into the vector beam walk with NO id materialization + // when the active metadata provider can produce one (native/cor). Absent on the + // JS path — there the materialized `candidateIds` restricts the walk instead. + let preResolvedAllowedIds: OpaqueIdSet | undefined if (params.where || params.type || params.subtype || params.service || params.excludeVFS) { preResolvedFilter = {} @@ -4854,6 +4861,19 @@ export class Brainy implements BrainyInterface { if (preResolvedMetadataIds.length === 0) { return [] } + + // 8.0 #46: when the active metadata provider exposes the opaque-set producer + // (native/cor — the JS index does not), forward the matched universe to the + // vector beam walk as an OpaqueIdSet (zero id materialization). The opaque set + // is the RAW filter universe (the set is opaque, so the visibility exclusion + // cannot be AND-ed into it in TS) — hidden tiers are instead dropped by the + // post-search visibility hard filter below, and the JS path's materialized + // `candidateIds` stays visibility-precise. Both forms are passed; the native + // provider consumes the opaque one, the JS index the string one. + const mip = this.metadataIndex as unknown as MetadataIndexProvider + if (typeof mip.getIdSetForFilter === 'function') { + preResolvedAllowedIds = await mip.getIdSetForFilter(preResolvedFilter) + } } // Zero-Config Hybrid Search @@ -4867,14 +4887,14 @@ export class Brainy implements BrainyInterface { } // Handle semantic-only query (user explicitly wants vector search) else if ((searchMode === 'semantic' || searchMode === 'vector') && (params.query || params.vector)) { - results = await this.executeVectorSearch(params, preResolvedMetadataIds ?? undefined) + results = await this.executeVectorSearch(params, preResolvedMetadataIds ?? undefined, preResolvedAllowedIds) } // Handle explicit hybrid or auto mode with query else if ((searchMode === 'auto' || searchMode === 'hybrid') && params.query && params.query.trim() !== '' && !params.vector) { // Zero-config hybrid: combine text + semantic search with RRF fusion const [textResults, semanticResults] = await Promise.all([ this.executeTextSearch(params.query, limit * 2), - this.executeVectorSearch(params, preResolvedMetadataIds ?? undefined) + this.executeVectorSearch(params, preResolvedMetadataIds ?? undefined, preResolvedAllowedIds) ]) // Use user-specified alpha or auto-detect based on query length @@ -4888,7 +4908,7 @@ export class Brainy implements BrainyInterface { } // Handle direct vector search (no query text) - no hybrid needed else if (params.vector && !params.query) { - results = await this.executeVectorSearch(params, preResolvedMetadataIds ?? undefined) + results = await this.executeVectorSearch(params, preResolvedMetadataIds ?? undefined, preResolvedAllowedIds) } // Handle proximity search else if (params.near) { @@ -11133,13 +11153,29 @@ export class Brainy implements BrainyInterface { * When provided, HNSW search is restricted to these candidates: * - NativeHNSWWrapper: uses native searchWithCandidates (Rust bitmap filtering) * - JS JsHnswVectorIndex: converts to filter function with O(1) Set lookups + * @param allowedIds Optional 8.0 #46 predicate-pushdown universe as an + * {@link OpaqueIdSet} (native/cor roaring Buffer) — forwarded straight into the + * beam walk with no id materialization. The native vector provider consumes it; + * the JS index ignores the opaque form and relies on `candidateIds`. */ - private async executeVectorSearch(params: FindParams, candidateIds?: string[]): Promise[]> { + private async executeVectorSearch( + params: FindParams, + candidateIds?: string[], + allowedIds?: OpaqueIdSet + ): Promise[]> { const vector = params.vector || (await this.embed(params.query!)) const limit = params.limit || 10 - // Build search options for metadata-first candidate filtering - const searchOptions = candidateIds ? { candidateIds } : undefined + // Build search options for metadata-first candidate filtering. Pass both forms + // when available: the native provider prefers the opaque `allowedIds` (zero + // crossing), the JS index uses the materialized `candidateIds`. + const searchOptions = + candidateIds || allowedIds + ? { + ...(candidateIds && { candidateIds }), + ...(allowedIds && { allowedIds }) + } + : undefined // HNSW search with optional metadata-first candidate filtering const searchResults: [string, number][] = await this.index.search(vector, limit * 2, undefined, searchOptions) diff --git a/src/hnsw/hnswIndex.ts b/src/hnsw/hnswIndex.ts index 757ece69..8a42f206 100644 --- a/src/hnsw/hnswIndex.ts +++ b/src/hnsw/hnswIndex.ts @@ -14,7 +14,7 @@ import { euclideanDistance, calculateDistancesBatch } from '../utils/index.js' import type { BaseStorage } from '../storage/baseStorage.js' import { getGlobalCache, UnifiedCache } from '../utils/unifiedCache.js' import { prodLog } from '../utils/logger.js' -import type { VectorIndexProvider } from '../plugin.js' +import type { VectorIndexProvider, OpaqueIdSet } from '../plugin.js' import { ConnectionsCodec, compressedConnectionsKey } from './connectionsCodec.js' // Default HNSW parameters @@ -656,23 +656,52 @@ export class JsHnswVectorIndex implements VectorIndexProvider { * @param queryVector Query vector * @param k Number of results to return * @param filter Optional filter function - * @param options Additional search options (`candidateIds` to restrict the - * search to a pre-filtered set; `rerank` is provider-only, ignored here) + * @param options Additional search options (`candidateIds` / `allowedIds` to + * restrict the search to a pre-filtered set; `rerank` is provider-only, ignored here) */ public async search( queryVector: Vector, k: number = 10, filter?: (id: string) => Promise, - options?: { rerank?: { multiplier: number }; candidateIds?: string[] } + options?: { + rerank?: { multiplier: number } + candidateIds?: string[] + allowedIds?: OpaqueIdSet | ReadonlySet + } ): Promise> { if (this.nouns.size === 0) { return [] } - // Metadata-first: convert candidateIds to filter function if no explicit filter - if (!filter && options?.candidateIds && options.candidateIds.length > 0) { - const candidateSet = new Set(options.candidateIds) - filter = async (id: string) => candidateSet.has(id) + // Metadata-first candidate restriction. Build a filter from the pre-resolved + // universe(s) when the caller didn't pass an explicit one. Both `candidateIds` + // (legacy string[]) and the 8.0 `allowedIds` predicate-pushdown param restrict + // the SAME way here — applied INSIDE the beam walk (searchLayer keeps every + // neighbor as a traversal candidate but only COLLECTS allowed ids), which is + // what recovers the filtered recall that post-filtering loses. An `allowedIds` + // that arrives as an OpaqueIdSet (a native/cor roaring Buffer) is opaque to the + // JS index — it cannot decode it — so it is ignored here; only the native + // provider consumes that form. A `ReadonlySet` is honored. When both + // `candidateIds` and a Set `allowedIds` are present they AND together. + if (!filter) { + const universes: ReadonlySet[] = [] + if (options?.candidateIds && options.candidateIds.length > 0) { + universes.push(new Set(options.candidateIds)) + } + const allowed = options?.allowedIds + if ( + allowed && + !(allowed instanceof Uint8Array) && + typeof (allowed as ReadonlySet).has === 'function' + ) { + universes.push(allowed as ReadonlySet) + } + if (universes.length === 1) { + const universe = universes[0] + filter = async (id: string) => universe.has(id) + } else if (universes.length > 1) { + filter = async (id: string) => universes.every((u) => u.has(id)) + } } // Check if query vector is defined diff --git a/src/plugin.ts b/src/plugin.ts index faf23921..ae0ed3f0 100644 --- a/src/plugin.ts +++ b/src/plugin.ts @@ -130,6 +130,22 @@ export interface MetadataIndexProvider { getIds(field: string, value: any): Promise getIdsForFilter(filter: any): Promise + /** + * @description OPTIONAL: resolve a `where` filter to its matching id universe as + * an {@link OpaqueIdSet} (a serialized roaring `Buffer`) WITHOUT materializing + * the ids in TypeScript — the producer half of the predicate-pushdown + * (`CTX-PUSHDOWN-ALLOWEDIDS`) and graph query→expand fusion. Brainy forwards the + * returned set opaquely to `VectorIndexProvider.search({ allowedIds })` and + * `GraphAccelerationProvider.traverse(seeds)`; it NEVER inspects it. A native + * (cor) metadata index returns its roaring filter result directly (zero + * crossing); the JS index does not implement this — Brainy falls back to + * {@link MetadataIndexProvider.getIdsForFilter} + a `ReadonlySet` when + * absent. Present ⟺ the native stack is the active provider, so the matching + * native vector/graph engines can decode the same envelope. + * @param filter - The same filter shape accepted by `getIdsForFilter`. + * @returns The matching id universe as an opaque set. + */ + getIdSetForFilter?(filter: any): Promise getIdsForTextQuery(query: string): Promise> getSortedIdsForFilter(filter: any, orderBy: string, order?: 'asc' | 'desc'): Promise getFilterValues(field: string): Promise diff --git a/tests/unit/hnsw/allowed-ids-pushdown.test.ts b/tests/unit/hnsw/allowed-ids-pushdown.test.ts new file mode 100644 index 00000000..33c67459 --- /dev/null +++ b/tests/unit/hnsw/allowed-ids-pushdown.test.ts @@ -0,0 +1,176 @@ +/** + * @module tests/unit/hnsw/allowed-ids-pushdown + * @description 8.0 #46 (CTX-PUSHDOWN-ALLOWEDIDS) — the vector-search predicate + * pushdown. Two layers: + * + * 1. The JS HNSW index honors `allowedIds: ReadonlySet` by restricting the + * beam walk to allowed nodes INSIDE the traversal (walk-all, collect-allowed) — + * the semantics that recover the filtered recall a naive top-k-then-filter loses. + * An `allowedIds` arriving as an `OpaqueIdSet` (a native/cor roaring Buffer) is + * opaque to the JS index and ignored — only a native provider decodes it. + * + * 2. `find()` forwards the matched universe to the vector beam walk as an + * `OpaqueIdSet` (zero id materialization) when the active metadata provider + * exposes the `getIdSetForFilter` producer — the native-stack zero-crossing path. + */ + +import { describe, it, expect, beforeEach, afterEach } from 'vitest' +import { JsHnswVectorIndex } from '../../../src/hnsw/hnswIndex.js' +import { euclideanDistance } from '../../../src/utils/index.js' +import { MemoryStorage } from '../../../src/storage/adapters/memoryStorage.js' +import { Brainy } from '../../../src/index.js' +import { NounType } from '../../../src/types/graphTypes.js' +import { createTestConfig } from '../../helpers/test-factory.js' + +const DIM = 8 + +/** Deterministic PRNG (mulberry32) so the synthetic graph is identical every run. */ +function seededRand(seed: number): () => number { + let s = seed >>> 0 + return () => { + s = (s + 0x6d2b79f5) | 0 + let t = Math.imul(s ^ (s >>> 15), 1 | s) + t = (t + Math.imul(t ^ (t >>> 7), 61 | t)) ^ t + return ((t ^ (t >>> 14)) >>> 0) / 4294967296 + } +} + +/** + * A vector at the requested distance-from-origin pointing in a deterministic RANDOM + * direction (so the HNSW graph is well-connected — diverse directions, not a + * degenerate axis cluster). The query is the origin, so distance-from-query equals + * `magnitude` exactly — letting the test place nodes in clean distance bands. + */ +function vecAt(magnitude: number, idx: number): number[] { + const rand = seededRand(idx + 1) + const dir = Array.from({ length: DIM }, () => rand() * 2 - 1) + const norm = Math.hypot(...dir) || 1 + return dir.map((x) => (x / norm) * magnitude) +} + +describe('#46 JS HNSW allowedIds pushdown (recall-preserving)', () => { + let index: JsHnswVectorIndex + const query = new Array(DIM).fill(0) + // Two clean distance bands: the disallowed nodes are STRICTLY closer to the query + // than the allowed targets. A naive top-k-then-filter returns only the disallowed + // (then drops them ⇒ empty); the pushdown must reach the allowed targets. `M` ≥ N−1 + // makes the small graph complete, so reachability is guaranteed and the test asserts + // the FILTER mechanism (walk-all, collect-allowed), not emergent HNSW connectivity. + const disallowed: string[] = [] + const targets: string[] = [] + + beforeEach(async () => { + index = new JsHnswVectorIndex( + { M: 16, efConstruction: 200, efSearch: 50, ml: 16 }, + euclideanDistance, + { useParallelization: false, storage: new MemoryStorage() } + ) + let n = 0 + for (let i = 0; i < 5; i++) { + const id = `disallowed-${i}` + disallowed.push(id) + await index.addItem({ id, vector: vecAt(0.3, n++) }) // closer band + } + for (let i = 0; i < 5; i++) { + const id = `target-${i}` + targets.push(id) + await index.addItem({ id, vector: vecAt(0.8, n++) }) // farther band + } + }) + + afterEach(() => { + disallowed.length = 0 + targets.length = 0 + }) + + it('without restriction, the top-k are all disallowed (so post-filtering would lose recall)', async () => { + const results = await index.search(query, 3) + const ids = results.map(([id]) => id) + // Every nearest neighbor is in the (closer) disallowed band → naive "filter after" = 0 hits. + expect(ids.length).toBe(3) + expect(ids.every((id) => disallowed.includes(id))).toBe(true) + }) + + it('with allowedIds, the walk reaches the allowed targets the naive path would miss', async () => { + const allowed = new Set(targets) + const results = await index.search(query, 3, undefined, { allowedIds: allowed }) + const ids = results.map(([id]) => id) + + expect(ids.length).toBe(3) + expect(ids.some((id) => disallowed.includes(id))).toBe(false) // no disallowed leaks through + expect(ids.every((id) => allowed.has(id))).toBe(true) // only allowed targets + }) + + it('ANDs allowedIds with candidateIds when both are given', async () => { + const results = await index.search(query, 5, undefined, { + candidateIds: [targets[0], targets[1], disallowed[0]], + allowedIds: new Set([targets[0], disallowed[0], disallowed[1]]) + }) + const ids = results.map(([id]) => id) + // Intersection = { target-0, disallowed-0 } — nothing outside it may appear. + expect(ids.every((id) => id === targets[0] || id === disallowed[0])).toBe(true) + expect(ids).toContain(targets[0]) + }) + + it('ignores an OpaqueIdSet (Uint8Array) — only a native provider can decode it', async () => { + const opaque = new Uint8Array([0xca, 0xfe, 0x01, 0x02]) + const results = await index.search(query, 3, undefined, { allowedIds: opaque }) + const ids = results.map(([id]) => id) + // Unrestricted behavior: the opaque buffer is not interpreted as a filter. + expect(ids.every((id) => disallowed.includes(id))).toBe(true) + }) +}) + +describe('#46 find() forwards the opaque universe to the vector beam walk', () => { + let brain: Brainy + + beforeEach(async () => { + brain = new Brainy(createTestConfig()) + await brain.init() + await brain.add({ type: NounType.Document, data: 'machine learning notes', metadata: { author: 'alice' } }) + await brain.add({ type: NounType.Document, data: 'unrelated cooking', metadata: { author: 'bob' } }) + }) + + afterEach(async () => { + await brain.close() + }) + + it('passes the metadata getIdSetForFilter() OpaqueIdSet straight through as allowedIds', async () => { + const sentinel = new Uint8Array([0xab, 0xcd]) // stand-in for cor's roaring Buffer + // Stub the native producer the JS metadata manager does not implement. + ;(brain as any).metadataIndex.getIdSetForFilter = async () => sentinel + + // Record what the vector index actually receives, then run the real search. + const calls: any[] = [] + const realSearch = (brain as any).index.search.bind((brain as any).index) + ;(brain as any).index.search = async (vec: any, k: any, f: any, opts: any) => { + if (opts) calls.push(opts) + return realSearch(vec, k, f, opts) + } + + await brain.find({ query: 'machine learning', where: { author: 'alice' } }) + + const withUniverse = calls.find((o) => o.allowedIds !== undefined) + expect(withUniverse).toBeDefined() + // The opaque Buffer is forwarded verbatim (same reference — never materialized/copied). + expect(withUniverse.allowedIds).toBe(sentinel) + // The materialized candidateIds path is still present for the JS index. + expect(withUniverse.candidateIds).toBeDefined() + }) + + it('omits allowedIds when the metadata provider has no opaque producer (JS path)', async () => { + // No getIdSetForFilter stub — the JS manager genuinely lacks it. + const calls: any[] = [] + const realSearch = (brain as any).index.search.bind((brain as any).index) + ;(brain as any).index.search = async (vec: any, k: any, f: any, opts: any) => { + if (opts) calls.push(opts) + return realSearch(vec, k, f, opts) + } + + await brain.find({ query: 'machine learning', where: { author: 'alice' } }) + + const withOpts = calls.find((o) => o.candidateIds !== undefined) + expect(withOpts).toBeDefined() + expect(withOpts.allowedIds).toBeUndefined() // JS path: materialized candidateIds only + }) +})