soulcraft/brainy
1
0
Fork 0
Code Issues 1 Pull requests Projects Releases 335 Packages Wiki Activity Actions

feat(8.0): vector allowedIds predicate-pushdown into find() (#46)

Browse source 
Filtered semantic search now keeps its recall. A find({ query, where }) that
pairs vector search with a metadata filter restricts the HNSW beam walk to the
matching candidates INSIDE the traversal (walk-all, collect-allowed) rather than
filtering the top-k afterward — so a query whose nearest vectors are all filtered
out still returns the best matches that DO pass the filter, instead of empty.

- JS HNSW search() honors the 8.0 `allowedIds: OpaqueIdSet | ReadonlySet<string>`
  contract param (ANDs with candidateIds; ignores the opaque Buffer form it can't
  decode — only a native provider consumes that).
- MetadataIndexProvider gains an OPTIONAL `getIdSetForFilter(filter): OpaqueIdSet`
  producer — the native (cor) metadata index returns its roaring filter result as
  a serialized buffer; the JS index does not implement it.
- find() forwards the matched universe to the vector walk: the opaque buffer
  (zero id materialization) when the native producer is present, alongside the
  materialized visibility-precise candidateIds for the JS index. Visibility stays
  correct via the existing post-search hard filter.

Tested: JS recall/restriction/AND/opaque-ignore on the index directly, plus
find() forwarding the opaque universe through to the beam walk via a stubbed
native producer.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
David Snelling 2026-06-23 13:18:34 -07:00
parent 632d90aac5
commit dd325f2f94
5 changed files with 279 additions and 15 deletions
Download patch file Download diff file Expand all files Collapse all files

50
src/brainy.ts
View file

@ -49,7 +49,9 @@ import type {
Subgraph,
RankOptions,
CommunitiesOptions,
PathOptions
PathOptions,
MetadataIndexProvider,
OpaqueIdSet
} from './plugin.js'
import type {
BrainyPlugin,
@ -4807,6 +4809,11 @@ export class Brainy<T = any> implements BrainyInterface<T> {
// bitmap), JS HNSW converts to a Set-based filter function.
let preResolvedMetadataIds: string[] | null = null
let preResolvedFilter: any = null
// 8.0 #46 (CTX-PUSHDOWN-ALLOWEDIDS): the matched universe as an opaque roaring
// Buffer, forwarded straight into the vector beam walk with NO id materialization
// when the active metadata provider can produce one (native/cor). Absent on the
// JS path — there the materialized `candidateIds` restricts the walk instead.
let preResolvedAllowedIds: OpaqueIdSet | undefined
if (params.where || params.type || params.subtype || params.service || params.excludeVFS) {
preResolvedFilter = {}
@ -4854,6 +4861,19 @@ export class Brainy<T = any> implements BrainyInterface<T> {
if (preResolvedMetadataIds.length === 0) {
return []
}
// 8.0 #46: when the active metadata provider exposes the opaque-set producer
// (native/cor — the JS index does not), forward the matched universe to the
// vector beam walk as an OpaqueIdSet (zero id materialization). The opaque set
// is the RAW filter universe (the set is opaque, so the visibility exclusion
// cannot be AND-ed into it in TS) — hidden tiers are instead dropped by the
// post-search visibility hard filter below, and the JS path's materialized
// `candidateIds` stays visibility-precise. Both forms are passed; the native
// provider consumes the opaque one, the JS index the string one.
const mip = this.metadataIndex as unknown as MetadataIndexProvider
if (typeof mip.getIdSetForFilter === 'function') {
preResolvedAllowedIds = await mip.getIdSetForFilter(preResolvedFilter)
}
}
// Zero-Config Hybrid Search
@ -4867,14 +4887,14 @@ export class Brainy<T = any> implements BrainyInterface<T> {
}
// Handle semantic-only query (user explicitly wants vector search)
else if ((searchMode === 'semantic' || searchMode === 'vector') && (params.query || params.vector)) {
results = await this.executeVectorSearch(params, preResolvedMetadataIds ?? undefined)
results = await this.executeVectorSearch(params, preResolvedMetadataIds ?? undefined, preResolvedAllowedIds)
}
// Handle explicit hybrid or auto mode with query
else if ((searchMode === 'auto' || searchMode === 'hybrid') && params.query && params.query.trim() !== '' && !params.vector) {
// Zero-config hybrid: combine text + semantic search with RRF fusion
const [textResults, semanticResults] = await Promise.all([
this.executeTextSearch(params.query, limit * 2),
this.executeVectorSearch(params, preResolvedMetadataIds ?? undefined)
this.executeVectorSearch(params, preResolvedMetadataIds ?? undefined, preResolvedAllowedIds)
])
// Use user-specified alpha or auto-detect based on query length
@ -4888,7 +4908,7 @@ export class Brainy<T = any> implements BrainyInterface<T> {
}
// Handle direct vector search (no query text) - no hybrid needed
else if (params.vector && !params.query) {
results = await this.executeVectorSearch(params, preResolvedMetadataIds ?? undefined)
results = await this.executeVectorSearch(params, preResolvedMetadataIds ?? undefined, preResolvedAllowedIds)
}
// Handle proximity search
else if (params.near) {
@ -11133,13 +11153,29 @@ export class Brainy<T = any> implements BrainyInterface<T> {
* When provided, HNSW search is restricted to these candidates:
* - NativeHNSWWrapper: uses native searchWithCandidates (Rust bitmap filtering)
* - JS JsHnswVectorIndex: converts to filter function with O(1) Set lookups
* @param allowedIds Optional 8.0 #46 predicate-pushdown universe as an
* {@link OpaqueIdSet} (native/cor roaring Buffer) — forwarded straight into the
* beam walk with no id materialization. The native vector provider consumes it;
* the JS index ignores the opaque form and relies on `candidateIds`.
*/
private async executeVectorSearch(params: FindParams<T>, candidateIds?: string[]): Promise<Result<T>[]> {
private async executeVectorSearch(
params: FindParams<T>,
candidateIds?: string[],
allowedIds?: OpaqueIdSet
): Promise<Result<T>[]> {
const vector = params.vector || (await this.embed(params.query!))
const limit = params.limit || 10
// Build search options for metadata-first candidate filtering
const searchOptions = candidateIds ? { candidateIds } : undefined
// Build search options for metadata-first candidate filtering. Pass both forms
// when available: the native provider prefers the opaque `allowedIds` (zero
// crossing), the JS index uses the materialized `candidateIds`.
const searchOptions =
candidateIds || allowedIds
? {
...(candidateIds && { candidateIds }),
...(allowedIds && { allowedIds })
}
: undefined
// HNSW search with optional metadata-first candidate filtering
const searchResults: [string, number][] = await this.index.search(vector, limit * 2, undefined, searchOptions)

45
src/hnsw/hnswIndex.ts
View file

@ -14,7 +14,7 @@ import { euclideanDistance, calculateDistancesBatch } from '../utils/index.js'
import type { BaseStorage } from '../storage/baseStorage.js'
import { getGlobalCache, UnifiedCache } from '../utils/unifiedCache.js'
import { prodLog } from '../utils/logger.js'
import type { VectorIndexProvider } from '../plugin.js'
import type { VectorIndexProvider, OpaqueIdSet } from '../plugin.js'
import { ConnectionsCodec, compressedConnectionsKey } from './connectionsCodec.js'
// Default HNSW parameters
@ -656,23 +656,52 @@ export class JsHnswVectorIndex implements VectorIndexProvider {
* @param queryVector Query vector
* @param k Number of results to return
* @param filter Optional filter function
* @param options Additional search options (`candidateIds` to restrict the
* search to a pre-filtered set; `rerank` is provider-only, ignored here)
* @param options Additional search options (`candidateIds` / `allowedIds` to
* restrict the search to a pre-filtered set; `rerank` is provider-only, ignored here)
*/
public async search(
queryVector: Vector,
k: number = 10,
filter?: (id: string) => Promise<boolean>,
options?: { rerank?: { multiplier: number }; candidateIds?: string[] }
options?: {
rerank?: { multiplier: number }
candidateIds?: string[]
allowedIds?: OpaqueIdSet | ReadonlySet<string>
}
): Promise<Array<[string, number]>> {
if (this.nouns.size === 0) {
return []
}
// Metadata-first: convert candidateIds to filter function if no explicit filter
if (!filter && options?.candidateIds && options.candidateIds.length > 0) {
const candidateSet = new Set(options.candidateIds)
filter = async (id: string) => candidateSet.has(id)
// Metadata-first candidate restriction. Build a filter from the pre-resolved
// universe(s) when the caller didn't pass an explicit one. Both `candidateIds`
// (legacy string[]) and the 8.0 `allowedIds` predicate-pushdown param restrict
// the SAME way here — applied INSIDE the beam walk (searchLayer keeps every
// neighbor as a traversal candidate but only COLLECTS allowed ids), which is
// what recovers the filtered recall that post-filtering loses. An `allowedIds`
// that arrives as an OpaqueIdSet (a native/cor roaring Buffer) is opaque to the
// JS index — it cannot decode it — so it is ignored here; only the native
// provider consumes that form. A `ReadonlySet<string>` is honored. When both
// `candidateIds` and a Set `allowedIds` are present they AND together.
if (!filter) {
const universes: ReadonlySet<string>[] = []
if (options?.candidateIds && options.candidateIds.length > 0) {
universes.push(new Set(options.candidateIds))
}
const allowed = options?.allowedIds
if (
allowed &&
!(allowed instanceof Uint8Array) &&
typeof (allowed as ReadonlySet<string>).has === 'function'
) {
universes.push(allowed as ReadonlySet<string>)
}
if (universes.length === 1) {
const universe = universes[0]
filter = async (id: string) => universe.has(id)
} else if (universes.length > 1) {
filter = async (id: string) => universes.every((u) => u.has(id))
}
}
// Check if query vector is defined

16
src/plugin.ts
View file

@ -130,6 +130,22 @@ export interface MetadataIndexProvider {
getIds(field: string, value: any): Promise<string[]>
getIdsForFilter(filter: any): Promise<string[]>
/**
* @description OPTIONAL: resolve a `where` filter to its matching id universe as
* an {@link OpaqueIdSet} (a serialized roaring `Buffer`) WITHOUT materializing
* the ids in TypeScript — the producer half of the predicate-pushdown
* (`CTX-PUSHDOWN-ALLOWEDIDS`) and graph query→expand fusion. Brainy forwards the
* returned set opaquely to `VectorIndexProvider.search({ allowedIds })` and
* `GraphAccelerationProvider.traverse(seeds)`; it NEVER inspects it. A native
* (cor) metadata index returns its roaring filter result directly (zero
* crossing); the JS index does not implement this — Brainy falls back to
* {@link MetadataIndexProvider.getIdsForFilter} + a `ReadonlySet<string>` when
* absent. Present ⟺ the native stack is the active provider, so the matching
* native vector/graph engines can decode the same envelope.
* @param filter - The same filter shape accepted by `getIdsForFilter`.
* @returns The matching id universe as an opaque set.
*/
getIdSetForFilter?(filter: any): Promise<OpaqueIdSet>
getIdsForTextQuery(query: string): Promise<Array<{ id: string; matchCount: number }>>
getSortedIdsForFilter(filter: any, orderBy: string, order?: 'asc' | 'desc'): Promise<string[]>
getFilterValues(field: string): Promise<string[]>