feat(8.0): vector allowedIds predicate-pushdown into find() (#46)

Filtered semantic search now keeps its recall. A find({ query, where }) that
pairs vector search with a metadata filter restricts the HNSW beam walk to the
matching candidates INSIDE the traversal (walk-all, collect-allowed) rather than
filtering the top-k afterward — so a query whose nearest vectors are all filtered
out still returns the best matches that DO pass the filter, instead of empty.

- JS HNSW search() honors the 8.0 `allowedIds: OpaqueIdSet | ReadonlySet<string>`
  contract param (ANDs with candidateIds; ignores the opaque Buffer form it can't
  decode — only a native provider consumes that).
- MetadataIndexProvider gains an OPTIONAL `getIdSetForFilter(filter): OpaqueIdSet`
  producer — the native (cor) metadata index returns its roaring filter result as
  a serialized buffer; the JS index does not implement it.
- find() forwards the matched universe to the vector walk: the opaque buffer
  (zero id materialization) when the native producer is present, alongside the
  materialized visibility-precise candidateIds for the JS index. Visibility stays
  correct via the existing post-search hard filter.

Tested: JS recall/restriction/AND/opaque-ignore on the index directly, plus
find() forwarding the opaque universe through to the beam walk via a stubbed
native producer.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

RELEASES.md

@@ -66,6 +66,13 @@ no separate migration doc. **`8.0.0-rc.2` is live** — install with
 >   engine when present, and fall back to pure-TS kernels (PageRank, connected components / Tarjan
 >   SCC, BFS / Dijkstra) otherwise. Both paths return identical shapes and respect the default
 >   visibility filter (opt in with `includeInternal` / `includeSystem`).
+> - **Filtered vector search keeps its recall (`allowedIds` pushdown).** A `find({ query, where })`
+>   that combines semantic search with a metadata filter now restricts the vector walk to the
+>   matching candidates *inside* the search (walk-all, collect-allowed) instead of filtering the
+>   top-k afterward — so a query whose nearest vectors are all filtered out still returns the best
+>   matches that DO pass the filter, rather than coming back empty. No API change. With the native
+>   `@soulcraft/cor` 3.0 stack the matched universe is forwarded as an opaque roaring buffer with
+>   zero id materialization in TypeScript; the pure-JS path restricts the beam walk with a string set.
 
 ### Headline: Database as a Value
 

src/brainy.ts

@@ -49,7 +49,9 @@ import type {
   Subgraph,
   RankOptions,
   CommunitiesOptions,
-  PathOptions
+  PathOptions,
+  MetadataIndexProvider,
+  OpaqueIdSet
 } from './plugin.js'
 import type {
   BrainyPlugin,
@@ -4807,6 +4809,11 @@ export class Brainy<T = any> implements BrainyInterface<T> {
       // bitmap), JS HNSW converts to a Set-based filter function.
       let preResolvedMetadataIds: string[] | null = null
       let preResolvedFilter: any = null
+      // 8.0 #46 (CTX-PUSHDOWN-ALLOWEDIDS): the matched universe as an opaque roaring
+      // Buffer, forwarded straight into the vector beam walk with NO id materialization
+      // when the active metadata provider can produce one (native/cor). Absent on the
+      // JS path — there the materialized `candidateIds` restricts the walk instead.
+      let preResolvedAllowedIds: OpaqueIdSet | undefined
 
       if (params.where || params.type || params.subtype || params.service || params.excludeVFS) {
         preResolvedFilter = {}
@@ -4854,6 +4861,19 @@ export class Brainy<T = any> implements BrainyInterface<T> {
         if (preResolvedMetadataIds.length === 0) {
           return []
         }
+
+        // 8.0 #46: when the active metadata provider exposes the opaque-set producer
+        // (native/cor — the JS index does not), forward the matched universe to the
+        // vector beam walk as an OpaqueIdSet (zero id materialization). The opaque set
+        // is the RAW filter universe (the set is opaque, so the visibility exclusion
+        // cannot be AND-ed into it in TS) — hidden tiers are instead dropped by the
+        // post-search visibility hard filter below, and the JS path's materialized
+        // `candidateIds` stays visibility-precise. Both forms are passed; the native
+        // provider consumes the opaque one, the JS index the string one.
+        const mip = this.metadataIndex as unknown as MetadataIndexProvider
+        if (typeof mip.getIdSetForFilter === 'function') {
+          preResolvedAllowedIds = await mip.getIdSetForFilter(preResolvedFilter)
+        }
       }
 
       // Zero-Config Hybrid Search
@@ -4867,14 +4887,14 @@ export class Brainy<T = any> implements BrainyInterface<T> {
       }
       // Handle semantic-only query (user explicitly wants vector search)
       else if ((searchMode === 'semantic' || searchMode === 'vector') && (params.query || params.vector)) {
-        results = await this.executeVectorSearch(params, preResolvedMetadataIds ?? undefined)
+        results = await this.executeVectorSearch(params, preResolvedMetadataIds ?? undefined, preResolvedAllowedIds)
       }
       // Handle explicit hybrid or auto mode with query
       else if ((searchMode === 'auto' || searchMode === 'hybrid') && params.query && params.query.trim() !== '' && !params.vector) {
         // Zero-config hybrid: combine text + semantic search with RRF fusion
         const [textResults, semanticResults] = await Promise.all([
           this.executeTextSearch(params.query, limit * 2),
-          this.executeVectorSearch(params, preResolvedMetadataIds ?? undefined)
+          this.executeVectorSearch(params, preResolvedMetadataIds ?? undefined, preResolvedAllowedIds)
         ])
 
         // Use user-specified alpha or auto-detect based on query length
@@ -4888,7 +4908,7 @@ export class Brainy<T = any> implements BrainyInterface<T> {
       }
       // Handle direct vector search (no query text) - no hybrid needed
       else if (params.vector && !params.query) {
-        results = await this.executeVectorSearch(params, preResolvedMetadataIds ?? undefined)
+        results = await this.executeVectorSearch(params, preResolvedMetadataIds ?? undefined, preResolvedAllowedIds)
       }
       // Handle proximity search
       else if (params.near) {
@@ -11133,13 +11153,29 @@ export class Brainy<T = any> implements BrainyInterface<T> {
    *   When provided, HNSW search is restricted to these candidates:
    *   - NativeHNSWWrapper: uses native searchWithCandidates (Rust bitmap filtering)
    *   - JS JsHnswVectorIndex: converts to filter function with O(1) Set lookups
+   * @param allowedIds Optional 8.0 #46 predicate-pushdown universe as an
+   *   {@link OpaqueIdSet} (native/cor roaring Buffer) — forwarded straight into the
+   *   beam walk with no id materialization. The native vector provider consumes it;
+   *   the JS index ignores the opaque form and relies on `candidateIds`.
    */
-  private async executeVectorSearch(params: FindParams<T>, candidateIds?: string[]): Promise<Result<T>[]> {
+  private async executeVectorSearch(
+    params: FindParams<T>,
+    candidateIds?: string[],
+    allowedIds?: OpaqueIdSet
+  ): Promise<Result<T>[]> {
     const vector = params.vector || (await this.embed(params.query!))
     const limit = params.limit || 10
 
-    // Build search options for metadata-first candidate filtering
+    // Build search options for metadata-first candidate filtering. Pass both forms
-    const searchOptions = candidateIds ? { candidateIds } : undefined
+    // when available: the native provider prefers the opaque `allowedIds` (zero
+    // crossing), the JS index uses the materialized `candidateIds`.
+    const searchOptions =
+      candidateIds || allowedIds
+        ? {
+            ...(candidateIds && { candidateIds }),
+            ...(allowedIds && { allowedIds })
+          }
+        : undefined
 
     // HNSW search with optional metadata-first candidate filtering
     const searchResults: [string, number][] = await this.index.search(vector, limit * 2, undefined, searchOptions)

src/hnsw/hnswIndex.ts

@@ -14,7 +14,7 @@ import { euclideanDistance, calculateDistancesBatch } from '../utils/index.js'
 import type { BaseStorage } from '../storage/baseStorage.js'
 import { getGlobalCache, UnifiedCache } from '../utils/unifiedCache.js'
 import { prodLog } from '../utils/logger.js'
-import type { VectorIndexProvider } from '../plugin.js'
+import type { VectorIndexProvider, OpaqueIdSet } from '../plugin.js'
 import { ConnectionsCodec, compressedConnectionsKey } from './connectionsCodec.js'
 
 // Default HNSW parameters
@@ -656,23 +656,52 @@ export class JsHnswVectorIndex implements VectorIndexProvider {
    * @param queryVector Query vector
    * @param k Number of results to return
    * @param filter Optional filter function
-   * @param options Additional search options (`candidateIds` to restrict the
+   * @param options Additional search options (`candidateIds` / `allowedIds` to
-   *   search to a pre-filtered set; `rerank` is provider-only, ignored here)
+   *   restrict the search to a pre-filtered set; `rerank` is provider-only, ignored here)
    */
   public async search(
     queryVector: Vector,
     k: number = 10,
     filter?: (id: string) => Promise<boolean>,
-    options?: { rerank?: { multiplier: number }; candidateIds?: string[] }
+    options?: {
+      rerank?: { multiplier: number }
+      candidateIds?: string[]
+      allowedIds?: OpaqueIdSet | ReadonlySet<string>
+    }
   ): Promise<Array<[string, number]>> {
     if (this.nouns.size === 0) {
       return []
     }
 
-    // Metadata-first: convert candidateIds to filter function if no explicit filter
+    // Metadata-first candidate restriction. Build a filter from the pre-resolved
-    if (!filter && options?.candidateIds && options.candidateIds.length > 0) {
+    // universe(s) when the caller didn't pass an explicit one. Both `candidateIds`
-      const candidateSet = new Set(options.candidateIds)
+    // (legacy string[]) and the 8.0 `allowedIds` predicate-pushdown param restrict
-      filter = async (id: string) => candidateSet.has(id)
+    // the SAME way here — applied INSIDE the beam walk (searchLayer keeps every
+    // neighbor as a traversal candidate but only COLLECTS allowed ids), which is
+    // what recovers the filtered recall that post-filtering loses. An `allowedIds`
+    // that arrives as an OpaqueIdSet (a native/cor roaring Buffer) is opaque to the
+    // JS index — it cannot decode it — so it is ignored here; only the native
+    // provider consumes that form. A `ReadonlySet<string>` is honored. When both
+    // `candidateIds` and a Set `allowedIds` are present they AND together.
+    if (!filter) {
+      const universes: ReadonlySet<string>[] = []
+      if (options?.candidateIds && options.candidateIds.length > 0) {
+        universes.push(new Set(options.candidateIds))
+      }
+      const allowed = options?.allowedIds
+      if (
+        allowed &&
+        !(allowed instanceof Uint8Array) &&
+        typeof (allowed as ReadonlySet<string>).has === 'function'
+      ) {
+        universes.push(allowed as ReadonlySet<string>)
+      }
+      if (universes.length === 1) {
+        const universe = universes[0]
+        filter = async (id: string) => universe.has(id)
+      } else if (universes.length > 1) {
+        filter = async (id: string) => universes.every((u) => u.has(id))
+      }
     }
 
     // Check if query vector is defined

src/plugin.ts

@@ -130,6 +130,22 @@ export interface MetadataIndexProvider {
 
   getIds(field: string, value: any): Promise<string[]>
   getIdsForFilter(filter: any): Promise<string[]>
+  /**
+   * @description OPTIONAL: resolve a `where` filter to its matching id universe as
+   * an {@link OpaqueIdSet} (a serialized roaring `Buffer`) WITHOUT materializing
+   * the ids in TypeScript — the producer half of the predicate-pushdown
+   * (`CTX-PUSHDOWN-ALLOWEDIDS`) and graph query→expand fusion. Brainy forwards the
+   * returned set opaquely to `VectorIndexProvider.search({ allowedIds })` and
+   * `GraphAccelerationProvider.traverse(seeds)`; it NEVER inspects it. A native
+   * (cor) metadata index returns its roaring filter result directly (zero
+   * crossing); the JS index does not implement this — Brainy falls back to
+   * {@link MetadataIndexProvider.getIdsForFilter} + a `ReadonlySet<string>` when
+   * absent. Present ⟺ the native stack is the active provider, so the matching
+   * native vector/graph engines can decode the same envelope.
+   * @param filter - The same filter shape accepted by `getIdsForFilter`.
+   * @returns The matching id universe as an opaque set.
+   */
+  getIdSetForFilter?(filter: any): Promise<OpaqueIdSet>
   getIdsForTextQuery(query: string): Promise<Array<{ id: string; matchCount: number }>>
   getSortedIdsForFilter(filter: any, orderBy: string, order?: 'asc' | 'desc'): Promise<string[]>
   getFilterValues(field: string): Promise<string[]>

tests/unit/hnsw/allowed-ids-pushdown.test.ts

@@ -0,0 +1,176 @@
+/**
+ * @module tests/unit/hnsw/allowed-ids-pushdown
+ * @description 8.0 #46 (CTX-PUSHDOWN-ALLOWEDIDS) — the vector-search predicate
+ * pushdown. Two layers:
+ *
+ * 1. The JS HNSW index honors `allowedIds: ReadonlySet<string>` by restricting the
+ *    beam walk to allowed nodes INSIDE the traversal (walk-all, collect-allowed) —
+ *    the semantics that recover the filtered recall a naive top-k-then-filter loses.
+ *    An `allowedIds` arriving as an `OpaqueIdSet` (a native/cor roaring Buffer) is
+ *    opaque to the JS index and ignored — only a native provider decodes it.
+ *
+ * 2. `find()` forwards the matched universe to the vector beam walk as an
+ *    `OpaqueIdSet` (zero id materialization) when the active metadata provider
+ *    exposes the `getIdSetForFilter` producer — the native-stack zero-crossing path.
+ */
+
+import { describe, it, expect, beforeEach, afterEach } from 'vitest'
+import { JsHnswVectorIndex } from '../../../src/hnsw/hnswIndex.js'
+import { euclideanDistance } from '../../../src/utils/index.js'
+import { MemoryStorage } from '../../../src/storage/adapters/memoryStorage.js'
+import { Brainy } from '../../../src/index.js'
+import { NounType } from '../../../src/types/graphTypes.js'
+import { createTestConfig } from '../../helpers/test-factory.js'
+
+const DIM = 8
+
+/** Deterministic PRNG (mulberry32) so the synthetic graph is identical every run. */
+function seededRand(seed: number): () => number {
+  let s = seed >>> 0
+  return () => {
+    s = (s + 0x6d2b79f5) | 0
+    let t = Math.imul(s ^ (s >>> 15), 1 | s)
+    t = (t + Math.imul(t ^ (t >>> 7), 61 | t)) ^ t
+    return ((t ^ (t >>> 14)) >>> 0) / 4294967296
+  }
+}
+
+/**
+ * A vector at the requested distance-from-origin pointing in a deterministic RANDOM
+ * direction (so the HNSW graph is well-connected — diverse directions, not a
+ * degenerate axis cluster). The query is the origin, so distance-from-query equals
+ * `magnitude` exactly — letting the test place nodes in clean distance bands.
+ */
+function vecAt(magnitude: number, idx: number): number[] {
+  const rand = seededRand(idx + 1)
+  const dir = Array.from({ length: DIM }, () => rand() * 2 - 1)
+  const norm = Math.hypot(...dir) || 1
+  return dir.map((x) => (x / norm) * magnitude)
+}
+
+describe('#46 JS HNSW allowedIds pushdown (recall-preserving)', () => {
+  let index: JsHnswVectorIndex
+  const query = new Array(DIM).fill(0)
+  // Two clean distance bands: the disallowed nodes are STRICTLY closer to the query
+  // than the allowed targets. A naive top-k-then-filter returns only the disallowed
+  // (then drops them ⇒ empty); the pushdown must reach the allowed targets. `M` ≥ N−1
+  // makes the small graph complete, so reachability is guaranteed and the test asserts
+  // the FILTER mechanism (walk-all, collect-allowed), not emergent HNSW connectivity.
+  const disallowed: string[] = []
+  const targets: string[] = []
+
+  beforeEach(async () => {
+    index = new JsHnswVectorIndex(
+      { M: 16, efConstruction: 200, efSearch: 50, ml: 16 },
+      euclideanDistance,
+      { useParallelization: false, storage: new MemoryStorage() }
+    )
+    let n = 0
+    for (let i = 0; i < 5; i++) {
+      const id = `disallowed-${i}`
+      disallowed.push(id)
+      await index.addItem({ id, vector: vecAt(0.3, n++) }) // closer band
+    }
+    for (let i = 0; i < 5; i++) {
+      const id = `target-${i}`
+      targets.push(id)
+      await index.addItem({ id, vector: vecAt(0.8, n++) }) // farther band
+    }
+  })
+
+  afterEach(() => {
+    disallowed.length = 0
+    targets.length = 0
+  })
+
+  it('without restriction, the top-k are all disallowed (so post-filtering would lose recall)', async () => {
+    const results = await index.search(query, 3)
+    const ids = results.map(([id]) => id)
+    // Every nearest neighbor is in the (closer) disallowed band → naive "filter after" = 0 hits.
+    expect(ids.length).toBe(3)
+    expect(ids.every((id) => disallowed.includes(id))).toBe(true)
+  })
+
+  it('with allowedIds, the walk reaches the allowed targets the naive path would miss', async () => {
+    const allowed = new Set(targets)
+    const results = await index.search(query, 3, undefined, { allowedIds: allowed })
+    const ids = results.map(([id]) => id)
+
+    expect(ids.length).toBe(3)
+    expect(ids.some((id) => disallowed.includes(id))).toBe(false) // no disallowed leaks through
+    expect(ids.every((id) => allowed.has(id))).toBe(true) // only allowed targets
+  })
+
+  it('ANDs allowedIds with candidateIds when both are given', async () => {
+    const results = await index.search(query, 5, undefined, {
+      candidateIds: [targets[0], targets[1], disallowed[0]],
+      allowedIds: new Set([targets[0], disallowed[0], disallowed[1]])
+    })
+    const ids = results.map(([id]) => id)
+    // Intersection = { target-0, disallowed-0 } — nothing outside it may appear.
+    expect(ids.every((id) => id === targets[0] || id === disallowed[0])).toBe(true)
+    expect(ids).toContain(targets[0])
+  })
+
+  it('ignores an OpaqueIdSet (Uint8Array) — only a native provider can decode it', async () => {
+    const opaque = new Uint8Array([0xca, 0xfe, 0x01, 0x02])
+    const results = await index.search(query, 3, undefined, { allowedIds: opaque })
+    const ids = results.map(([id]) => id)
+    // Unrestricted behavior: the opaque buffer is not interpreted as a filter.
+    expect(ids.every((id) => disallowed.includes(id))).toBe(true)
+  })
+})
+
+describe('#46 find() forwards the opaque universe to the vector beam walk', () => {
+  let brain: Brainy
+
+  beforeEach(async () => {
+    brain = new Brainy(createTestConfig())
+    await brain.init()
+    await brain.add({ type: NounType.Document, data: 'machine learning notes', metadata: { author: 'alice' } })
+    await brain.add({ type: NounType.Document, data: 'unrelated cooking', metadata: { author: 'bob' } })
+  })
+
+  afterEach(async () => {
+    await brain.close()
+  })
+
+  it('passes the metadata getIdSetForFilter() OpaqueIdSet straight through as allowedIds', async () => {
+    const sentinel = new Uint8Array([0xab, 0xcd]) // stand-in for cor's roaring Buffer
+    // Stub the native producer the JS metadata manager does not implement.
+    ;(brain as any).metadataIndex.getIdSetForFilter = async () => sentinel
+
+    // Record what the vector index actually receives, then run the real search.
+    const calls: any[] = []
+    const realSearch = (brain as any).index.search.bind((brain as any).index)
+    ;(brain as any).index.search = async (vec: any, k: any, f: any, opts: any) => {
+      if (opts) calls.push(opts)
+      return realSearch(vec, k, f, opts)
+    }
+
+    await brain.find({ query: 'machine learning', where: { author: 'alice' } })
+
+    const withUniverse = calls.find((o) => o.allowedIds !== undefined)
+    expect(withUniverse).toBeDefined()
+    // The opaque Buffer is forwarded verbatim (same reference — never materialized/copied).
+    expect(withUniverse.allowedIds).toBe(sentinel)
+    // The materialized candidateIds path is still present for the JS index.
+    expect(withUniverse.candidateIds).toBeDefined()
+  })
+
+  it('omits allowedIds when the metadata provider has no opaque producer (JS path)', async () => {
+    // No getIdSetForFilter stub — the JS manager genuinely lacks it.
+    const calls: any[] = []
+    const realSearch = (brain as any).index.search.bind((brain as any).index)
+    ;(brain as any).index.search = async (vec: any, k: any, f: any, opts: any) => {
+      if (opts) calls.push(opts)
+      return realSearch(vec, k, f, opts)
+    }
+
+    await brain.find({ query: 'machine learning', where: { author: 'alice' } })
+
+    const withOpts = calls.find((o) => o.candidateIds !== undefined)
+    expect(withOpts).toBeDefined()
+    expect(withOpts.allowedIds).toBeUndefined() // JS path: materialized candidateIds only
+  })
+})