feat(8.0): vector allowedIds predicate-pushdown into find() (#46)

Filtered semantic search now keeps its recall. A find({ query, where }) that
pairs vector search with a metadata filter restricts the HNSW beam walk to the
matching candidates INSIDE the traversal (walk-all, collect-allowed) rather than
filtering the top-k afterward — so a query whose nearest vectors are all filtered
out still returns the best matches that DO pass the filter, instead of empty.

- JS HNSW search() honors the 8.0 `allowedIds: OpaqueIdSet | ReadonlySet<string>`
  contract param (ANDs with candidateIds; ignores the opaque Buffer form it can't
  decode — only a native provider consumes that).
- MetadataIndexProvider gains an OPTIONAL `getIdSetForFilter(filter): OpaqueIdSet`
  producer — the native (cor) metadata index returns its roaring filter result as
  a serialized buffer; the JS index does not implement it.
- find() forwards the matched universe to the vector walk: the opaque buffer
  (zero id materialization) when the native producer is present, alongside the
  materialized visibility-precise candidateIds for the JS index. Visibility stays
  correct via the existing post-search hard filter.

Tested: JS recall/restriction/AND/opaque-ignore on the index directly, plus
find() forwarding the opaque universe through to the beam walk via a stubbed
native producer.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

src/hnsw/hnswIndex.ts

@@ -14,7 +14,7 @@ import { euclideanDistance, calculateDistancesBatch } from '../utils/index.js'
 import type { BaseStorage } from '../storage/baseStorage.js'
 import { getGlobalCache, UnifiedCache } from '../utils/unifiedCache.js'
 import { prodLog } from '../utils/logger.js'
-import type { VectorIndexProvider } from '../plugin.js'
+import type { VectorIndexProvider, OpaqueIdSet } from '../plugin.js'
 import { ConnectionsCodec, compressedConnectionsKey } from './connectionsCodec.js'
 
 // Default HNSW parameters
@@ -656,23 +656,52 @@ export class JsHnswVectorIndex implements VectorIndexProvider {
    * @param queryVector Query vector
    * @param k Number of results to return
    * @param filter Optional filter function
-   * @param options Additional search options (`candidateIds` to restrict the
-   *   search to a pre-filtered set; `rerank` is provider-only, ignored here)
+   * @param options Additional search options (`candidateIds` / `allowedIds` to
+   *   restrict the search to a pre-filtered set; `rerank` is provider-only, ignored here)
    */
   public async search(
     queryVector: Vector,
     k: number = 10,
     filter?: (id: string) => Promise<boolean>,
-    options?: { rerank?: { multiplier: number }; candidateIds?: string[] }
+    options?: {
+      rerank?: { multiplier: number }
+      candidateIds?: string[]
+      allowedIds?: OpaqueIdSet | ReadonlySet<string>
+    }
   ): Promise<Array<[string, number]>> {
     if (this.nouns.size === 0) {
       return []
     }
 
-    // Metadata-first: convert candidateIds to filter function if no explicit filter
-    if (!filter && options?.candidateIds && options.candidateIds.length > 0) {
-      const candidateSet = new Set(options.candidateIds)
-      filter = async (id: string) => candidateSet.has(id)
+    // Metadata-first candidate restriction. Build a filter from the pre-resolved
+    // universe(s) when the caller didn't pass an explicit one. Both `candidateIds`
+    // (legacy string[]) and the 8.0 `allowedIds` predicate-pushdown param restrict
+    // the SAME way here — applied INSIDE the beam walk (searchLayer keeps every
+    // neighbor as a traversal candidate but only COLLECTS allowed ids), which is
+    // what recovers the filtered recall that post-filtering loses. An `allowedIds`
+    // that arrives as an OpaqueIdSet (a native/cor roaring Buffer) is opaque to the
+    // JS index — it cannot decode it — so it is ignored here; only the native
+    // provider consumes that form. A `ReadonlySet<string>` is honored. When both
+    // `candidateIds` and a Set `allowedIds` are present they AND together.
+    if (!filter) {
+      const universes: ReadonlySet<string>[] = []
+      if (options?.candidateIds && options.candidateIds.length > 0) {
+        universes.push(new Set(options.candidateIds))
+      }
+      const allowed = options?.allowedIds
+      if (
+        allowed &&
+        !(allowed instanceof Uint8Array) &&
+        typeof (allowed as ReadonlySet<string>).has === 'function'
+      ) {
+        universes.push(allowed as ReadonlySet<string>)
+      }
+      if (universes.length === 1) {
+        const universe = universes[0]
+        filter = async (id: string) => universe.has(id)
+      } else if (universes.length > 1) {
+        filter = async (id: string) => universes.every((u) => u.has(id))
+      }
     }
 
     // Check if query vector is defined