feat(8.0): #35 part-3 — supply at-gen candidate vectors for the native exact-rerank

Completes brainy's #35 side. When the at-gen vector gate serves a filtered
semantic read, Brainy now ships the historically-correct candidate vectors so the
native provider reranks against them with zero per-vector FFI crossing — the
provider need not duplicate the per-generation retention Brainy already does.

- New `AtGenerationVectors { ids: BigInt64Array; vectors: Float32Array; dim }` on
  the VectorIndexProvider.search options bag (row-major: vectors[i*dim..] == ids[i];
  invariant vectors.length === ids.length*dim; ids = the candidate set; dim must
  match the index dim). The built-in JS index ignores it.
- buildAtGenerationVectors resolves each universe id's vector AS OF the generation
  (the record before-image, or live getNoun when untouched since the pin — not
  readNounRaw, whose canonical path is empty under lazy-vector eviction), interns
  the id via the shared mapper, and packs row-major. Absent/vectorless/wrong-dim
  ids are dropped (not vector-rankable). Bounded by the filtered universe.

Shape + the four clarifications (row-major, ids-are-the-candidate-set, dim-asserted,
filtered-path-only) confirmed with cor in the handoff #35 thread. Inert until cor's
native at-gen rerank advertises isGenerationVisible (honesty guard).

Tested: the mock versioned provider now asserts it receives atGenerationVectors with
the row-major invariant, correct dim, and ids resolving back to the at-gen universe.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

src/brainy.ts

@@ -51,7 +51,8 @@ import type {
   CommunitiesOptions,
   PathOptions,
   MetadataIndexProvider,
-  OpaqueIdSet
+  OpaqueIdSet,
+  AtGenerationVectors
 } from './plugin.js'
 import type {
   BrainyPlugin,
@@ -7288,16 +7289,68 @@ export class Brainy<T = any> implements BrainyInterface<T> {
     generation: number
   ): Promise<Array<[string, number]>> {
     const vector = params.vector || (await this.embed(params.query!))
-    // The provider scores the at-gen vectors restricted to `allowedIds`. (#35 part-3:
-    // the gen-g candidate VECTORS — `atGenerationVectors` — are supplied here once that
-    // seam is confirmed with cor; a provider needing them refuses the generation until
-    // then, so `canServeVectorAtGeneration` gates this path off in the interim.)
+    // #35 part-3: supply the at-gen candidate VECTORS (resolved from Brainy's
+    // per-generation record before-images) so the provider reranks the historically
+    // correct vectors with zero per-vector crossing. `atGenerationVectors.ids` is the
+    // candidate set; `allowedIds` rides along (the provider may AND it).
+    const atGenerationVectors = await this.buildAtGenerationVectors(allowedIds, generation)
     return this.index.search(vector, k, undefined, {
       allowedIds,
-      generation: BigInt(generation)
+      generation: BigInt(generation),
+      ...(atGenerationVectors && { atGenerationVectors })
     })
   }
 
+  /**
+   * @description 8.0 #35 part-3 — assemble the {@link AtGenerationVectors} columnar
+   * payload for a filtered at-gen exact-rerank: resolve each universe id's vector AS
+   * OF `generation` (the canonical record before-image, or live storage when the id
+   * was untouched since the pin), intern the id via the shared mapper, and pack the
+   * vectors flat row-major (`vectors[i*dim .. (i+1)*dim]` ↔ `ids[i]`). Ids absent at
+   * the generation, vectorless, or of the wrong dimension are dropped (they cannot be
+   * vector-ranked). Bounded by the filtered universe — far cheaper than the full-corpus
+   * rebuild it replaces. Returns `undefined` when no dimension is known or nothing maps.
+   * @param allowedIds - The at-gen filtered universe (candidate ids).
+   * @param generation - The as-of generation.
+   * @returns The columnar payload, or `undefined` when there is nothing to ship.
+   */
+  private async buildAtGenerationVectors(
+    allowedIds: ReadonlySet<string>,
+    generation: number
+  ): Promise<AtGenerationVectors | undefined> {
+    const dim = this.dimensions
+    if (!dim || allowedIds.size === 0) return undefined
+
+    const idMapper = this.metadataIndex.getIdMapper()
+    const ints: bigint[] = []
+    const rows: number[][] = []
+    for (const id of allowedIds) {
+      const resolved = await this.generationStore.resolveAt('noun', id, generation)
+      let vec: number[] | null = null
+      if (resolved.source === 'record') {
+        // The generation record carries the at-gen vector before-image.
+        vec = resolved.vector
+      } else if (resolved.source === 'current') {
+        // Untouched since the pin → the live vector IS the at-gen vector. Use
+        // getNoun (full hydration, lazy-load aware) — readNounRaw's canonical path
+        // is empty under lazy-vector eviction.
+        const noun = await this.storage.getNoun(id)
+        vec = noun?.vector ?? null
+      }
+      // 'absent' / vectorless / wrong-dim → skip (not vector-rankable at this gen).
+      if (Array.isArray(vec) && vec.length === dim) {
+        ints.push(BigInt(idMapper.getInt(id) ?? idMapper.getOrAssign(id)))
+        rows.push(vec)
+      }
+    }
+    if (ints.length === 0) return undefined
+
+    // Row-major pack: INVARIANT vectors.length === ids.length * dim.
+    const vectors = new Float32Array(ints.length * dim)
+    for (let i = 0; i < rows.length; i++) vectors.set(rows[i], i * dim)
+    return { ids: BigInt64Array.from(ints), vectors, dim }
+  }
+
   // --- Transact planner ------------------------------------------------------
 
   /**

src/hnsw/hnswIndex.ts

@@ -14,7 +14,7 @@ import { euclideanDistance, calculateDistancesBatch } from '../utils/index.js'
 import type { BaseStorage } from '../storage/baseStorage.js'
 import { getGlobalCache, UnifiedCache } from '../utils/unifiedCache.js'
 import { prodLog } from '../utils/logger.js'
-import type { VectorIndexProvider, OpaqueIdSet } from '../plugin.js'
+import type { VectorIndexProvider, OpaqueIdSet, AtGenerationVectors } from '../plugin.js'
 import { ConnectionsCodec, compressedConnectionsKey } from './connectionsCodec.js'
 
 // Default HNSW parameters
@@ -673,6 +673,9 @@ export class JsHnswVectorIndex implements VectorIndexProvider {
       // historical read here (its defer-gate checks `isGenerationVisible` first),
       // so an ignored `generation` cannot silently return now-vectors-as-at-gen.
       generation?: bigint
+      // 8.0 #35 part-3: at-gen candidate vectors for the native exact-rerank. The JS
+      // index serves "now" only and never receives this (gated off upstream); ignored.
+      atGenerationVectors?: AtGenerationVectors
     }
   ): Promise<Array<[string, number]>> {
     if (this.nouns.size === 0) {

src/plugin.ts

@@ -734,6 +734,34 @@ export function isGraphAccelerationProvider(
   return typeof c.traverse === 'function' && typeof c.graphCursorOpen === 'function'
 }
 
+/**
+ * Columnar at-`generation` candidate vectors for the 8.0 #35 FILTERED exact-rerank.
+ * Brainy resolves the per-generation vector before-images for the at-gen filtered
+ * universe (from its canonical generation records) and ships them flat, so a native
+ * provider reranks against the historically-correct vectors with ZERO per-vector FFI
+ * crossing — without the provider duplicating the per-gen retention Brainy already does.
+ *
+ * - **Row-major:** row `i` is `vectors[i*dim .. (i+1)*dim]` and belongs to `ids[i]`.
+ *   INVARIANT: `vectors.length === ids.length * dim` (the provider asserts + refuses on mismatch).
+ * - **`ids` IS the candidate set:** entity ints (interned via the shared mapper) for the
+ *   at-gen metadata∩graph filtered universe. The provider reranks EXACTLY these
+ *   `(id, vector)` pairs and returns top-k by distance; any `allowedIds` passed alongside
+ *   is redundant on this path (the provider may AND it as belt-and-suspenders).
+ * - **`dim`** must equal the provider's configured index dimension (asserted; mismatch → refuse).
+ *
+ * Supplied only when there is a BOUNDED filtered universe; the unfiltered-deep at-gen
+ * case stays the provider's own retained-segment ANN ({@link VersionedIndexProvider.isGenerationVisible}
+ * reports false there, so Brainy falls back to its materialization overlay).
+ */
+export interface AtGenerationVectors {
+  /** Entity ints (shared-mapper interned), one per candidate; the candidate set. */
+  ids: BigInt64Array
+  /** Flat row-major vectors: `vectors[i*dim .. (i+1)*dim]` is `ids[i]`'s at-gen vector. */
+  vectors: Float32Array
+  /** Vector dimension; MUST equal the provider's configured index dim. */
+  dim: number
+}
+
 /**
  * The object returned by the `'vector'` provider factory — Brainy's vector
  * index contract. Implementations include Brainy's own JS HNSW index and any
@@ -782,6 +810,15 @@ export interface VectorIndexProvider {
        * `generation` is the same u64 counter handed to the graph index on writes.
        */
       generation?: bigint
+      /**
+       * 8.0 #35 part-3: the at-`generation` candidate vectors for the FILTERED
+       * exact-rerank — Brainy supplies the historically-correct vectors so the
+       * provider need not retain them. Present only alongside `generation` on the
+       * filtered at-gen path; the provider reranks `atGenerationVectors.ids` by
+       * distance and returns top-k. See {@link AtGenerationVectors}. The built-in
+       * JS index ignores it (it serves "now" only).
+       */
+      atGenerationVectors?: AtGenerationVectors
     }
   ): Promise<Array<[string, number]>>
   size(): number

tests/integration/db-asof-vector-defer.test.ts

@@ -17,7 +17,11 @@ import { createTestConfig } from '../helpers/test-factory.js'
 
 /** A versioned vector provider that advertises visibility + records each search. */
 function makeVersionedVectorMock(ranking: string[]) {
-  const calls: Array<{ generation?: bigint; allowedIds?: ReadonlySet<string> }> = []
+  const calls: Array<{
+    generation?: bigint
+    allowedIds?: ReadonlySet<string>
+    atGenerationVectors?: { ids: BigInt64Array; vectors: Float32Array; dim: number }
+  }> = []
   const provider = {
     // VersionedIndexProvider quartet (duck-typed by isVersionedIndexProvider).
     generation: () => 0n,
@@ -26,7 +30,11 @@ function makeVersionedVectorMock(ranking: string[]) {
     release: (_g: bigint) => {},
     // The at-gen vector leg: record the seam fields, return ranked allowed ids.
     search: async (_vec: number[], _k: number, _filter: unknown, opts: any) => {
-      calls.push({ generation: opts?.generation, allowedIds: opts?.allowedIds })
+      calls.push({
+        generation: opts?.generation,
+        allowedIds: opts?.allowedIds,
+        atGenerationVectors: opts?.atGenerationVectors
+      })
       const allowed = opts?.allowedIds as ReadonlySet<string> | undefined
       const out: Array<[string, number]> = []
       let distance = 0.1
@@ -105,4 +113,26 @@ describe('#35 asOf filtered vector read defers to the versioned provider', () =>
     expect(page.map((r) => r.id)).toEqual([b]) // top-1 by the provider ranking
     await db.release()
   })
+
+  it('supplies the at-gen candidate vectors (part-3) — row-major, dim-correct, ids = the universe', async () => {
+    const db = brain.now()
+    await brain.add({ type: NounType.Document, data: 'zeta machine learning', metadata: { team: 'z' } }) // advance gen
+    const mock = makeVersionedVectorMock([b, a])
+    ;(brain as any).index = mock.provider
+
+    await db.find({ query: 'machine learning', where: { team: 'x' } })
+
+    const agv = mock.calls[0].atGenerationVectors
+    expect(agv).toBeDefined()
+    // ids = the at-gen `team: x` universe {a, b}, interned via the shared mapper.
+    expect(agv!.ids.length).toBe(2)
+    expect(agv!.dim).toBeGreaterThan(0)
+    // The row-major invariant cor asserts on ingest.
+    expect(agv!.vectors.length).toBe(agv!.ids.length * agv!.dim)
+    // Each interned id resolves back to a universe entity.
+    const mapper = (brain as any).metadataIndex.getIdMapper()
+    const uuids = Array.from(agv!.ids).map((i) => mapper.getUuid(Number(i)))
+    expect(new Set(uuids)).toEqual(new Set([a, b]))
+    await db.release()
+  })
 })