diff --git a/src/brainy.ts b/src/brainy.ts index e2e58962..23f3d4f1 100644 --- a/src/brainy.ts +++ b/src/brainy.ts @@ -51,7 +51,8 @@ import type { CommunitiesOptions, PathOptions, MetadataIndexProvider, - OpaqueIdSet + OpaqueIdSet, + AtGenerationVectors } from './plugin.js' import type { BrainyPlugin, @@ -7288,16 +7289,68 @@ export class Brainy implements BrainyInterface { generation: number ): Promise> { const vector = params.vector || (await this.embed(params.query!)) - // The provider scores the at-gen vectors restricted to `allowedIds`. (#35 part-3: - // the gen-g candidate VECTORS — `atGenerationVectors` — are supplied here once that - // seam is confirmed with cor; a provider needing them refuses the generation until - // then, so `canServeVectorAtGeneration` gates this path off in the interim.) + // #35 part-3: supply the at-gen candidate VECTORS (resolved from Brainy's + // per-generation record before-images) so the provider reranks the historically + // correct vectors with zero per-vector crossing. `atGenerationVectors.ids` is the + // candidate set; `allowedIds` rides along (the provider may AND it). + const atGenerationVectors = await this.buildAtGenerationVectors(allowedIds, generation) return this.index.search(vector, k, undefined, { allowedIds, - generation: BigInt(generation) + generation: BigInt(generation), + ...(atGenerationVectors && { atGenerationVectors }) }) } + /** + * @description 8.0 #35 part-3 — assemble the {@link AtGenerationVectors} columnar + * payload for a filtered at-gen exact-rerank: resolve each universe id's vector AS + * OF `generation` (the canonical record before-image, or live storage when the id + * was untouched since the pin), intern the id via the shared mapper, and pack the + * vectors flat row-major (`vectors[i*dim .. (i+1)*dim]` ↔ `ids[i]`). Ids absent at + * the generation, vectorless, or of the wrong dimension are dropped (they cannot be + * vector-ranked). Bounded by the filtered universe — far cheaper than the full-corpus + * rebuild it replaces. Returns `undefined` when no dimension is known or nothing maps. + * @param allowedIds - The at-gen filtered universe (candidate ids). + * @param generation - The as-of generation. + * @returns The columnar payload, or `undefined` when there is nothing to ship. + */ + private async buildAtGenerationVectors( + allowedIds: ReadonlySet, + generation: number + ): Promise { + const dim = this.dimensions + if (!dim || allowedIds.size === 0) return undefined + + const idMapper = this.metadataIndex.getIdMapper() + const ints: bigint[] = [] + const rows: number[][] = [] + for (const id of allowedIds) { + const resolved = await this.generationStore.resolveAt('noun', id, generation) + let vec: number[] | null = null + if (resolved.source === 'record') { + // The generation record carries the at-gen vector before-image. + vec = resolved.vector + } else if (resolved.source === 'current') { + // Untouched since the pin → the live vector IS the at-gen vector. Use + // getNoun (full hydration, lazy-load aware) — readNounRaw's canonical path + // is empty under lazy-vector eviction. + const noun = await this.storage.getNoun(id) + vec = noun?.vector ?? null + } + // 'absent' / vectorless / wrong-dim → skip (not vector-rankable at this gen). + if (Array.isArray(vec) && vec.length === dim) { + ints.push(BigInt(idMapper.getInt(id) ?? idMapper.getOrAssign(id))) + rows.push(vec) + } + } + if (ints.length === 0) return undefined + + // Row-major pack: INVARIANT vectors.length === ids.length * dim. + const vectors = new Float32Array(ints.length * dim) + for (let i = 0; i < rows.length; i++) vectors.set(rows[i], i * dim) + return { ids: BigInt64Array.from(ints), vectors, dim } + } + // --- Transact planner ------------------------------------------------------ /** diff --git a/src/hnsw/hnswIndex.ts b/src/hnsw/hnswIndex.ts index 10b0317a..c5acd20f 100644 --- a/src/hnsw/hnswIndex.ts +++ b/src/hnsw/hnswIndex.ts @@ -14,7 +14,7 @@ import { euclideanDistance, calculateDistancesBatch } from '../utils/index.js' import type { BaseStorage } from '../storage/baseStorage.js' import { getGlobalCache, UnifiedCache } from '../utils/unifiedCache.js' import { prodLog } from '../utils/logger.js' -import type { VectorIndexProvider, OpaqueIdSet } from '../plugin.js' +import type { VectorIndexProvider, OpaqueIdSet, AtGenerationVectors } from '../plugin.js' import { ConnectionsCodec, compressedConnectionsKey } from './connectionsCodec.js' // Default HNSW parameters @@ -673,6 +673,9 @@ export class JsHnswVectorIndex implements VectorIndexProvider { // historical read here (its defer-gate checks `isGenerationVisible` first), // so an ignored `generation` cannot silently return now-vectors-as-at-gen. generation?: bigint + // 8.0 #35 part-3: at-gen candidate vectors for the native exact-rerank. The JS + // index serves "now" only and never receives this (gated off upstream); ignored. + atGenerationVectors?: AtGenerationVectors } ): Promise> { if (this.nouns.size === 0) { diff --git a/src/plugin.ts b/src/plugin.ts index 673d2f1a..30af7062 100644 --- a/src/plugin.ts +++ b/src/plugin.ts @@ -734,6 +734,34 @@ export function isGraphAccelerationProvider( return typeof c.traverse === 'function' && typeof c.graphCursorOpen === 'function' } +/** + * Columnar at-`generation` candidate vectors for the 8.0 #35 FILTERED exact-rerank. + * Brainy resolves the per-generation vector before-images for the at-gen filtered + * universe (from its canonical generation records) and ships them flat, so a native + * provider reranks against the historically-correct vectors with ZERO per-vector FFI + * crossing — without the provider duplicating the per-gen retention Brainy already does. + * + * - **Row-major:** row `i` is `vectors[i*dim .. (i+1)*dim]` and belongs to `ids[i]`. + * INVARIANT: `vectors.length === ids.length * dim` (the provider asserts + refuses on mismatch). + * - **`ids` IS the candidate set:** entity ints (interned via the shared mapper) for the + * at-gen metadata∩graph filtered universe. The provider reranks EXACTLY these + * `(id, vector)` pairs and returns top-k by distance; any `allowedIds` passed alongside + * is redundant on this path (the provider may AND it as belt-and-suspenders). + * - **`dim`** must equal the provider's configured index dimension (asserted; mismatch → refuse). + * + * Supplied only when there is a BOUNDED filtered universe; the unfiltered-deep at-gen + * case stays the provider's own retained-segment ANN ({@link VersionedIndexProvider.isGenerationVisible} + * reports false there, so Brainy falls back to its materialization overlay). + */ +export interface AtGenerationVectors { + /** Entity ints (shared-mapper interned), one per candidate; the candidate set. */ + ids: BigInt64Array + /** Flat row-major vectors: `vectors[i*dim .. (i+1)*dim]` is `ids[i]`'s at-gen vector. */ + vectors: Float32Array + /** Vector dimension; MUST equal the provider's configured index dim. */ + dim: number +} + /** * The object returned by the `'vector'` provider factory — Brainy's vector * index contract. Implementations include Brainy's own JS HNSW index and any @@ -782,6 +810,15 @@ export interface VectorIndexProvider { * `generation` is the same u64 counter handed to the graph index on writes. */ generation?: bigint + /** + * 8.0 #35 part-3: the at-`generation` candidate vectors for the FILTERED + * exact-rerank — Brainy supplies the historically-correct vectors so the + * provider need not retain them. Present only alongside `generation` on the + * filtered at-gen path; the provider reranks `atGenerationVectors.ids` by + * distance and returns top-k. See {@link AtGenerationVectors}. The built-in + * JS index ignores it (it serves "now" only). + */ + atGenerationVectors?: AtGenerationVectors } ): Promise> size(): number diff --git a/tests/integration/db-asof-vector-defer.test.ts b/tests/integration/db-asof-vector-defer.test.ts index 5e73468d..09a3f3d9 100644 --- a/tests/integration/db-asof-vector-defer.test.ts +++ b/tests/integration/db-asof-vector-defer.test.ts @@ -17,7 +17,11 @@ import { createTestConfig } from '../helpers/test-factory.js' /** A versioned vector provider that advertises visibility + records each search. */ function makeVersionedVectorMock(ranking: string[]) { - const calls: Array<{ generation?: bigint; allowedIds?: ReadonlySet }> = [] + const calls: Array<{ + generation?: bigint + allowedIds?: ReadonlySet + atGenerationVectors?: { ids: BigInt64Array; vectors: Float32Array; dim: number } + }> = [] const provider = { // VersionedIndexProvider quartet (duck-typed by isVersionedIndexProvider). generation: () => 0n, @@ -26,7 +30,11 @@ function makeVersionedVectorMock(ranking: string[]) { release: (_g: bigint) => {}, // The at-gen vector leg: record the seam fields, return ranked allowed ids. search: async (_vec: number[], _k: number, _filter: unknown, opts: any) => { - calls.push({ generation: opts?.generation, allowedIds: opts?.allowedIds }) + calls.push({ + generation: opts?.generation, + allowedIds: opts?.allowedIds, + atGenerationVectors: opts?.atGenerationVectors + }) const allowed = opts?.allowedIds as ReadonlySet | undefined const out: Array<[string, number]> = [] let distance = 0.1 @@ -105,4 +113,26 @@ describe('#35 asOf filtered vector read defers to the versioned provider', () => expect(page.map((r) => r.id)).toEqual([b]) // top-1 by the provider ranking await db.release() }) + + it('supplies the at-gen candidate vectors (part-3) — row-major, dim-correct, ids = the universe', async () => { + const db = brain.now() + await brain.add({ type: NounType.Document, data: 'zeta machine learning', metadata: { team: 'z' } }) // advance gen + const mock = makeVersionedVectorMock([b, a]) + ;(brain as any).index = mock.provider + + await db.find({ query: 'machine learning', where: { team: 'x' } }) + + const agv = mock.calls[0].atGenerationVectors + expect(agv).toBeDefined() + // ids = the at-gen `team: x` universe {a, b}, interned via the shared mapper. + expect(agv!.ids.length).toBe(2) + expect(agv!.dim).toBeGreaterThan(0) + // The row-major invariant cor asserts on ingest. + expect(agv!.vectors.length).toBe(agv!.ids.length * agv!.dim) + // Each interned id resolves back to a universe entity. + const mapper = (brain as any).metadataIndex.getIdMapper() + const uuids = Array.from(agv!.ids).map((i) => mapper.getUuid(Number(i))) + expect(new Set(uuids)).toEqual(new Set([a, b])) + await db.release() + }) })