feat(8.0): #35 part-3 — supply at-gen candidate vectors for the native exact-rerank
Completes brainy's #35 side. When the at-gen vector gate serves a filtered semantic read, Brainy now ships the historically-correct candidate vectors so the native provider reranks against them with zero per-vector FFI crossing — the provider need not duplicate the per-generation retention Brainy already does. - New `AtGenerationVectors { ids: BigInt64Array; vectors: Float32Array; dim }` on the VectorIndexProvider.search options bag (row-major: vectors[i*dim..] == ids[i]; invariant vectors.length === ids.length*dim; ids = the candidate set; dim must match the index dim). The built-in JS index ignores it. - buildAtGenerationVectors resolves each universe id's vector AS OF the generation (the record before-image, or live getNoun when untouched since the pin — not readNounRaw, whose canonical path is empty under lazy-vector eviction), interns the id via the shared mapper, and packs row-major. Absent/vectorless/wrong-dim ids are dropped (not vector-rankable). Bounded by the filtered universe. Shape + the four clarifications (row-major, ids-are-the-candidate-set, dim-asserted, filtered-path-only) confirmed with cor in the handoff #35 thread. Inert until cor's native at-gen rerank advertises isGenerationVisible (honesty guard). Tested: the mock versioned provider now asserts it receives atGenerationVectors with the row-major invariant, correct dim, and ids resolving back to the at-gen universe. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
parent
6991bbe3d2
commit
c9e2169415
4 changed files with 132 additions and 9 deletions
|
|
@ -17,7 +17,11 @@ import { createTestConfig } from '../helpers/test-factory.js'
|
|||
|
||||
/** A versioned vector provider that advertises visibility + records each search. */
|
||||
function makeVersionedVectorMock(ranking: string[]) {
|
||||
const calls: Array<{ generation?: bigint; allowedIds?: ReadonlySet<string> }> = []
|
||||
const calls: Array<{
|
||||
generation?: bigint
|
||||
allowedIds?: ReadonlySet<string>
|
||||
atGenerationVectors?: { ids: BigInt64Array; vectors: Float32Array; dim: number }
|
||||
}> = []
|
||||
const provider = {
|
||||
// VersionedIndexProvider quartet (duck-typed by isVersionedIndexProvider).
|
||||
generation: () => 0n,
|
||||
|
|
@ -26,7 +30,11 @@ function makeVersionedVectorMock(ranking: string[]) {
|
|||
release: (_g: bigint) => {},
|
||||
// The at-gen vector leg: record the seam fields, return ranked allowed ids.
|
||||
search: async (_vec: number[], _k: number, _filter: unknown, opts: any) => {
|
||||
calls.push({ generation: opts?.generation, allowedIds: opts?.allowedIds })
|
||||
calls.push({
|
||||
generation: opts?.generation,
|
||||
allowedIds: opts?.allowedIds,
|
||||
atGenerationVectors: opts?.atGenerationVectors
|
||||
})
|
||||
const allowed = opts?.allowedIds as ReadonlySet<string> | undefined
|
||||
const out: Array<[string, number]> = []
|
||||
let distance = 0.1
|
||||
|
|
@ -105,4 +113,26 @@ describe('#35 asOf filtered vector read defers to the versioned provider', () =>
|
|||
expect(page.map((r) => r.id)).toEqual([b]) // top-1 by the provider ranking
|
||||
await db.release()
|
||||
})
|
||||
|
||||
it('supplies the at-gen candidate vectors (part-3) — row-major, dim-correct, ids = the universe', async () => {
|
||||
const db = brain.now()
|
||||
await brain.add({ type: NounType.Document, data: 'zeta machine learning', metadata: { team: 'z' } }) // advance gen
|
||||
const mock = makeVersionedVectorMock([b, a])
|
||||
;(brain as any).index = mock.provider
|
||||
|
||||
await db.find({ query: 'machine learning', where: { team: 'x' } })
|
||||
|
||||
const agv = mock.calls[0].atGenerationVectors
|
||||
expect(agv).toBeDefined()
|
||||
// ids = the at-gen `team: x` universe {a, b}, interned via the shared mapper.
|
||||
expect(agv!.ids.length).toBe(2)
|
||||
expect(agv!.dim).toBeGreaterThan(0)
|
||||
// The row-major invariant cor asserts on ingest.
|
||||
expect(agv!.vectors.length).toBe(agv!.ids.length * agv!.dim)
|
||||
// Each interned id resolves back to a universe entity.
|
||||
const mapper = (brain as any).metadataIndex.getIdMapper()
|
||||
const uuids = Array.from(agv!.ids).map((i) => mapper.getUuid(Number(i)))
|
||||
expect(new Set(uuids)).toEqual(new Set([a, b]))
|
||||
await db.release()
|
||||
})
|
||||
})
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue