diff --git a/RELEASES.md b/RELEASES.md index 06a07242..2c453af7 100644 --- a/RELEASES.md +++ b/RELEASES.md @@ -73,6 +73,14 @@ no separate migration doc. **`8.0.0-rc.2` is live** — install with > matches that DO pass the filter, rather than coming back empty. No API change. With the native > `@soulcraft/cor` 3.0 stack the matched universe is forwarded as an opaque roaring buffer with > zero id materialization in TypeScript; the pure-JS path restricts the beam walk with a string set. +> - **Historical (`asOf`) semantic search can skip the rebuild.** A filtered semantic query at a +> past generation — `db.asOf(g).find({ query, where })` — no longer always rebuilds an ephemeral +> in-memory vector index over every at-`g` vector (O(n@G)). When a native versioned vector engine +> is registered and can serve the pinned generation, Brainy resolves the at-`g` filter universe from +> its record layer (no rebuild) and routes the vector leg to the engine with the matched ids + the +> generation; without one it falls back to the materialization (unchanged). The `VectorIndexProvider.search` +> options gained an optional `generation?: bigint` ("omitted = now") to carry this — additive, the +> built-in index ignores it. No public API change. > - **`find({ where, orderBy })` bounds the sort to the page.** A broad filter + `orderBy` > returning one page no longer materializes every matching sorted id (it produced the full sorted > match set, then sliced) — the page bound (`offset + limit`) is threaded into the column store's diff --git a/src/brainy.ts b/src/brainy.ts index d7805c93..e2e58962 100644 --- a/src/brainy.ts +++ b/src/brainy.ts @@ -6929,6 +6929,9 @@ export class Brainy implements BrainyInterface { persistPinned: (targetPath, generation) => this.persistPinnedGeneration(targetPath, generation), materializeAt: (generation) => this.materializeAtGeneration(generation), + canServeVectorAtGeneration: (generation) => this.canServeVectorAtGeneration(generation), + vectorSearchAtGeneration: (params, allowedIds, k, generation) => + this.vectorSearchAtGeneration(params, allowedIds, k, generation), pinGeneration: (generation) => this.pinGeneration(generation), releaseGeneration: (generation) => this.releaseGeneration(generation), registerDbForFinalization: (db, generation, closeOnRelease) => { @@ -7251,6 +7254,50 @@ export class Brainy implements BrainyInterface { } } + /** + * @description 8.0 #35 — true when the vector index is a + * {@link VersionedIndexProvider} that advertises `isGenerationVisible(generation)`: + * it can serve the at-`generation` vector leg from retained segments, so a + * historical filtered semantic read needs no O(n@G) JS-HNSW rebuild. The built-in + * `JsHnswVectorIndex` is not versioned (only ever "now"), and a native provider + * that cannot honor `generation` MUST return `false` here (refuse, never fabricate + * now-vectors-as-at-gen), so the caller falls back to materialization. + */ + private canServeVectorAtGeneration(generation: number): boolean { + return ( + isVersionedIndexProvider(this.index) && + this.index.isGenerationVisible(BigInt(generation)) + ) + } + + /** + * @description 8.0 #35 — run the vector kNN AS OF `generation`, restricted to + * `allowedIds` (the at-gen metadata∩graph universe the `Db` resolved from the + * record layer). The versioned provider serves the at-gen walk; Brainy composes + * the metadata half. Only reached when {@link canServeVectorAtGeneration} is true. + * @param params - The semantic (`query`) or explicit-`vector` find params. + * @param allowedIds - The at-gen candidate universe (membership-correct at `generation`). + * @param k - Over-fetch (page + headroom). + * @param generation - The as-of generation (Brainy's u64 commit counter). + * @returns `[id, distance]` pairs, ascending distance (descending relevance). + */ + private async vectorSearchAtGeneration( + params: FindParams, + allowedIds: ReadonlySet, + k: number, + generation: number + ): Promise> { + const vector = params.vector || (await this.embed(params.query!)) + // The provider scores the at-gen vectors restricted to `allowedIds`. (#35 part-3: + // the gen-g candidate VECTORS — `atGenerationVectors` — are supplied here once that + // seam is confirmed with cor; a provider needing them refuses the generation until + // then, so `canServeVectorAtGeneration` gates this path off in the interim.) + return this.index.search(vector, k, undefined, { + allowedIds, + generation: BigInt(generation) + }) + } + // --- Transact planner ------------------------------------------------------ /** diff --git a/src/db/db.ts b/src/db/db.ts index 7a4ce858..89b83300 100644 --- a/src/db/db.ts +++ b/src/db/db.ts @@ -144,6 +144,29 @@ export interface DbHost { * caller) and freed via the returned handle's `close()`. */ materializeAt(generation: number): Promise> + /** + * 8.0 #35 at-gen vector defer: true when the vector index is a + * {@link ../plugin.js VersionedIndexProvider} that advertised + * `isGenerationVisible(generation)` — i.e. it can serve the at-`generation` + * vector leg natively from retained segments, so a filtered semantic read needs + * NO O(n@G) materialization. False on the JS index (and whenever a native + * provider refuses the generation), so the caller falls back to `materializeAt`. + */ + canServeVectorAtGeneration(generation: number): boolean + /** + * Run the vector kNN for `params` (semantic or explicit-vector) AS OF + * `generation`, restricted to `allowedIds` (the at-gen metadata∩graph universe + * the caller resolved from the record layer). Returns `[id, distance]` pairs, + * descending relevance. Only reached when {@link DbHost.canServeVectorAtGeneration} + * is true — the provider serves the at-gen walk; Brainy composes the metadata + * half. `k` is the over-fetch (page + headroom). + */ + vectorSearchAtGeneration( + params: FindParams, + allowedIds: ReadonlySet, + k: number, + generation: number + ): Promise> /** Add one refcounted pin (store + versioned providers) on `generation`. */ pinGeneration(generation: number): void /** Release one refcounted pin (store + versioned providers) on `generation`. */ @@ -339,6 +362,12 @@ export class Db { if (this.overlay) { this.assertOverlayCompatibleFind(params) } else if (findRequiresIndexes(params)) { + // 8.0 #35: a FILTERED semantic/vector read at a historical generation can be + // served by a native at-gen vector provider (no O(n@G) materialization) — the + // metadata∩graph universe comes free from the record-overlay path, the vector + // leg from the provider. Falls through to materialization when not available. + const native = await this.tryAtGenerationVectorFind(params) + if (native !== null) return native const materialized = await this.materialize() return materialized.find(params) } @@ -416,6 +445,71 @@ export class Db { } } + /** + * @description 8.0 #35 — try to serve a FILTERED semantic/vector read at this + * historical generation via the native at-gen vector provider, with no O(n@G) + * materialization. Eligible only when: the query needs the vector leg + * (`query`/`vector`) WITH a metadata filter (the `allowedIds` source) and NO + * other index dimension (`near`/`connected`/`cursor`/`aggregate`/ + * `includeRelations`), AND the host's vector index can serve this generation + * ({@link DbHost.canServeVectorAtGeneration}). The at-gen metadata∩graph universe + * is resolved through this view's OWN record-overlay path (the metadata-only + * `find`, which costs no materialization); the provider then ranks the vector + * leg restricted to that universe, and rows are hydrated from the at-gen universe + * entities (so metadata reflects `generation`, not now). Returns `null` when + * ineligible — the caller falls back to materialization. + */ + private async tryAtGenerationVectorFind(params: FindParams): Promise[] | null> { + const wantsVector = params.query !== undefined || params.vector !== undefined + const hasOtherIndexDim = + params.near !== undefined || + params.connected !== undefined || + params.cursor !== undefined || + params.aggregate !== undefined || + params.includeRelations === true + const hasMetadataFilter = Boolean( + params.where || params.type || params.subtype || params.service || params.excludeVFS + ) + if (!wantsVector || hasOtherIndexDim || !hasMetadataFilter) return null + if (!this.host.canServeVectorAtGeneration(this.gen)) return null + + // At-gen metadata∩graph universe via the record-overlay path (no materialization): + // the same query with the vector legs stripped resolves through the metadata + // historical branch. Capped so a pathological filter can't materialize an + // unbounded universe; the provider walk is restricted to whatever the cap yields. + const universeParams: FindParams = { ...params } + delete universeParams.query + delete universeParams.vector + delete universeParams.searchMode + delete universeParams.mode + delete universeParams.orderBy + delete universeParams.order + universeParams.limit = AT_GEN_VECTOR_UNIVERSE_CAP + universeParams.offset = 0 + const universe = await this.find(universeParams) + if (universe.length === 0) return [] + + const limit = params.limit ?? 10 + const offset = params.offset ?? 0 + const allowedIds = new Set(universe.map((r) => r.id)) + const scored = await this.host.vectorSearchAtGeneration( + params, + allowedIds, + (offset + limit) * 2, + this.gen + ) + + // Compose: each at-gen metadata entity (from the universe) ranked by the + // provider's at-gen vector distance. + const byId = new Map(universe.map((r) => [r.id, r])) + const ranked: Result[] = [] + for (const [id, distance] of scored) { + const row = byId.get(id) + if (row) ranked.push({ ...row, score: Math.max(0, Math.min(1, 1 / (1 + distance))) }) + } + return ranked.slice(offset, offset + limit) + } + /** * @description Serialize part or all of this database value into a portable * `PortableGraph` document, read **as of this view's generation**. Because `export` @@ -986,6 +1080,14 @@ function findRequiresIndexes(params: FindParams): boolean { return indexOnlyFindDimensions(params).some(([, present]) => present) } +/** + * Cap on the at-gen metadata universe materialized for a native at-gen vector find + * (#35) — bounds a pathological filter; the provider's vector walk is restricted to + * whatever the cap yields. Only the metadata (no vectors) is held, so this is far + * cheaper than the full-corpus JS-HNSW rebuild it replaces. + */ +const AT_GEN_VECTOR_UNIVERSE_CAP = 100_000 + /** * @description Build a `Result` row from a record/overlay-resolved entity, * mirroring the live metadata-only find path (flattened fields, score `1.0`). diff --git a/src/hnsw/hnswIndex.ts b/src/hnsw/hnswIndex.ts index 8a42f206..10b0317a 100644 --- a/src/hnsw/hnswIndex.ts +++ b/src/hnsw/hnswIndex.ts @@ -667,6 +667,12 @@ export class JsHnswVectorIndex implements VectorIndexProvider { rerank?: { multiplier: number } candidateIds?: string[] allowedIds?: OpaqueIdSet | ReadonlySet + // 8.0 #35 at-gen seam: the JS index has no retained per-generation segments + // (it only ever holds "now"), so it ignores `generation` — exactly the + // refuse/fall-back posture the contract requires. Brainy never routes a + // historical read here (its defer-gate checks `isGenerationVisible` first), + // so an ignored `generation` cannot silently return now-vectors-as-at-gen. + generation?: bigint } ): Promise> { if (this.nouns.size === 0) { diff --git a/src/plugin.ts b/src/plugin.ts index b9b81f74..673d2f1a 100644 --- a/src/plugin.ts +++ b/src/plugin.ts @@ -769,6 +769,19 @@ export interface VectorIndexProvider { * falls back to `filter`. */ allowedIds?: OpaqueIdSet | ReadonlySet + /** + * As-of generation for historical (time-travel) reads — the vector-side + * mirror of the {@link GraphAccelerationProvider} trailing `generation?`. + * Omitted = "now" (the live index). When set, a {@link VersionedIndexProvider} + * serves the kNN / exact-rerank over its retained at-`generation` segments + * instead of the current vectors, so `db.asOf(g)` semantic queries need no + * O(n@G) JS-HNSW rebuild. Brainy only passes it when the provider advertised + * `isGenerationVisible(generation)` at pin time; a provider that does not + * honor it MUST refuse/fall back (never score current vectors as if at-gen) — + * Brainy then serves the historical leg from its own materialization. Brainy's + * `generation` is the same u64 counter handed to the graph index on writes. + */ + generation?: bigint } ): Promise> size(): number diff --git a/tests/integration/db-asof-vector-defer.test.ts b/tests/integration/db-asof-vector-defer.test.ts new file mode 100644 index 00000000..5e73468d --- /dev/null +++ b/tests/integration/db-asof-vector-defer.test.ts @@ -0,0 +1,108 @@ +/** + * @module tests/integration/db-asof-vector-defer + * @description 8.0 #35 — a FILTERED semantic/vector read at a historical generation + * defers to a native at-gen `VersionedIndexProvider` instead of the O(n@G) JS-HNSW + * materialization. Brainy resolves the at-gen metadata∩graph universe from the record + * overlay (no rebuild) and hands the provider `{ allowedIds, generation }`; the + * provider serves the vector leg, brainy composes. CI has no native provider, so this + * registers a faithful MOCK versioned vector index (it advertises visibility + records + * what it receives) to lock the routing + the seam. The JS index is NOT versioned, so + * the un-mocked path correctly falls through to materialization. + */ + +import { describe, it, expect, beforeEach, afterEach } from 'vitest' +import { Brainy } from '../../src/index.js' +import { NounType } from '../../src/types/graphTypes.js' +import { createTestConfig } from '../helpers/test-factory.js' + +/** A versioned vector provider that advertises visibility + records each search. */ +function makeVersionedVectorMock(ranking: string[]) { + const calls: Array<{ generation?: bigint; allowedIds?: ReadonlySet }> = [] + const provider = { + // VersionedIndexProvider quartet (duck-typed by isVersionedIndexProvider). + generation: () => 0n, + isGenerationVisible: (_g: bigint) => true, + pin: (_g: bigint) => {}, + release: (_g: bigint) => {}, + // The at-gen vector leg: record the seam fields, return ranked allowed ids. + search: async (_vec: number[], _k: number, _filter: unknown, opts: any) => { + calls.push({ generation: opts?.generation, allowedIds: opts?.allowedIds }) + const allowed = opts?.allowedIds as ReadonlySet | undefined + const out: Array<[string, number]> = [] + let distance = 0.1 + for (const id of ranking) { + if (!allowed || allowed.has(id)) { + out.push([id, distance]) + distance += 0.1 + } + } + return out + }, + // Unused VectorIndexProvider surface — no-op stubs so close()/flush() are safe. + addItem: async () => '', + removeItem: async () => true, + size: () => 0, + clear: () => {}, + rebuild: async () => {}, + flush: async () => 0, + getPersistMode: () => 'immediate' as const + } + return { provider, calls } +} + +describe('#35 asOf filtered vector read defers to the versioned provider', () => { + let brain: Brainy + let a: string, b: string, c: string + + beforeEach(async () => { + brain = new Brainy(createTestConfig()) + await brain.init() + a = await brain.add({ type: NounType.Document, data: 'alpha machine learning', metadata: { team: 'x' } }) + b = await brain.add({ type: NounType.Document, data: 'beta machine learning', metadata: { team: 'x' } }) + c = await brain.add({ type: NounType.Document, data: 'gamma cooking', metadata: { team: 'y' } }) + }) + + afterEach(async () => { + await brain.close() + }) + + it('routes to the provider with {allowedIds, generation} (no materialization) — at-gen universe is correct', async () => { + const db = brain.now() // pin the generation after a, b, c + // A later write advances the generation → `db` is now historical, and `later` + // (team x) did NOT exist at the pinned generation. + const later = await brain.add({ type: NounType.Document, data: 'delta machine learning', metadata: { team: 'x' } }) + + // Inject a versioned vector provider that can serve the pinned generation. + const mock = makeVersionedVectorMock([b, a]) // ranks b before a + ;(brain as any).index = mock.provider + + const results = await db.find({ query: 'machine learning', where: { team: 'x' } }) + + // Served natively — the provider was hit exactly once with the seam fields. + expect(mock.calls).toHaveLength(1) + expect(typeof mock.calls[0].generation).toBe('bigint') + + // allowedIds = the at-gen `team: x` universe = {a, b}; NOT `later` (born after the pin). + const allowed = mock.calls[0].allowedIds! + expect(allowed.has(a)).toBe(true) + expect(allowed.has(b)).toBe(true) + expect(allowed.has(later)).toBe(false) + expect(allowed.has(c)).toBe(false) // team y, filtered out + + // Results = the at-gen entities, in the provider's ranking (b before a). + expect(results.map((r) => r.id)).toEqual([b, a]) + + await db.release() + }) + + it('respects the page window on the provider-served path', async () => { + const db = brain.now() + await brain.add({ type: NounType.Document, data: 'epsilon', metadata: { team: 'z' } }) // advance gen + const mock = makeVersionedVectorMock([b, a]) + ;(brain as any).index = mock.provider + + const page = await db.find({ query: 'machine learning', where: { team: 'x' }, limit: 1 }) + expect(page.map((r) => r.id)).toEqual([b]) // top-1 by the provider ranking + await db.release() + }) +}) diff --git a/tests/unit/hnsw/allowed-ids-pushdown.test.ts b/tests/unit/hnsw/allowed-ids-pushdown.test.ts index 33c67459..183d5bd6 100644 --- a/tests/unit/hnsw/allowed-ids-pushdown.test.ts +++ b/tests/unit/hnsw/allowed-ids-pushdown.test.ts @@ -119,6 +119,14 @@ describe('#46 JS HNSW allowedIds pushdown (recall-preserving)', () => { // Unrestricted behavior: the opaque buffer is not interpreted as a filter. expect(ids.every((id) => disallowed.includes(id))).toBe(true) }) + + it('ignores the #35 `generation` field — the JS index only ever holds "now"', async () => { + // The JS index has no retained per-generation segments, so it accepts and + // ignores `generation` (the refuse/fall-back posture) rather than throwing. + const withGen = await index.search(query, 3, undefined, { generation: 42n }) + const withoutGen = await index.search(query, 3) + expect(withGen.map(([id]) => id)).toEqual(withoutGen.map(([id]) => id)) + }) }) describe('#46 find() forwards the opaque universe to the vector beam walk', () => {