diff --git a/src/brainy.ts b/src/brainy.ts index 7568a6f3..cbd3fdf0 100644 --- a/src/brainy.ts +++ b/src/brainy.ts @@ -8000,18 +8000,6 @@ export class Brainy implements BrainyInterface { const searchMode = params.searchMode || 'auto' const limit = params.limit || 10 - // HYDRATE LAST (the hybrid path): its legs and its fusion rank IDS, and - // canonical is read at the two page exits below — never for a row the - // metadata filter is about to discard. This closure re-applies a hybrid - // row's match visibility once its entity is in hand; it is set only by - // the hybrid branch, so every other path hydrates unchanged. - let finishHybridRow: ((row: Result, pending: Result) => void) | undefined - - // Set once the metadata block below has already ranked and CUT the page. - // The tail must not cut it a second time: `offset` has been consumed, and - // re-slicing a `limit`-long page by `offset` returns nothing at all. - let pagedEarly = false - // Handle text-only query (user explicitly wants text search) if (searchMode === 'text' && params.query && params.query.trim() !== '') { results = await this.executeTextSearch(params.query, limit * 2) @@ -8022,32 +8010,20 @@ export class Brainy implements BrainyInterface { } // Handle explicit hybrid or auto mode with query else if ((searchMode === 'auto' || searchMode === 'hybrid') && params.query && params.query.trim() !== '' && !params.vector) { - // Zero-config hybrid: combine text + semantic search with RRF fusion. - // BOTH legs are held to the metadata filter's universe: the vector leg - // walks it as its candidate set, and the text leg ranks inside it - // instead of ranking the whole store and discarding what the filter - // would drop. Neither leg reads canonical — the page does, once. - const [textScored, semanticScored] = await Promise.all([ - this.executeTextSearchScored(params.query, limit * 2, preResolvedMetadataIds ?? undefined), - this.executeVectorSearchScored(params, preResolvedMetadataIds ?? undefined, preResolvedAllowedIds) + // Zero-config hybrid: combine text + semantic search with RRF fusion + const [textResults, semanticResults] = await Promise.all([ + this.executeTextSearch(params.query, limit * 2), + this.executeVectorSearch(params, preResolvedMetadataIds ?? undefined, preResolvedAllowedIds) ]) // Use user-specified alpha or auto-detect based on query length const alpha = params.hybridAlpha ?? this.autoAlpha(params.query) - // Tokenize query for match visibility. The word list needs the entity, - // so it is computed on the page, at hydration. + // Tokenize query for match visibility const queryWords = this.metadataIndex.tokenize(params.query) - const textResultIds = new Set(textScored.map((r) => r.id)) - finishHybridRow = (row, pending) => { - row.textMatches = this.findMatchingWords(row.entity, queryWords, textResultIds) - row.textScore = pending.textScore - row.semanticScore = pending.semanticScore - row.matchSource = pending.matchSource - } - // RRF fusion combines both ranked id sets with match visibility - results = this.rrfFusion(textScored, semanticScored, alpha) + // RRF fusion combines both result sets with match visibility + results = await this.rrfFusion(textResults, semanticResults, alpha, queryWords) } // Handle direct vector search (no query text) - no hybrid needed else if (params.vector && !params.query) { @@ -8108,13 +8084,20 @@ export class Brainy implements BrainyInterface { const k = offset + limit const order = rankIndicesByScore(results.map(r => r.score), k, true) results = reorderByIndices(results, order).slice(offset, k) - pagedEarly = true - // Batch-load entities only for the paginated results (10x faster on GCS). - // This is the hydrate-last seam for the deferring paths: a row that - // arrives as a ranked shell is rebuilt in full here — flattened - // fields, entity and match visibility — never `entity` alone. - results = await this.hydrateResultPage(results, finishHybridRow) + // Batch-load entities only for the paginated results (10x faster on GCS) + const idsToLoad = results.filter(r => !r.entity).map(r => r.id) + if (idsToLoad.length > 0) { + const entitiesMap = await this.batchGet(idsToLoad) + for (const result of results) { + if (!result.entity) { + const entity = entitiesMap.get(result.id) + if (entity) { + result.entity = entity + } + } + } + } // Early return if no other processing needed if (!params.connected && !params.fusion) { @@ -8220,20 +8203,8 @@ export class Brainy implements BrainyInterface { const finalOffset = params.offset || 0 - // Efficient pagination - only slice what we need (limit already defined - // above), THEN read canonical for the page. Rows that arrived hydrated - // pass straight through; a deferred path reads exactly these rows. - // - // A page the metadata block already cut is NOT cut again: it holds the - // rows at [offset, offset+limit) of the ranking, so slicing it by - // `offset` a second time drops the whole page. That is how - // `find({ query, connected, where, offset })` — the shapes that reach - // here after early paging, `connected` and `fusion` — answered [] for - // every page but the first. - return await this.hydrateResultPage( - pagedEarly ? results : results.slice(finalOffset, finalOffset + limit), - finishHybridRow - ) + // Efficient pagination - only slice what we need (limit already defined above) + return results.slice(finalOffset, finalOffset + limit) })() // Index-integrity guard — applied ONCE here so every find() path (metadata, @@ -13327,31 +13298,6 @@ export class Brainy implements BrainyInterface { } } - /** - * The text-leg twin of {@link filterIdsWithinBelted}: rank `query` INSIDE the - * candidate universe, through the provider's own posting-list merge so the - * answer can never drift from `getIdsForTextQuery`'s. A provider without the - * door is served by its whole-store answer intersected here — the same rows - * in the same order, but it pays the whole-store marshal. - * - * @param query - The text query. - * @param ids - The candidate universe (the metadata filter's ids). - * @returns `{ id, matchCount }` rows inside `ids`, ranked by match count. - */ - private async textIdsWithinBelted( - query: string, - ids: readonly string[] - ): Promise> { - this.ensureIndexesLoaded(['metadata']) - const mip = this.metadataIndex as unknown as MetadataIndexProvider - if (typeof mip.getIdsForTextQueryWithin === 'function') { - return await mip.getIdsForTextQueryWithin(query, ids) - } - const within = new Set(ids) - const all = await this.metadataIndex.getIdsForTextQuery(query) - return all.filter((m) => within.has(m.id)) - } - async getIndexStatus(): Promise<{ initialized: boolean /** `true` once open()'s index-build-if-needed step has run. Named for API @@ -16244,44 +16190,6 @@ export class Brainy implements BrainyInterface { candidateIds?: string[], allowedIds?: OpaqueIdSet ): Promise[]> { - const scored = await this.executeVectorSearchScored(params, candidateIds, allowedIds) - - // Batch-load entities for 10-50x faster cloud storage performance - // GCS: 10 results = 1×50ms vs 10×50ms = 500ms (10x faster) - const entitiesMap = await this.batchGet(scored.map((s) => s.id)) - - const results: Result[] = [] - for (const { id, score } of scored) { - const entity = entitiesMap.get(id) - if (entity) { - results.push(this.createResult(id, score, entity)) - } - } - - return results - } - - /** - * The semantic leg WITHOUT hydration — ranked ids and their scores. - * - * The beam walk is already restricted to the candidate universe (that is what - * `candidateIds` / `allowedIds` are for), so the leg's cost is the walk. Its - * ROWS, though, are candidates for a fusion that will keep one page of them — - * so the hybrid path takes them unhydrated and reads exactly the page it - * returns. {@link executeVectorSearch} is the eager form, for the search modes - * whose leg output IS the answer. - * - * @param params - Find parameters (supplies the query/vector and the limit). - * @param candidateIds - Optional pre-resolved metadata universe (see - * {@link executeVectorSearch}). - * @param allowedIds - Optional opaque predicate-pushdown universe. - * @returns Ranked `{ id, score }` rows — no entity reads. - */ - private async executeVectorSearchScored( - params: FindParams, - candidateIds?: string[], - allowedIds?: OpaqueIdSet - ): Promise> { // Vector cold-read guard: before trusting a semantic/vector result, verify the // vector index actually SERVES a known persisted vector (one-shot per brain). // A pure semantic find({ query }) has no filter, so verifyMetadataLive never @@ -16307,10 +16215,21 @@ export class Brainy implements BrainyInterface { // HNSW search with optional metadata-first candidate filtering const searchResults: [string, number][] = await this.index.search(vector, limit * 2, undefined, searchOptions) - return searchResults.map(([id, distance]) => ({ - id, - score: Math.max(0, Math.min(1, 1 / (1 + distance))) - })) + // Batch-load entities for 10-50x faster cloud storage performance + // GCS: 10 results = 1×50ms vs 10×50ms = 500ms (10x faster) + const ids = searchResults.map(([id]) => id) + const entitiesMap = await this.batchGet(ids) + + const results: Result[] = [] + for (const [id, distance] of searchResults) { + const entity = entitiesMap.get(id) + if (entity) { + const score = Math.max(0, Math.min(1, 1 / (1 + distance))) + results.push(this.createResult(id, score, entity)) + } + } + + return results } /** @@ -16610,64 +16529,30 @@ export class Brainy implements BrainyInterface { * @returns Array of Results with scores based on match count */ private async executeTextSearch(query: string, limit: number): Promise[]> { - const scored = await this.executeTextSearchScored(query, limit) - if (scored.length === 0) return [] + const textMatches = await this.metadataIndex.getIdsForTextQuery(query) + if (textMatches.length === 0) return [] - // Batch-load entities for the whole leg — this is the eager form, kept for - // the text-only search mode whose results ARE the answer. - const entitiesMap = await this.batchGet(scored.map((s) => s.id)) + // Take top matches and load entities + const topMatches = textMatches.slice(0, limit * 2) // Get more for filtering + const ids = topMatches.map(m => m.id) + const entitiesMap = await this.batchGet(ids) + // Create results with scores based on match count + const maxMatches = topMatches[0]?.matchCount || 1 const results: Result[] = [] - for (const { id, score } of scored) { - const entity = entitiesMap.get(id) + + for (const match of topMatches) { + const entity = entitiesMap.get(match.id) if (entity) { - results.push(this.createResult(id, score, entity)) + // Normalize score to 0-1 range based on match count + const score = match.matchCount / maxMatches + results.push(this.createResult(match.id, score, entity)) } } return results } - /** - * The text leg WITHOUT hydration — ranked ids and their scores. - * - * FILTER BEFORE HYDRATE: when the caller already knows the candidate - * universe (the metadata filter's ids in a hybrid `find({ query, where })`), - * it is passed here and the word index ranks INSIDE that universe. The - * earlier order ranked the whole store, took the top `limit * 2`, hydrated - * every one of them, and only then intersected with the filter — so a - * filtered hybrid find on a large store hydrated hundreds of rows to return - * a handful, and a matching row outside the store-wide text prefix was - * silently dropped (the same defect `find({ connected })` had before the - * graph-first law). - * - * The score is the match count normalized against the top row's, so a - * restricted call normalizes against the top row IN THE UNIVERSE — the same - * rule applied to the set actually being ranked. - * - * @param query - Text query to search for. - * @param limit - Result budget; the leg keeps `limit * 2` for the fusion. - * @param candidateIds - Optional candidate universe to rank inside. - * @returns Ranked `{ id, score }` rows — no entity reads. - */ - private async executeTextSearchScored( - query: string, - limit: number, - candidateIds?: readonly string[] - ): Promise> { - const textMatches = candidateIds - ? await this.textIdsWithinBelted(query, candidateIds) - : await this.metadataIndex.getIdsForTextQuery(query) - if (textMatches.length === 0) return [] - - // Take top matches (more than the page, for the fusion to rank) - const topMatches = textMatches.slice(0, limit * 2) - - // Normalize score to 0-1 range based on match count - const maxMatches = topMatches[0]?.matchCount || 1 - return topMatches.map((m) => ({ id: m.id, score: m.matchCount / maxMatches })) - } - /** * Auto-detect optimal alpha for hybrid search * @@ -16694,56 +16579,55 @@ export class Brainy implements BrainyInterface { * * Formula: score(d) = sum(1 / (k + rank(d))) for each list * - * Now includes match visibility (textScore, semanticScore, matchSource; the - * `textMatches` word list needs the entity and is filled at hydration). + * Now includes match visibility (textMatches, textScore, semanticScore, matchSource) * - * HYDRATE LAST: both legs arrive as ranked ids + scores, and the fusion ranks - * ids — no entity is read here. The rows it returns are ranked SHELLS; the - * page is cut from them and only that page is read from canonical (see - * {@link hydrateResultPage}). The earlier order hydrated both legs in full — - * hundreds of rows — to return one page of them. - * - * @param textResults - Ranked ids + scores from text search - * @param semanticResults - Ranked ids + scores from semantic search + * @param textResults - Results from text search + * @param semanticResults - Results from semantic search * @param alpha - Weight for semantic (0=text only, 1=semantic only) + * @param queryWords - Original query words for match tracking * @param k - RRF constant (default: 60, standard in literature) - * @returns Fused result shells sorted by combined score with match visibility + * @returns Fused results sorted by combined score with match visibility */ - private rrfFusion( - textResults: ReadonlyArray<{ id: string; score: number }>, - semanticResults: ReadonlyArray<{ id: string; score: number }>, + private async rrfFusion( + textResults: Result[], + semanticResults: Result[], alpha: number, + queryWords: string[], k: number = 60 - ): Result[] { + ): Promise[]> { // Track scores and match details per entity interface MatchData { rrf: number textScore?: number semanticScore?: number + textMatches: string[] hasText: boolean hasSemantic: boolean } const matchData = new Map() + const entityMap = new Map>() // Text contribution (1 - alpha weight) const textWeight = 1 - alpha textResults.forEach((r, rank) => { const rrfScore = textWeight * (1 / (k + rank + 1)) - const existing = matchData.get(r.id) || { rrf: 0, hasText: false, hasSemantic: false } + const existing = matchData.get(r.id) || { rrf: 0, textMatches: [], hasText: false, hasSemantic: false } existing.rrf += rrfScore existing.textScore = r.score // Original text search score (0-1) existing.hasText = true matchData.set(r.id, existing) + if (r.entity) entityMap.set(r.id, r.entity) }) // Semantic contribution (alpha weight) semanticResults.forEach((r, rank) => { const rrfScore = alpha * (1 / (k + rank + 1)) - const existing = matchData.get(r.id) || { rrf: 0, hasText: false, hasSemantic: false } + const existing = matchData.get(r.id) || { rrf: 0, textMatches: [], hasText: false, hasSemantic: false } existing.rrf += rrfScore existing.semanticScore = r.score // Original semantic search score (0-1) existing.hasSemantic = true matchData.set(r.id, existing) + if (r.entity) entityMap.set(r.id, r.entity) }) // Sort by fused score @@ -16751,93 +16635,51 @@ export class Brainy implements BrainyInterface { .sort((a, b) => b[1].rrf - a[1].rrf) .map(([id, data]) => ({ id, data })) - // Create ranked shells with match visibility + // Build results - need to load any missing entities + const missingIds = sortedIds.filter(s => !entityMap.has(s.id)).map(s => s.id) + if (missingIds.length > 0) { + const loaded = await this.batchGet(missingIds) + for (const [id, entity] of loaded) { + entityMap.set(id, entity) + } + } + + // Performance: Build set of text result IDs for O(1) lookup + // This avoids re-extracting text for entities that weren't in text results + const textResultIds = new Set(textResults.map(r => r.id)) + + // Create final results with match visibility const results: Result[] = [] for (const { id, data } of sortedIds) { - // Determine match source - let matchSource: 'text' | 'semantic' | 'both' - if (data.hasText && data.hasSemantic) { - matchSource = 'both' - } else if (data.hasText) { - matchSource = 'text' - } else { - matchSource = 'semantic' + const entity = entityMap.get(id) + if (entity) { + // Find which query words matched - uses fast path if entity wasn't in text results + const textMatches = this.findMatchingWords(entity, queryWords, textResultIds) + + // Determine match source + let matchSource: 'text' | 'semantic' | 'both' + if (data.hasText && data.hasSemantic) { + matchSource = 'both' + } else if (data.hasText) { + matchSource = 'text' + } else { + matchSource = 'semantic' + } + + // Create result with match visibility + const result = this.createResult(id, data.rrf, entity) + result.textMatches = textMatches + result.textScore = data.textScore + result.semanticScore = data.semanticScore + result.matchSource = matchSource + + results.push(result) } - - const result = this.pendingResult(id, data.rrf) - result.textScore = data.textScore - result.semanticScore = data.semanticScore - result.matchSource = matchSource - - results.push(result) } return results } - /** - * A ranked candidate whose entity has NOT been read yet. - * - * The shell carries everything the ranking tail needs — the id, the score, - * and the match-visibility fields — and nothing that requires canonical. It - * is typed `Result` so it flows through the shared dedupe / visibility / - * filter / rank / page tail unchanged; {@link hydrateResultPage} turns the - * survivors into real results before any caller sees them, and find()'s - * index-integrity guard drops any row that never gained an entity. - * - * @param id - The candidate's canonical id. - * @param score - Its rank score. - */ - private pendingResult(id: string, score: number): Result { - return { id, score } as Result - } - - /** - * Read canonical for exactly the rows that need it — the hydrate-last seam. - * - * Rows that already carry an entity (the eager legs: metadata, text-only, - * semantic-only, proximity, graph) pass through untouched, so this is a no-op - * for every path that has not deferred. Rows that are shells are read in ONE - * batch and rebuilt through {@link createResult}, so a hydrated row is - * indistinguishable from an eagerly-built one — same flattened fields, same - * `entity`, same key order — with `finish` re-applying the fields only the - * deferring path knows about (a hybrid row's match visibility). - * - * A shell whose id has no canonical row is dropped, exactly as the eager legs - * dropped it; find()'s index-integrity guard makes the same judgement on the - * page it returns. - * - * @param rows - The page's rows, ranked and paged already. - * @param finish - Applied to each rebuilt row, with its shell, after the - * flattened fields are set. - * @returns The page with every surviving row hydrated. - */ - private async hydrateResultPage( - rows: Result[], - finish?: (row: Result, pending: Result) => void - ): Promise[]> { - const pendingIds: string[] = [] - for (const row of rows) { - if (!row.entity) pendingIds.push(row.id) - } - if (pendingIds.length === 0) return rows - - const entitiesMap = await this.batchGet(pendingIds) - const hydrated: Result[] = [] - for (const row of rows) { - if (row.entity) { - hydrated.push(row) - continue - } - const entity = entitiesMap.get(row.id) - if (!entity) continue - const filled = this.createResult(row.id, row.score, entity, row.explanation) - finish?.(filled, row) - hydrated.push(filled) - } - return hydrated - } - /** * Find which query words match in an entity's text content * diff --git a/src/plugin.ts b/src/plugin.ts index 64abfe26..54d300bb 100644 --- a/src/plugin.ts +++ b/src/plugin.ts @@ -473,28 +473,6 @@ export interface MetadataIndexProvider { graphIndex: unknown ): Promise<{ ids: string[]; emptyAt: 'graph' | 'filter' | 'visibility' | 'none' } | null> getIdsForTextQuery(query: string): Promise> - /** - * @description OPTIONAL: score `query` over `ids` ONLY — the text-leg twin of - * {@link filterIdsWithin}, and the door a hybrid `find({ query, where })` - * walks. The metadata filter's universe is the candidate set there, so the - * text leg must cost O(|ids|) membership checks and marshal at most `|ids|` - * rows, never the whole posting list of every query word. A native index - * intersects its own postings with the candidate set (membership by entity - * int) before any string crosses the boundary; the reference index answers - * from its own `getIdsForTextQuery`, so the two doors can never disagree. - * Absent → Brainy intersects `getIdsForTextQuery`'s answer with `ids` itself - * (correct, and still hydrate-last, but it marshals the whole answer). - * - * The answer keeps `getIdsForTextQuery`'s contract: `{ id, matchCount }` - * sorted by `matchCount` descending, ties in the order the whole-store answer - * would have produced. Only rows in `ids` may appear. - * @param query - The same text query accepted by `getIdsForTextQuery`. - * @param ids - The candidate ids (canonical). The answer is a subset. - */ - getIdsForTextQueryWithin?( - query: string, - ids: readonly string[] - ): Promise> getSortedIdsForFilter(filter: any, orderBy: string, order?: 'asc' | 'desc', topK?: number): Promise getFilterValues(field: string): Promise getFilterFields(): Promise diff --git a/src/utils/metadataIndex.ts b/src/utils/metadataIndex.ts index a3aa7679..0fd312e2 100644 --- a/src/utils/metadataIndex.ts +++ b/src/utils/metadataIndex.ts @@ -1509,56 +1509,11 @@ export class MetadataIndexManager implements MetadataIndexProvider { * @returns Array of { id, matchCount } sorted by matchCount descending */ async getIdsForTextQuery(query: string): Promise> { - return this.scoreTextQuery(query) - } - - /** - * Score a text query over `ids` ONLY — the reference implementation of the - * optional `getIdsForTextQueryWithin` door (see - * {@link import('../plugin.js').MetadataIndexProvider}). The hybrid - * `find({ query, where })` path passes the metadata filter's universe here so - * the text leg ranks INSIDE that universe instead of ranking the whole store - * and discarding the rows the filter would have dropped. - * - * It answers from the same posting-list merge as {@link getIdsForTextQuery}, - * with the candidate membership applied as each word's postings are counted, - * so the two doors can never disagree: the answer is exactly the whole-store - * answer restricted to `ids`, in the same order. - * - * @param query - Text query to search for. - * @param ids - Candidate entity ids; only these may appear in the answer. - * @returns Array of { id, matchCount } sorted by matchCount descending. - */ - async getIdsForTextQueryWithin( - query: string, - ids: readonly string[] - ): Promise> { - if (ids.length === 0) return [] - return this.scoreTextQuery(query, new Set(ids)) - } - - /** - * The one posting-list merge behind both text doors. - * - * Each query word contributes AT MOST one match per entity (a posting list - * can name an id more than once), and entities are ranked by how many of the - * query's words they matched. `within`, when given, restricts the count to - * those candidates — applied during the merge, so a restricted call never - * materializes a whole-store match map. - * - * @param query - Text query to search for. - * @param within - Optional candidate universe; absent = the whole store. - * @returns Array of { id, matchCount } sorted by matchCount descending. - */ - private async scoreTextQuery( - query: string, - within?: ReadonlySet - ): Promise> { const queryWords = this.tokenize(query) if (queryWords.length === 0) return [] - // Count matches per entity, one word's postings at a time. - const matchCounts = new Map() + // Get IDs for each word hash + const wordIdSets: Map[] = [] for (const word of queryWords) { const wordHash = this.hashWord(word) let ids: string[] @@ -1574,12 +1529,19 @@ export class MetadataIndexManager implements MetadataIndexProvider { throw err } } - // One count per (word, entity) — dedupe this word's postings first. - const counted = new Set() + const idSet = new Map() for (const id of ids) { - if (counted.has(id)) continue - counted.add(id) - if (within && !within.has(id)) continue + idSet.set(id, 1) + } + wordIdSets.push(idSet) + } + + if (wordIdSets.length === 0) return [] + + // Count matches per entity + const matchCounts = new Map() + for (const idSet of wordIdSets) { + for (const [id] of idSet) { matchCounts.set(id, (matchCounts.get(id) || 0) + 1) } } diff --git a/tests/integration/find-hybrid-filter-before-hydrate.test.ts b/tests/integration/find-hybrid-filter-before-hydrate.test.ts deleted file mode 100644 index 3e74f5d8..00000000 --- a/tests/integration/find-hybrid-filter-before-hydrate.test.ts +++ /dev/null @@ -1,635 +0,0 @@ -/** - * @module tests/integration/find-hybrid-filter-before-hydrate - * @description FILTER BEFORE HYDRATE, applied to the hybrid `find({ query })` path. - * - * A hybrid find fuses two legs. The semantic leg already walked only the - * metadata filter's universe (`candidateIds` / `allowedIds`). The TEXT leg did - * not: it ranked the WHOLE store, took the top `limit * 4`, read every one of - * those rows from canonical, and only then intersected with the filter — so a - * filtered hybrid find on a large store read hundreds of rows to return a - * handful of them, and a matching row outside the store-wide text prefix was - * silently dropped. That is the same defect `find({ connected })` carried - * before the graph-first law, one leg over. - * - * Both halves are pinned here. - * - * THE ANSWER. Where the filter did not truncate the text leg — the universe - * covers every text match, so both orders rank the same rows — the new - * pipeline's answer is IDENTICAL to the old one's: same rows, same order, same - * scores, same match visibility, same row shape. The oracle below is the - * pre-change pipeline itself, replayed on the same brain through the same - * doors, so the comparison is against what actually ran, not a remembered - * expectation. - * - * THE CORRECTION. Where the filter DID truncate it — the query's words are - * common outside the universe — the old order let the text leg contribute - * nothing at all: every row it ranked was discarded by the filter, and the - * answer came from the semantic leg alone. The new order ranks inside the - * universe, so the text leg contributes the rows it always should have. - * - * THE COST. Canonical is read for exactly the page: one batch, `limit` rows, - * never the legs. And the text leg is asked about the universe's ids only — - * what it marshals is bounded by the universe, not by the store. - */ -import { describe, it, expect, beforeAll, vi } from 'vitest' -import { Brainy } from '../../src/brainy' -import { NounType, VerbType } from '../../src/types/graphTypes' -import { rankIndicesByScore, reorderByIndices } from '../../src/utils/resultRanking' -import { resolveEntityId } from '../../src/utils/idNormalization' - -/** Embedding width of the default model — the row vectors must match it. */ -const DIM = 384 - -/** - * A deterministic, per-row-distinct unit vector. Distinct so the semantic leg - * has a real ranking to produce (identical vectors would make its order a tie - * break), deterministic so the oracle and the pipeline see the same one. - */ -function seededVector(seed: number): number[] { - const v = new Array(DIM) - for (let i = 0; i < DIM; i++) { - v[i] = Math.sin((i + 1) * 0.11 + seed * 0.37) * 0.5 + Math.cos((i + 1) * 0.05 + seed * 0.13) * 0.3 - } - const magnitude = Math.sqrt(v.reduce((sum, x) => sum + x * x, 0)) - return v.map((x) => x / magnitude) -} - -/** The fields a caller reads off a hybrid row — the whole comparable surface. */ -function project(rows: any[]): any[] { - return rows.map((r) => ({ - id: r.id, - score: r.score, - type: r.type, - metadata: r.metadata, - textMatches: r.textMatches, - textScore: r.textScore, - semanticScore: r.semanticScore, - matchSource: r.matchSource - })) -} - -/** - * The PRE-CHANGE hybrid pipeline, replayed on a live brain through the same - * provider doors it used: whole-store text ranking with both legs hydrated in - * full, RRF fusion, then the metadata intersection, then the page. - * - * Supports the shapes these pins exercise (query + where/type/excludeVFS + - * connected + offset); `orderBy`, `fusion` and `near` are not replayed. - */ -async function legacyHybridFind(brain: any, params: any): Promise { - const index = brain.metadataIndex - const limit = params.limit ?? 10 - const offset = params.offset ?? 0 - const hasFilter = Boolean( - params.where || params.type || params.subtype || params.service || params.excludeVFS - ) - - let preResolvedMetadataIds: string[] | null = null - let preResolvedFilter: any = null - let graphFirstIds: string[] | null = null - - if (params.connected) { - // find() normalizes the anchors to canonical ids before this stage runs. - const anchored = { - ...params, - connected: { - ...params.connected, - ...(params.connected.from && { from: resolveEntityId(params.connected.from) }), - ...(params.connected.to && { to: resolveEntityId(params.connected.to) }) - } - } - graphFirstIds = await brain.resolveConnectedIds(anchored) - if (graphFirstIds!.length > 0 && hasFilter) { - preResolvedFilter = brain.buildMetadataFilter(params) - graphFirstIds = await brain.filterIdsWithinBelted(preResolvedFilter, graphFirstIds) - } - if (graphFirstIds!.length === 0) return [] - preResolvedMetadataIds = graphFirstIds - } else if (hasFilter) { - preResolvedFilter = brain.buildMetadataFilter(params) - preResolvedMetadataIds = await brain.filterIdsBelted(preResolvedFilter) - if (preResolvedMetadataIds!.length === 0) return [] - } - - // Text leg — the whole store, then the top `limit * 4`, hydrated in full. - const allTextMatches = await index.getIdsForTextQuery(params.query) - const topMatches = allTextMatches.slice(0, limit * 2 * 2) - const maxMatches = topMatches[0]?.matchCount || 1 - const textEntities = await brain.batchGet(topMatches.map((m: any) => m.id)) - const textResults = topMatches - .filter((m: any) => textEntities.has(m.id)) - .map((m: any) => ({ id: m.id, score: m.matchCount / maxMatches })) - - // Semantic leg — the beam walk over the universe, hydrated in full. - const vector = await brain.embed(params.query) - const searchOptions = preResolvedMetadataIds ? { candidateIds: preResolvedMetadataIds } : undefined - const searchResults: [string, number][] = await brain.index.search( - vector, - limit * 2, - undefined, - searchOptions - ) - const semanticEntities = await brain.batchGet(searchResults.map(([id]) => id)) - const semanticResults = searchResults - .filter(([id]) => semanticEntities.has(id)) - .map(([id, distance]) => ({ id, score: Math.max(0, Math.min(1, 1 / (1 + distance))) })) - - // RRF fusion, with the match visibility the rows carried. - const alpha = params.hybridAlpha ?? brain.autoAlpha(params.query) - const k = 60 - const matchData = new Map() - const textWeight = 1 - alpha - textResults.forEach((r: any, rank: number) => { - const existing = matchData.get(r.id) || { rrf: 0, hasText: false, hasSemantic: false } - existing.rrf += textWeight * (1 / (k + rank + 1)) - existing.textScore = r.score - existing.hasText = true - matchData.set(r.id, existing) - }) - semanticResults.forEach((r: any, rank: number) => { - const existing = matchData.get(r.id) || { rrf: 0, hasText: false, hasSemantic: false } - existing.rrf += alpha * (1 / (k + rank + 1)) - existing.semanticScore = r.score - existing.hasSemantic = true - matchData.set(r.id, existing) - }) - - const queryWords: string[] = index.tokenize(params.query) - const textResultIds = new Set(textResults.map((r: any) => r.id)) - const fusedIds = Array.from(matchData.entries()) - .sort((a, b) => b[1].rrf - a[1].rrf) - .map(([id, data]) => ({ id, data })) - - const allEntities = await brain.batchGet(fusedIds.map((f) => f.id)) - let rows: any[] = [] - for (const { id, data } of fusedIds) { - const entity = allEntities.get(id) - if (!entity) continue - const textContent = textResultIds.has(id) - ? index.extractTextContent({ data: entity.data, metadata: entity.metadata }).toLowerCase() - : null - rows.push({ - id, - score: data.rrf, - type: entity.type, - metadata: entity.metadata, - textMatches: - textContent === null ? [] : queryWords.filter((w) => textContent.includes(w.toLowerCase())), - textScore: data.textScore, - semanticScore: data.semanticScore, - matchSource: data.hasText && data.hasSemantic ? 'both' : data.hasText ? 'text' : 'semantic' - }) - } - - // The metadata intersection — after the legs, as it was. - if (preResolvedMetadataIds && preResolvedFilter) { - const filteredIdSet = new Set(preResolvedMetadataIds) - rows = rows.filter((r) => filteredIdSet.has(r.id)) - } - if (graphFirstIds !== null) { - const neighbourSet = new Set(graphFirstIds) - rows = rows.filter((r) => neighbourSet.has(r.id)) - } - - // Rank to the page, then cut it. - const order = rankIndicesByScore( - rows.map((r) => r.score), - offset + limit, - true - ) - return reorderByIndices(rows, order).slice(offset, offset + limit) -} - -/** - * FIXTURE A — the filter's universe covers every text match, so the two orders - * rank exactly the same rows and the answers must be identical. - */ -describe('hybrid find: filter before hydrate — the answer is unchanged', () => { - let brain: Brainy - const QUERY = 'orbital telemetry' - const MATCHES = 24 - const FILLER = 120 - const OUTSIDE = 30 - const VFS = 10 - const RETRACTED = 6 - const anchor = 'array-anchor' - const matchIds: string[] = [] - - beforeAll(async () => { - brain = new Brainy({ requireSubtype: false, storage: { type: 'memory' } }) - await brain.init() - - let seed = 1 - await brain.add({ - id: anchor, - data: 'ground station anchor record', - type: NounType.Thing, - metadata: { lane: 'alpha', role: 'anchor' }, - vector: seededVector(seed++) - }) - - // Rows the query's words actually match — all inside every filter below. - for (let i = 0; i < MATCHES; i++) { - const id = `match-${i}` - await brain.add({ - id, - data: `orbital telemetry packet ${i} recorded downlink`, - type: NounType.Document, - metadata: { lane: 'alpha', rank: i }, - vector: seededVector(seed++) - }) - matchIds.push(resolveEntityId(id)) - await brain.relate({ from: anchor, to: id, type: VerbType.RelatedTo }) - } - // Rows inside the universe that the query's words do NOT match. - for (let i = 0; i < FILLER; i++) { - await brain.add({ - id: `filler-${i}`, - data: `cistern ledger entry ${i} archived`, - type: NounType.Document, - metadata: { lane: 'alpha', rank: 1000 + i }, - vector: seededVector(seed++) - }) - } - // Rows outside the universe. - for (let i = 0; i < OUTSIDE; i++) { - await brain.add({ - id: `outside-${i}`, - data: `unrelated dossier ${i}`, - type: NounType.Person, - metadata: { lane: 'beta' }, - vector: seededVector(seed++) - }) - } - // VFS infrastructure rows — excluded by excludeVFS. - for (let i = 0; i < VFS; i++) { - await brain.add({ - id: `vfs-${i}`, - data: `mounted path ${i}`, - type: NounType.Document, - metadata: { lane: 'alpha', vfsType: 'file' }, - vector: seededVector(seed++) - }) - } - // Retracted rows — excluded by a `missing` negation. - for (let i = 0; i < RETRACTED; i++) { - await brain.add({ - id: `retracted-${i}`, - data: `withdrawn note ${i}`, - type: NounType.Document, - metadata: { lane: 'alpha', retracted: true }, - vector: seededVector(seed++) - }) - } - - // The reference index has no opaque-set door, so the pipeline and the - // oracle both restrict the beam walk with the materialized candidate ids. - expect(typeof (brain as any).metadataIndex.getIdSetForFilter).not.toBe('function') - }) - - it('the fixture does not truncate the text leg — the universe covers every text match', async () => { - const index = (brain as any).metadataIndex - const textMatches = await index.getIdsForTextQuery(QUERY) - expect(textMatches).toHaveLength(MATCHES) - const universe = await (brain as any).filterIdsBelted({ lane: 'alpha' }) - const inUniverse = new Set(universe) - for (const m of textMatches) expect(inUniverse.has(m.id)).toBe(true) - }) - - it('hybrid + where: identical rows, identical order, identical scores', async () => { - const params = { query: QUERY, where: { lane: 'alpha' }, limit: 8 } - const expected = await legacyHybridFind(brain as any, params) - const actual = await brain.find(params as any) - expect(actual.length).toBe(expected.length) - expect(project(actual)).toEqual(expected) - }) - - it('hybrid + where + offset: identical page two', async () => { - const params = { query: QUERY, where: { lane: 'alpha' }, limit: 6, offset: 6 } - const expected = await legacyHybridFind(brain as any, params) - const actual = await brain.find(params as any) - expect(actual.length).toBe(expected.length) - expect(project(actual)).toEqual(expected) - }) - - it('hybrid + type list + excludeVFS + a `missing` negation: identical', async () => { - const params = { - query: QUERY, - type: [NounType.Document, NounType.Person], - excludeVFS: true, - where: { lane: 'alpha', retracted: { missing: true } }, - limit: 8 - } - const expected = await legacyHybridFind(brain as any, params) - const actual = await brain.find(params as any) - expect(actual.length).toBe(expected.length) - expect(project(actual)).toEqual(expected) - for (const r of actual) { - expect(r.metadata.retracted).toBeUndefined() - expect(r.metadata.vfsType).toBeUndefined() - } - }) - - it('hybrid + type list + excludeVFS + a `missing` negation, offset: identical', async () => { - const params = { - query: QUERY, - type: [NounType.Document, NounType.Person], - excludeVFS: true, - where: { lane: 'alpha', retracted: { missing: true } }, - limit: 5, - offset: 5 - } - const expected = await legacyHybridFind(brain as any, params) - const actual = await brain.find(params as any) - expect(actual.length).toBe(expected.length) - expect(project(actual)).toEqual(expected) - }) - - it('hybrid + connected: identical, and never a non-neighbour', async () => { - const params = { - query: QUERY, - connected: { from: anchor, direction: 'out' as const }, - where: { lane: 'alpha' }, - limit: 8 - } - const expected = await legacyHybridFind(brain as any, params) - const actual = await brain.find(params as any) - expect(actual.length).toBe(expected.length) - expect(project(actual)).toEqual(expected) - const neighbours = new Set(matchIds) - for (const r of actual) expect(neighbours.has(r.id)).toBe(true) - }) - - it('hybrid + connected + offset: page two is the page, not an empty answer', async () => { - const params = { - query: QUERY, - connected: { from: anchor, direction: 'out' as const }, - where: { lane: 'alpha' }, - limit: 5, - offset: 5 - } - const expected = await legacyHybridFind(brain as any, params) - expect(expected).toHaveLength(5) - const actual = await brain.find(params as any) - expect(actual.length).toBe(expected.length) - expect(project(actual)).toEqual(expected) - }) - - it('hybrid + connected: paging reaches every matching neighbour exactly once', async () => { - const seen = new Set() - for (let offset = 0; offset < MATCHES; offset += 6) { - const page = await brain.find({ - query: QUERY, - connected: { from: anchor, direction: 'out' as const }, - where: { lane: 'alpha' }, - limit: 6, - offset - } as any) - for (const r of page) { - expect(seen.has(r.id)).toBe(false) - seen.add(r.id) - } - } - // Every row the fused candidate set holds is reachable by paging, and the - // neighbour set is the ceiling. - expect(seen.size).toBeGreaterThanOrEqual(MATCHES) - const neighbours = new Set(matchIds) - for (const id of seen) expect(neighbours.has(id)).toBe(true) - }) - - it('hybrid + fusion + offset: page two is the page', async () => { - const plain = await brain.find({ - query: QUERY, - where: { lane: 'alpha' }, - limit: 5, - offset: 5 - } as any) - const fused = await brain.find({ - query: QUERY, - where: { lane: 'alpha' }, - fusion: 'weighted', - limit: 5, - offset: 5 - } as any) - expect(fused).toHaveLength(plain.length) - expect(fused.map((r) => r.id)).toEqual(plain.map((r) => r.id)) - }) - - it('a hydrated hybrid row is shaped exactly as an eagerly-built one', async () => { - const rows = await brain.find({ query: QUERY, where: { lane: 'alpha' }, limit: 8 } as any) - const row = rows[0] - expect(Object.keys(row)).toEqual([ - 'id', - 'score', - 'type', - 'subtype', - 'visibility', - 'metadata', - 'data', - 'confidence', - 'weight', - '_rev', - 'entity', - 'textMatches', - 'textScore', - 'semanticScore', - 'matchSource' - ]) - // The flattened fields are projections of the entity, as always. - expect(row.entity).toBeDefined() - expect(row.type).toBe(row.entity.type) - expect(row.metadata).toBe(row.entity.metadata) - expect(row.data).toBe(row.entity.data) - expect(row._rev).toBe(row.entity._rev) - // The match visibility survives the deferral — every leg's fields, on the - // rows that leg contributed, exactly as the eager pipeline set them. - expect(['text', 'semantic', 'both']).toContain(row.matchSource) - for (const r of rows) { - if (r.matchSource === 'semantic') { - expect(r.textMatches).toEqual([]) - expect(r.textScore).toBeUndefined() - } else { - expect(r.textMatches).toEqual(['orbital', 'telemetry']) - expect(typeof r.textScore).toBe('number') - } - if (r.matchSource === 'text') { - expect(r.semanticScore).toBeUndefined() - } else { - expect(typeof r.semanticScore).toBe('number') - } - } - }) - - it('reads canonical for the page only — one batch, `limit` rows', async () => { - // Warm any first-read verification before the counters are read. - await brain.find({ query: QUERY, where: { lane: 'alpha' }, limit: 1 } as any) - - const hydrate = vi.spyOn(brain as any, 'batchGet') - try { - const results = await brain.find({ query: QUERY, where: { lane: 'alpha' }, limit: 10 } as any) - expect(results).toHaveLength(10) - expect(hydrate).toHaveBeenCalledTimes(1) - expect((hydrate.mock.calls[0][0] as string[]).length).toBe(10) - } finally { - hydrate.mockRestore() - } - }) - - it('asks the text index about the universe only, never the whole store', async () => { - const index = (brain as any).metadataIndex - const wholeStore = vi.spyOn(index, 'getIdsForTextQuery') - const within = vi.spyOn(index, 'getIdsForTextQueryWithin') - try { - await brain.find({ query: QUERY, where: { lane: 'alpha' }, limit: 10 } as any) - expect(wholeStore).not.toHaveBeenCalled() - expect(within).toHaveBeenCalledTimes(1) - - const askedIds = within.mock.calls[0][1] as string[] - const universe = await (brain as any).filterIdsBelted({ lane: 'alpha' }) - expect(askedIds).toHaveLength(universe.length) - - // What the text leg marshals is bounded by the universe, not the store. - const marshalled = (await within.mock.results[0].value) as unknown[] - expect(marshalled.length).toBeLessThanOrEqual(universe.length) - expect(marshalled).toHaveLength(MATCHES) - } finally { - wholeStore.mockRestore() - within.mockRestore() - } - }) - - it('the two text doors agree: within is the whole-store answer restricted', async () => { - const index = (brain as any).metadataIndex - const universe: string[] = await (brain as any).filterIdsBelted({ - lane: 'alpha', - retracted: { missing: true } - }) - const inUniverse = new Set(universe) - const whole = await index.getIdsForTextQuery(QUERY) - const within = await index.getIdsForTextQueryWithin(QUERY, universe) - expect(within).toEqual(whole.filter((m: any) => inUniverse.has(m.id))) - expect(await index.getIdsForTextQueryWithin(QUERY, [])).toEqual([]) - }) -}) - -/** - * FIXTURE B — the query's words are common OUTSIDE the universe, so the old - * order's text leg was entirely consumed by rows the filter then discarded. - * This is the corrected behaviour, held by name. - */ -describe('hybrid find: the text leg ranks inside the filter, not around it', () => { - let brain: Brainy - const QUERY = 'orbital telemetry drift' - const NOISE = 150 - const KEEP = 15 - const keepIds: string[] = [] - - beforeAll(async () => { - brain = new Brainy({ requireSubtype: false, storage: { type: 'memory' } }) - await brain.init() - - let seed = 5000 - // Added FIRST and matching one more query word, so they lead the - // store-wide text ranking outright — and none of them pass the filter. - for (let i = 0; i < NOISE; i++) { - await brain.add({ - id: `noise-${i}`, - data: `orbital telemetry drift report ${i}`, - type: NounType.Document, - metadata: { lane: 'beta' }, - vector: seededVector(seed++) - }) - } - for (let i = 0; i < KEEP; i++) { - const id = `keep-${i}` - await brain.add({ - id, - data: `orbital telemetry summary ${i}`, - type: NounType.Document, - metadata: { lane: 'alpha' }, - vector: seededVector(seed++) - }) - keepIds.push(resolveEntityId(id)) - } - }) - - it('the old order let the filter consume the whole text leg', async () => { - const index = (brain as any).metadataIndex - const universe: string[] = await (brain as any).filterIdsBelted({ lane: 'alpha' }) - expect(universe).toHaveLength(KEEP) - const inUniverse = new Set(universe) - - // The store-wide prefix the old text leg took (limit 10 → limit * 4). - const prefix = (await index.getIdsForTextQuery(QUERY)).slice(0, 40) - expect(prefix).toHaveLength(40) - expect(prefix.filter((m: any) => inUniverse.has(m.id))).toHaveLength(0) - - // Every row the old text leg ranked was then discarded by the filter, so - // the old answer carried NO text contribution at all — fifteen rows that - // match the query's words exactly, and not one of them reached the page - // through the text leg. What the old order returned was whatever the - // semantic leg alone happened to reach. - const legacy = await legacyHybridFind(brain as any, { - query: QUERY, - where: { lane: 'alpha' }, - limit: 10 - }) - for (const r of legacy) { - expect(r.matchSource).toBe('semantic') - expect(r.textScore).toBeUndefined() - expect(r.textMatches).toEqual([]) - } - }) - - it('the new order ranks the text leg inside the universe', async () => { - const results = await brain.find({ - query: QUERY, - where: { lane: 'alpha' }, - limit: 10 - } as any) - - expect(results).toHaveLength(10) - const keeps = new Set(keepIds) - for (const r of results) { - expect(keeps.has(r.id)).toBe(true) - expect(r.metadata.lane).toBe('alpha') - // The text leg is the contributor the old order threw away. - expect(['text', 'both']).toContain(r.matchSource) - expect(r.textScore).toBe(1) - expect(r.textMatches).toEqual(['orbital', 'telemetry']) - } - }) - - it('paging reaches every matching row the old order could not see', async () => { - const seen = new Set() - for (let offset = 0; offset < KEEP; offset += 5) { - const page = await brain.find({ - query: QUERY, - where: { lane: 'alpha' }, - limit: 5, - offset - } as any) - expect(page).toHaveLength(5) - for (const r of page) { - expect(seen.has(r.id)).toBe(false) - seen.add(r.id) - } - } - expect(seen.size).toBe(KEEP) - expect([...seen].sort()).toEqual([...keepIds].sort()) - }) - - it('reads canonical for the page only, on the truncating shape too', async () => { - await brain.find({ query: QUERY, where: { lane: 'alpha' }, limit: 1 } as any) - - const hydrate = vi.spyOn(brain as any, 'batchGet') - try { - const results = await brain.find({ query: QUERY, where: { lane: 'alpha' }, limit: 10 } as any) - expect(results).toHaveLength(10) - expect(hydrate).toHaveBeenCalledTimes(1) - expect((hydrate.mock.calls[0][0] as string[]).length).toBe(10) - } finally { - hydrate.mockRestore() - } - }) -})