fix(find): the hybrid legs rank inside the filter, and only the page is read
A hybrid find fuses a text leg and a semantic leg. The semantic leg already
walked only the metadata filter's universe. The text leg did not: it ranked
the WHOLE store, took the top `limit * 4`, read every one of those rows from
canonical, and only then intersected with the filter. On a large store with a
selective filter that is hundreds of rows read to return a handful — and a row
matching both the query and the filter, but sitting outside the store-wide
text prefix, was silently dropped. The same defect `find({ connected })`
carried before the graph-first law, one leg over.
Both legs now rank ids inside the universe and neither reads canonical. The
text leg goes through a new optional `getIdsForTextQueryWithin` door on
MetadataIndexProvider — the text twin of `filterIdsWithin`, so a native index
can intersect its postings before any string crosses the boundary; the
reference index implements it from its own posting-list merge, so the two
doors can never disagree, and a provider without it is served by the
whole-store answer intersected here. The fusion ranks shells, the page is cut
from them, and canonical is read once for exactly that page — with the row
rebuilt in full, so a hydrated row is indistinguishable from an eagerly-built
one (same flattened fields, same entity, same match visibility, same key
order). The eager forms of both legs stay for the search modes whose leg
output IS the answer.
Measured on the production recall shape (query + type list + `missing`
negation + excludeVFS, limit 60) the old order read 241 rows in two batches to
return one; the new order reads the page.
Pinned in tests/integration/find-hybrid-filter-before-hydrate.test.ts. The
oracle there is the pre-change pipeline itself, replayed on the same brain
through the same doors: where the filter does not truncate the text leg the
answer is identical — rows, order, scores, match visibility and row shape —
across hybrid + where, + type list + excludeVFS + a `missing` negation, +
connected, with and without offset. Where it does truncate, the correction is
held by name: the old order's text leg contributed nothing at all, the new one
returns the matching rows and paging reaches every one of them. The cost pins
read the engine's own counters: one batchGet of `limit` ids, the whole-store
text door never called, and what the text leg marshals bounded by the universe.
This commit is contained in:
parent
67ae0046de
commit
b1c7054467
4 changed files with 903 additions and 118 deletions
353
src/brainy.ts
353
src/brainy.ts
|
|
@ -7651,6 +7651,13 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
|||
const searchMode = params.searchMode || 'auto'
|
||||
const limit = params.limit || 10
|
||||
|
||||
// HYDRATE LAST (the hybrid path): its legs and its fusion rank IDS, and
|
||||
// canonical is read at the two page exits below — never for a row the
|
||||
// metadata filter is about to discard. This closure re-applies a hybrid
|
||||
// row's match visibility once its entity is in hand; it is set only by
|
||||
// the hybrid branch, so every other path hydrates unchanged.
|
||||
let finishHybridRow: ((row: Result<T>, pending: Result<T>) => void) | undefined
|
||||
|
||||
// Handle text-only query (user explicitly wants text search)
|
||||
if (searchMode === 'text' && params.query && params.query.trim() !== '') {
|
||||
results = await this.executeTextSearch(params.query, limit * 2)
|
||||
|
|
@ -7661,20 +7668,32 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
|||
}
|
||||
// Handle explicit hybrid or auto mode with query
|
||||
else if ((searchMode === 'auto' || searchMode === 'hybrid') && params.query && params.query.trim() !== '' && !params.vector) {
|
||||
// Zero-config hybrid: combine text + semantic search with RRF fusion
|
||||
const [textResults, semanticResults] = await Promise.all([
|
||||
this.executeTextSearch(params.query, limit * 2),
|
||||
this.executeVectorSearch(params, preResolvedMetadataIds ?? undefined, preResolvedAllowedIds)
|
||||
// Zero-config hybrid: combine text + semantic search with RRF fusion.
|
||||
// BOTH legs are held to the metadata filter's universe: the vector leg
|
||||
// walks it as its candidate set, and the text leg ranks inside it
|
||||
// instead of ranking the whole store and discarding what the filter
|
||||
// would drop. Neither leg reads canonical — the page does, once.
|
||||
const [textScored, semanticScored] = await Promise.all([
|
||||
this.executeTextSearchScored(params.query, limit * 2, preResolvedMetadataIds ?? undefined),
|
||||
this.executeVectorSearchScored(params, preResolvedMetadataIds ?? undefined, preResolvedAllowedIds)
|
||||
])
|
||||
|
||||
// Use user-specified alpha or auto-detect based on query length
|
||||
const alpha = params.hybridAlpha ?? this.autoAlpha(params.query)
|
||||
|
||||
// Tokenize query for match visibility
|
||||
// Tokenize query for match visibility. The word list needs the entity,
|
||||
// so it is computed on the page, at hydration.
|
||||
const queryWords = this.metadataIndex.tokenize(params.query)
|
||||
const textResultIds = new Set(textScored.map((r) => r.id))
|
||||
finishHybridRow = (row, pending) => {
|
||||
row.textMatches = this.findMatchingWords(row.entity, queryWords, textResultIds)
|
||||
row.textScore = pending.textScore
|
||||
row.semanticScore = pending.semanticScore
|
||||
row.matchSource = pending.matchSource
|
||||
}
|
||||
|
||||
// RRF fusion combines both result sets with match visibility
|
||||
results = await this.rrfFusion(textResults, semanticResults, alpha, queryWords)
|
||||
// RRF fusion combines both ranked id sets with match visibility
|
||||
results = this.rrfFusion(textScored, semanticScored, alpha)
|
||||
}
|
||||
// Handle direct vector search (no query text) - no hybrid needed
|
||||
else if (params.vector && !params.query) {
|
||||
|
|
@ -7736,19 +7755,11 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
|||
const order = rankIndicesByScore(results.map(r => r.score), k, true)
|
||||
results = reorderByIndices(results, order).slice(offset, k)
|
||||
|
||||
// Batch-load entities only for the paginated results (10x faster on GCS)
|
||||
const idsToLoad = results.filter(r => !r.entity).map(r => r.id)
|
||||
if (idsToLoad.length > 0) {
|
||||
const entitiesMap = await this.batchGet(idsToLoad)
|
||||
for (const result of results) {
|
||||
if (!result.entity) {
|
||||
const entity = entitiesMap.get(result.id)
|
||||
if (entity) {
|
||||
result.entity = entity
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
// Batch-load entities only for the paginated results (10x faster on GCS).
|
||||
// This is the hydrate-last seam for the deferring paths: a row that
|
||||
// arrives as a ranked shell is rebuilt in full here — flattened
|
||||
// fields, entity and match visibility — never `entity` alone.
|
||||
results = await this.hydrateResultPage(results, finishHybridRow)
|
||||
|
||||
// Early return if no other processing needed
|
||||
if (!params.connected && !params.fusion) {
|
||||
|
|
@ -7854,8 +7865,13 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
|||
|
||||
const finalOffset = params.offset || 0
|
||||
|
||||
// Efficient pagination - only slice what we need (limit already defined above)
|
||||
return results.slice(finalOffset, finalOffset + limit)
|
||||
// Efficient pagination - only slice what we need (limit already defined
|
||||
// above), THEN read canonical for the page. Rows that arrived hydrated
|
||||
// pass straight through; a deferred path reads exactly these rows.
|
||||
return await this.hydrateResultPage(
|
||||
results.slice(finalOffset, finalOffset + limit),
|
||||
finishHybridRow
|
||||
)
|
||||
})()
|
||||
|
||||
// Index-integrity guard — applied ONCE here so every find() path (metadata,
|
||||
|
|
@ -12949,6 +12965,31 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
|||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* The text-leg twin of {@link filterIdsWithinBelted}: rank `query` INSIDE the
|
||||
* candidate universe, through the provider's own posting-list merge so the
|
||||
* answer can never drift from `getIdsForTextQuery`'s. A provider without the
|
||||
* door is served by its whole-store answer intersected here — the same rows
|
||||
* in the same order, but it pays the whole-store marshal.
|
||||
*
|
||||
* @param query - The text query.
|
||||
* @param ids - The candidate universe (the metadata filter's ids).
|
||||
* @returns `{ id, matchCount }` rows inside `ids`, ranked by match count.
|
||||
*/
|
||||
private async textIdsWithinBelted(
|
||||
query: string,
|
||||
ids: readonly string[]
|
||||
): Promise<Array<{ id: string; matchCount: number }>> {
|
||||
this.ensureIndexesLoaded(['metadata'])
|
||||
const mip = this.metadataIndex as unknown as MetadataIndexProvider
|
||||
if (typeof mip.getIdsForTextQueryWithin === 'function') {
|
||||
return await mip.getIdsForTextQueryWithin(query, ids)
|
||||
}
|
||||
const within = new Set(ids)
|
||||
const all = await this.metadataIndex.getIdsForTextQuery(query)
|
||||
return all.filter((m) => within.has(m.id))
|
||||
}
|
||||
|
||||
async getIndexStatus(): Promise<{
|
||||
initialized: boolean
|
||||
/** `true` once open()'s index-build-if-needed step has run. Named for API
|
||||
|
|
@ -15841,6 +15882,44 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
|||
candidateIds?: string[],
|
||||
allowedIds?: OpaqueIdSet
|
||||
): Promise<Result<T>[]> {
|
||||
const scored = await this.executeVectorSearchScored(params, candidateIds, allowedIds)
|
||||
|
||||
// Batch-load entities for 10-50x faster cloud storage performance
|
||||
// GCS: 10 results = 1×50ms vs 10×50ms = 500ms (10x faster)
|
||||
const entitiesMap = await this.batchGet(scored.map((s) => s.id))
|
||||
|
||||
const results: Result<T>[] = []
|
||||
for (const { id, score } of scored) {
|
||||
const entity = entitiesMap.get(id)
|
||||
if (entity) {
|
||||
results.push(this.createResult(id, score, entity))
|
||||
}
|
||||
}
|
||||
|
||||
return results
|
||||
}
|
||||
|
||||
/**
|
||||
* The semantic leg WITHOUT hydration — ranked ids and their scores.
|
||||
*
|
||||
* The beam walk is already restricted to the candidate universe (that is what
|
||||
* `candidateIds` / `allowedIds` are for), so the leg's cost is the walk. Its
|
||||
* ROWS, though, are candidates for a fusion that will keep one page of them —
|
||||
* so the hybrid path takes them unhydrated and reads exactly the page it
|
||||
* returns. {@link executeVectorSearch} is the eager form, for the search modes
|
||||
* whose leg output IS the answer.
|
||||
*
|
||||
* @param params - Find parameters (supplies the query/vector and the limit).
|
||||
* @param candidateIds - Optional pre-resolved metadata universe (see
|
||||
* {@link executeVectorSearch}).
|
||||
* @param allowedIds - Optional opaque predicate-pushdown universe.
|
||||
* @returns Ranked `{ id, score }` rows — no entity reads.
|
||||
*/
|
||||
private async executeVectorSearchScored(
|
||||
params: FindParams<T>,
|
||||
candidateIds?: string[],
|
||||
allowedIds?: OpaqueIdSet
|
||||
): Promise<Array<{ id: string; score: number }>> {
|
||||
// Vector cold-read guard: before trusting a semantic/vector result, verify the
|
||||
// vector index actually SERVES a known persisted vector (one-shot per brain).
|
||||
// A pure semantic find({ query }) has no filter, so verifyMetadataLive never
|
||||
|
|
@ -15866,21 +15945,10 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
|||
// HNSW search with optional metadata-first candidate filtering
|
||||
const searchResults: [string, number][] = await this.index.search(vector, limit * 2, undefined, searchOptions)
|
||||
|
||||
// Batch-load entities for 10-50x faster cloud storage performance
|
||||
// GCS: 10 results = 1×50ms vs 10×50ms = 500ms (10x faster)
|
||||
const ids = searchResults.map(([id]) => id)
|
||||
const entitiesMap = await this.batchGet(ids)
|
||||
|
||||
const results: Result<T>[] = []
|
||||
for (const [id, distance] of searchResults) {
|
||||
const entity = entitiesMap.get(id)
|
||||
if (entity) {
|
||||
const score = Math.max(0, Math.min(1, 1 / (1 + distance)))
|
||||
results.push(this.createResult(id, score, entity))
|
||||
}
|
||||
}
|
||||
|
||||
return results
|
||||
return searchResults.map(([id, distance]) => ({
|
||||
id,
|
||||
score: Math.max(0, Math.min(1, 1 / (1 + distance)))
|
||||
}))
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -16180,30 +16248,64 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
|||
* @returns Array of Results with scores based on match count
|
||||
*/
|
||||
private async executeTextSearch(query: string, limit: number): Promise<Result<T>[]> {
|
||||
const textMatches = await this.metadataIndex.getIdsForTextQuery(query)
|
||||
if (textMatches.length === 0) return []
|
||||
const scored = await this.executeTextSearchScored(query, limit)
|
||||
if (scored.length === 0) return []
|
||||
|
||||
// Take top matches and load entities
|
||||
const topMatches = textMatches.slice(0, limit * 2) // Get more for filtering
|
||||
const ids = topMatches.map(m => m.id)
|
||||
const entitiesMap = await this.batchGet(ids)
|
||||
// Batch-load entities for the whole leg — this is the eager form, kept for
|
||||
// the text-only search mode whose results ARE the answer.
|
||||
const entitiesMap = await this.batchGet(scored.map((s) => s.id))
|
||||
|
||||
// Create results with scores based on match count
|
||||
const maxMatches = topMatches[0]?.matchCount || 1
|
||||
const results: Result<T>[] = []
|
||||
|
||||
for (const match of topMatches) {
|
||||
const entity = entitiesMap.get(match.id)
|
||||
for (const { id, score } of scored) {
|
||||
const entity = entitiesMap.get(id)
|
||||
if (entity) {
|
||||
// Normalize score to 0-1 range based on match count
|
||||
const score = match.matchCount / maxMatches
|
||||
results.push(this.createResult(match.id, score, entity))
|
||||
results.push(this.createResult(id, score, entity))
|
||||
}
|
||||
}
|
||||
|
||||
return results
|
||||
}
|
||||
|
||||
/**
|
||||
* The text leg WITHOUT hydration — ranked ids and their scores.
|
||||
*
|
||||
* FILTER BEFORE HYDRATE: when the caller already knows the candidate
|
||||
* universe (the metadata filter's ids in a hybrid `find({ query, where })`),
|
||||
* it is passed here and the word index ranks INSIDE that universe. The
|
||||
* earlier order ranked the whole store, took the top `limit * 2`, hydrated
|
||||
* every one of them, and only then intersected with the filter — so a
|
||||
* filtered hybrid find on a large store hydrated hundreds of rows to return
|
||||
* a handful, and a matching row outside the store-wide text prefix was
|
||||
* silently dropped (the same defect `find({ connected })` had before the
|
||||
* graph-first law).
|
||||
*
|
||||
* The score is the match count normalized against the top row's, so a
|
||||
* restricted call normalizes against the top row IN THE UNIVERSE — the same
|
||||
* rule applied to the set actually being ranked.
|
||||
*
|
||||
* @param query - Text query to search for.
|
||||
* @param limit - Result budget; the leg keeps `limit * 2` for the fusion.
|
||||
* @param candidateIds - Optional candidate universe to rank inside.
|
||||
* @returns Ranked `{ id, score }` rows — no entity reads.
|
||||
*/
|
||||
private async executeTextSearchScored(
|
||||
query: string,
|
||||
limit: number,
|
||||
candidateIds?: readonly string[]
|
||||
): Promise<Array<{ id: string; score: number }>> {
|
||||
const textMatches = candidateIds
|
||||
? await this.textIdsWithinBelted(query, candidateIds)
|
||||
: await this.metadataIndex.getIdsForTextQuery(query)
|
||||
if (textMatches.length === 0) return []
|
||||
|
||||
// Take top matches (more than the page, for the fusion to rank)
|
||||
const topMatches = textMatches.slice(0, limit * 2)
|
||||
|
||||
// Normalize score to 0-1 range based on match count
|
||||
const maxMatches = topMatches[0]?.matchCount || 1
|
||||
return topMatches.map((m) => ({ id: m.id, score: m.matchCount / maxMatches }))
|
||||
}
|
||||
|
||||
/**
|
||||
* Auto-detect optimal alpha for hybrid search
|
||||
*
|
||||
|
|
@ -16230,55 +16332,56 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
|||
*
|
||||
* Formula: score(d) = sum(1 / (k + rank(d))) for each list
|
||||
*
|
||||
* Now includes match visibility (textMatches, textScore, semanticScore, matchSource)
|
||||
* Now includes match visibility (textScore, semanticScore, matchSource; the
|
||||
* `textMatches` word list needs the entity and is filled at hydration).
|
||||
*
|
||||
* @param textResults - Results from text search
|
||||
* @param semanticResults - Results from semantic search
|
||||
* HYDRATE LAST: both legs arrive as ranked ids + scores, and the fusion ranks
|
||||
* ids — no entity is read here. The rows it returns are ranked SHELLS; the
|
||||
* page is cut from them and only that page is read from canonical (see
|
||||
* {@link hydrateResultPage}). The earlier order hydrated both legs in full —
|
||||
* hundreds of rows — to return one page of them.
|
||||
*
|
||||
* @param textResults - Ranked ids + scores from text search
|
||||
* @param semanticResults - Ranked ids + scores from semantic search
|
||||
* @param alpha - Weight for semantic (0=text only, 1=semantic only)
|
||||
* @param queryWords - Original query words for match tracking
|
||||
* @param k - RRF constant (default: 60, standard in literature)
|
||||
* @returns Fused results sorted by combined score with match visibility
|
||||
* @returns Fused result shells sorted by combined score with match visibility
|
||||
*/
|
||||
private async rrfFusion(
|
||||
textResults: Result<T>[],
|
||||
semanticResults: Result<T>[],
|
||||
private rrfFusion(
|
||||
textResults: ReadonlyArray<{ id: string; score: number }>,
|
||||
semanticResults: ReadonlyArray<{ id: string; score: number }>,
|
||||
alpha: number,
|
||||
queryWords: string[],
|
||||
k: number = 60
|
||||
): Promise<Result<T>[]> {
|
||||
): Result<T>[] {
|
||||
// Track scores and match details per entity
|
||||
interface MatchData {
|
||||
rrf: number
|
||||
textScore?: number
|
||||
semanticScore?: number
|
||||
textMatches: string[]
|
||||
hasText: boolean
|
||||
hasSemantic: boolean
|
||||
}
|
||||
const matchData = new Map<string, MatchData>()
|
||||
const entityMap = new Map<string, Entity<T>>()
|
||||
|
||||
// Text contribution (1 - alpha weight)
|
||||
const textWeight = 1 - alpha
|
||||
textResults.forEach((r, rank) => {
|
||||
const rrfScore = textWeight * (1 / (k + rank + 1))
|
||||
const existing = matchData.get(r.id) || { rrf: 0, textMatches: [], hasText: false, hasSemantic: false }
|
||||
const existing = matchData.get(r.id) || { rrf: 0, hasText: false, hasSemantic: false }
|
||||
existing.rrf += rrfScore
|
||||
existing.textScore = r.score // Original text search score (0-1)
|
||||
existing.hasText = true
|
||||
matchData.set(r.id, existing)
|
||||
if (r.entity) entityMap.set(r.id, r.entity)
|
||||
})
|
||||
|
||||
// Semantic contribution (alpha weight)
|
||||
semanticResults.forEach((r, rank) => {
|
||||
const rrfScore = alpha * (1 / (k + rank + 1))
|
||||
const existing = matchData.get(r.id) || { rrf: 0, textMatches: [], hasText: false, hasSemantic: false }
|
||||
const existing = matchData.get(r.id) || { rrf: 0, hasText: false, hasSemantic: false }
|
||||
existing.rrf += rrfScore
|
||||
existing.semanticScore = r.score // Original semantic search score (0-1)
|
||||
existing.hasSemantic = true
|
||||
matchData.set(r.id, existing)
|
||||
if (r.entity) entityMap.set(r.id, r.entity)
|
||||
})
|
||||
|
||||
// Sort by fused score
|
||||
|
|
@ -16286,51 +16389,93 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
|||
.sort((a, b) => b[1].rrf - a[1].rrf)
|
||||
.map(([id, data]) => ({ id, data }))
|
||||
|
||||
// Build results - need to load any missing entities
|
||||
const missingIds = sortedIds.filter(s => !entityMap.has(s.id)).map(s => s.id)
|
||||
if (missingIds.length > 0) {
|
||||
const loaded = await this.batchGet(missingIds)
|
||||
for (const [id, entity] of loaded) {
|
||||
entityMap.set(id, entity)
|
||||
}
|
||||
}
|
||||
|
||||
// Performance: Build set of text result IDs for O(1) lookup
|
||||
// This avoids re-extracting text for entities that weren't in text results
|
||||
const textResultIds = new Set(textResults.map(r => r.id))
|
||||
|
||||
// Create final results with match visibility
|
||||
// Create ranked shells with match visibility
|
||||
const results: Result<T>[] = []
|
||||
for (const { id, data } of sortedIds) {
|
||||
const entity = entityMap.get(id)
|
||||
if (entity) {
|
||||
// Find which query words matched - uses fast path if entity wasn't in text results
|
||||
const textMatches = this.findMatchingWords(entity, queryWords, textResultIds)
|
||||
|
||||
// Determine match source
|
||||
let matchSource: 'text' | 'semantic' | 'both'
|
||||
if (data.hasText && data.hasSemantic) {
|
||||
matchSource = 'both'
|
||||
} else if (data.hasText) {
|
||||
matchSource = 'text'
|
||||
} else {
|
||||
matchSource = 'semantic'
|
||||
}
|
||||
|
||||
// Create result with match visibility
|
||||
const result = this.createResult(id, data.rrf, entity)
|
||||
result.textMatches = textMatches
|
||||
result.textScore = data.textScore
|
||||
result.semanticScore = data.semanticScore
|
||||
result.matchSource = matchSource
|
||||
|
||||
results.push(result)
|
||||
// Determine match source
|
||||
let matchSource: 'text' | 'semantic' | 'both'
|
||||
if (data.hasText && data.hasSemantic) {
|
||||
matchSource = 'both'
|
||||
} else if (data.hasText) {
|
||||
matchSource = 'text'
|
||||
} else {
|
||||
matchSource = 'semantic'
|
||||
}
|
||||
|
||||
const result = this.pendingResult(id, data.rrf)
|
||||
result.textScore = data.textScore
|
||||
result.semanticScore = data.semanticScore
|
||||
result.matchSource = matchSource
|
||||
|
||||
results.push(result)
|
||||
}
|
||||
|
||||
return results
|
||||
}
|
||||
|
||||
/**
|
||||
* A ranked candidate whose entity has NOT been read yet.
|
||||
*
|
||||
* The shell carries everything the ranking tail needs — the id, the score,
|
||||
* and the match-visibility fields — and nothing that requires canonical. It
|
||||
* is typed `Result` so it flows through the shared dedupe / visibility /
|
||||
* filter / rank / page tail unchanged; {@link hydrateResultPage} turns the
|
||||
* survivors into real results before any caller sees them, and find()'s
|
||||
* index-integrity guard drops any row that never gained an entity.
|
||||
*
|
||||
* @param id - The candidate's canonical id.
|
||||
* @param score - Its rank score.
|
||||
*/
|
||||
private pendingResult(id: string, score: number): Result<T> {
|
||||
return { id, score } as Result<T>
|
||||
}
|
||||
|
||||
/**
|
||||
* Read canonical for exactly the rows that need it — the hydrate-last seam.
|
||||
*
|
||||
* Rows that already carry an entity (the eager legs: metadata, text-only,
|
||||
* semantic-only, proximity, graph) pass through untouched, so this is a no-op
|
||||
* for every path that has not deferred. Rows that are shells are read in ONE
|
||||
* batch and rebuilt through {@link createResult}, so a hydrated row is
|
||||
* indistinguishable from an eagerly-built one — same flattened fields, same
|
||||
* `entity`, same key order — with `finish` re-applying the fields only the
|
||||
* deferring path knows about (a hybrid row's match visibility).
|
||||
*
|
||||
* A shell whose id has no canonical row is dropped, exactly as the eager legs
|
||||
* dropped it; find()'s index-integrity guard makes the same judgement on the
|
||||
* page it returns.
|
||||
*
|
||||
* @param rows - The page's rows, ranked and paged already.
|
||||
* @param finish - Applied to each rebuilt row, with its shell, after the
|
||||
* flattened fields are set.
|
||||
* @returns The page with every surviving row hydrated.
|
||||
*/
|
||||
private async hydrateResultPage(
|
||||
rows: Result<T>[],
|
||||
finish?: (row: Result<T>, pending: Result<T>) => void
|
||||
): Promise<Result<T>[]> {
|
||||
const pendingIds: string[] = []
|
||||
for (const row of rows) {
|
||||
if (!row.entity) pendingIds.push(row.id)
|
||||
}
|
||||
if (pendingIds.length === 0) return rows
|
||||
|
||||
const entitiesMap = await this.batchGet(pendingIds)
|
||||
const hydrated: Result<T>[] = []
|
||||
for (const row of rows) {
|
||||
if (row.entity) {
|
||||
hydrated.push(row)
|
||||
continue
|
||||
}
|
||||
const entity = entitiesMap.get(row.id)
|
||||
if (!entity) continue
|
||||
const filled = this.createResult(row.id, row.score, entity, row.explanation)
|
||||
finish?.(filled, row)
|
||||
hydrated.push(filled)
|
||||
}
|
||||
return hydrated
|
||||
}
|
||||
|
||||
/**
|
||||
* Find which query words match in an entity's text content
|
||||
*
|
||||
|
|
|
|||
|
|
@ -473,6 +473,28 @@ export interface MetadataIndexProvider {
|
|||
graphIndex: unknown
|
||||
): Promise<{ ids: string[]; emptyAt: 'graph' | 'filter' | 'visibility' | 'none' } | null>
|
||||
getIdsForTextQuery(query: string): Promise<Array<{ id: string; matchCount: number }>>
|
||||
/**
|
||||
* @description OPTIONAL: score `query` over `ids` ONLY — the text-leg twin of
|
||||
* {@link filterIdsWithin}, and the door a hybrid `find({ query, where })`
|
||||
* walks. The metadata filter's universe is the candidate set there, so the
|
||||
* text leg must cost O(|ids|) membership checks and marshal at most `|ids|`
|
||||
* rows, never the whole posting list of every query word. A native index
|
||||
* intersects its own postings with the candidate set (membership by entity
|
||||
* int) before any string crosses the boundary; the reference index answers
|
||||
* from its own `getIdsForTextQuery`, so the two doors can never disagree.
|
||||
* Absent → Brainy intersects `getIdsForTextQuery`'s answer with `ids` itself
|
||||
* (correct, and still hydrate-last, but it marshals the whole answer).
|
||||
*
|
||||
* The answer keeps `getIdsForTextQuery`'s contract: `{ id, matchCount }`
|
||||
* sorted by `matchCount` descending, ties in the order the whole-store answer
|
||||
* would have produced. Only rows in `ids` may appear.
|
||||
* @param query - The same text query accepted by `getIdsForTextQuery`.
|
||||
* @param ids - The candidate ids (canonical). The answer is a subset.
|
||||
*/
|
||||
getIdsForTextQueryWithin?(
|
||||
query: string,
|
||||
ids: readonly string[]
|
||||
): Promise<Array<{ id: string; matchCount: number }>>
|
||||
getSortedIdsForFilter(filter: any, orderBy: string, order?: 'asc' | 'desc', topK?: number): Promise<string[]>
|
||||
getFilterValues(field: string): Promise<string[]>
|
||||
getFilterFields(): Promise<string[]>
|
||||
|
|
|
|||
|
|
@ -1509,11 +1509,56 @@ export class MetadataIndexManager implements MetadataIndexProvider {
|
|||
* @returns Array of { id, matchCount } sorted by matchCount descending
|
||||
*/
|
||||
async getIdsForTextQuery(query: string): Promise<Array<{ id: string; matchCount: number }>> {
|
||||
return this.scoreTextQuery(query)
|
||||
}
|
||||
|
||||
/**
|
||||
* Score a text query over `ids` ONLY — the reference implementation of the
|
||||
* optional `getIdsForTextQueryWithin` door (see
|
||||
* {@link import('../plugin.js').MetadataIndexProvider}). The hybrid
|
||||
* `find({ query, where })` path passes the metadata filter's universe here so
|
||||
* the text leg ranks INSIDE that universe instead of ranking the whole store
|
||||
* and discarding the rows the filter would have dropped.
|
||||
*
|
||||
* It answers from the same posting-list merge as {@link getIdsForTextQuery},
|
||||
* with the candidate membership applied as each word's postings are counted,
|
||||
* so the two doors can never disagree: the answer is exactly the whole-store
|
||||
* answer restricted to `ids`, in the same order.
|
||||
*
|
||||
* @param query - Text query to search for.
|
||||
* @param ids - Candidate entity ids; only these may appear in the answer.
|
||||
* @returns Array of { id, matchCount } sorted by matchCount descending.
|
||||
*/
|
||||
async getIdsForTextQueryWithin(
|
||||
query: string,
|
||||
ids: readonly string[]
|
||||
): Promise<Array<{ id: string; matchCount: number }>> {
|
||||
if (ids.length === 0) return []
|
||||
return this.scoreTextQuery(query, new Set(ids))
|
||||
}
|
||||
|
||||
/**
|
||||
* The one posting-list merge behind both text doors.
|
||||
*
|
||||
* Each query word contributes AT MOST one match per entity (a posting list
|
||||
* can name an id more than once), and entities are ranked by how many of the
|
||||
* query's words they matched. `within`, when given, restricts the count to
|
||||
* those candidates — applied during the merge, so a restricted call never
|
||||
* materializes a whole-store match map.
|
||||
*
|
||||
* @param query - Text query to search for.
|
||||
* @param within - Optional candidate universe; absent = the whole store.
|
||||
* @returns Array of { id, matchCount } sorted by matchCount descending.
|
||||
*/
|
||||
private async scoreTextQuery(
|
||||
query: string,
|
||||
within?: ReadonlySet<string>
|
||||
): Promise<Array<{ id: string; matchCount: number }>> {
|
||||
const queryWords = this.tokenize(query)
|
||||
if (queryWords.length === 0) return []
|
||||
|
||||
// Get IDs for each word hash
|
||||
const wordIdSets: Map<string, number>[] = []
|
||||
// Count matches per entity, one word's postings at a time.
|
||||
const matchCounts = new Map<string, number>()
|
||||
for (const word of queryWords) {
|
||||
const wordHash = this.hashWord(word)
|
||||
let ids: string[]
|
||||
|
|
@ -1529,19 +1574,12 @@ export class MetadataIndexManager implements MetadataIndexProvider {
|
|||
throw err
|
||||
}
|
||||
}
|
||||
const idSet = new Map<string, number>()
|
||||
// One count per (word, entity) — dedupe this word's postings first.
|
||||
const counted = new Set<string>()
|
||||
for (const id of ids) {
|
||||
idSet.set(id, 1)
|
||||
}
|
||||
wordIdSets.push(idSet)
|
||||
}
|
||||
|
||||
if (wordIdSets.length === 0) return []
|
||||
|
||||
// Count matches per entity
|
||||
const matchCounts = new Map<string, number>()
|
||||
for (const idSet of wordIdSets) {
|
||||
for (const [id] of idSet) {
|
||||
if (counted.has(id)) continue
|
||||
counted.add(id)
|
||||
if (within && !within.has(id)) continue
|
||||
matchCounts.set(id, (matchCounts.get(id) || 0) + 1)
|
||||
}
|
||||
}
|
||||
|
|
|
|||
580
tests/integration/find-hybrid-filter-before-hydrate.test.ts
Normal file
580
tests/integration/find-hybrid-filter-before-hydrate.test.ts
Normal file
|
|
@ -0,0 +1,580 @@
|
|||
/**
|
||||
* @module tests/integration/find-hybrid-filter-before-hydrate
|
||||
* @description FILTER BEFORE HYDRATE, applied to the hybrid `find({ query })` path.
|
||||
*
|
||||
* A hybrid find fuses two legs. The semantic leg already walked only the
|
||||
* metadata filter's universe (`candidateIds` / `allowedIds`). The TEXT leg did
|
||||
* not: it ranked the WHOLE store, took the top `limit * 4`, read every one of
|
||||
* those rows from canonical, and only then intersected with the filter — so a
|
||||
* filtered hybrid find on a large store read hundreds of rows to return a
|
||||
* handful of them, and a matching row outside the store-wide text prefix was
|
||||
* silently dropped. That is the same defect `find({ connected })` carried
|
||||
* before the graph-first law, one leg over.
|
||||
*
|
||||
* Both halves are pinned here.
|
||||
*
|
||||
* THE ANSWER. Where the filter did not truncate the text leg — the universe
|
||||
* covers every text match, so both orders rank the same rows — the new
|
||||
* pipeline's answer is IDENTICAL to the old one's: same rows, same order, same
|
||||
* scores, same match visibility, same row shape. The oracle below is the
|
||||
* pre-change pipeline itself, replayed on the same brain through the same
|
||||
* doors, so the comparison is against what actually ran, not a remembered
|
||||
* expectation.
|
||||
*
|
||||
* THE CORRECTION. Where the filter DID truncate it — the query's words are
|
||||
* common outside the universe — the old order let the text leg contribute
|
||||
* nothing at all: every row it ranked was discarded by the filter, and the
|
||||
* answer came from the semantic leg alone. The new order ranks inside the
|
||||
* universe, so the text leg contributes the rows it always should have.
|
||||
*
|
||||
* THE COST. Canonical is read for exactly the page: one batch, `limit` rows,
|
||||
* never the legs. And the text leg is asked about the universe's ids only —
|
||||
* what it marshals is bounded by the universe, not by the store.
|
||||
*/
|
||||
import { describe, it, expect, beforeAll, vi } from 'vitest'
|
||||
import { Brainy } from '../../src/brainy'
|
||||
import { NounType, VerbType } from '../../src/types/graphTypes'
|
||||
import { rankIndicesByScore, reorderByIndices } from '../../src/utils/resultRanking'
|
||||
import { resolveEntityId } from '../../src/utils/idNormalization'
|
||||
|
||||
/** Embedding width of the default model — the row vectors must match it. */
|
||||
const DIM = 384
|
||||
|
||||
/**
|
||||
* A deterministic, per-row-distinct unit vector. Distinct so the semantic leg
|
||||
* has a real ranking to produce (identical vectors would make its order a tie
|
||||
* break), deterministic so the oracle and the pipeline see the same one.
|
||||
*/
|
||||
function seededVector(seed: number): number[] {
|
||||
const v = new Array<number>(DIM)
|
||||
for (let i = 0; i < DIM; i++) {
|
||||
v[i] = Math.sin((i + 1) * 0.11 + seed * 0.37) * 0.5 + Math.cos((i + 1) * 0.05 + seed * 0.13) * 0.3
|
||||
}
|
||||
const magnitude = Math.sqrt(v.reduce((sum, x) => sum + x * x, 0))
|
||||
return v.map((x) => x / magnitude)
|
||||
}
|
||||
|
||||
/** The fields a caller reads off a hybrid row — the whole comparable surface. */
|
||||
function project(rows: any[]): any[] {
|
||||
return rows.map((r) => ({
|
||||
id: r.id,
|
||||
score: r.score,
|
||||
type: r.type,
|
||||
metadata: r.metadata,
|
||||
textMatches: r.textMatches,
|
||||
textScore: r.textScore,
|
||||
semanticScore: r.semanticScore,
|
||||
matchSource: r.matchSource
|
||||
}))
|
||||
}
|
||||
|
||||
/**
|
||||
* The PRE-CHANGE hybrid pipeline, replayed on a live brain through the same
|
||||
* provider doors it used: whole-store text ranking with both legs hydrated in
|
||||
* full, RRF fusion, then the metadata intersection, then the page.
|
||||
*
|
||||
* Supports the shapes these pins exercise (query + where/type/excludeVFS +
|
||||
* connected + offset); `orderBy`, `fusion` and `near` are not replayed.
|
||||
*/
|
||||
async function legacyHybridFind(brain: any, params: any): Promise<any[]> {
|
||||
const index = brain.metadataIndex
|
||||
const limit = params.limit ?? 10
|
||||
const offset = params.offset ?? 0
|
||||
const hasFilter = Boolean(
|
||||
params.where || params.type || params.subtype || params.service || params.excludeVFS
|
||||
)
|
||||
|
||||
let preResolvedMetadataIds: string[] | null = null
|
||||
let preResolvedFilter: any = null
|
||||
let graphFirstIds: string[] | null = null
|
||||
|
||||
if (params.connected) {
|
||||
// find() normalizes the anchors to canonical ids before this stage runs.
|
||||
const anchored = {
|
||||
...params,
|
||||
connected: {
|
||||
...params.connected,
|
||||
...(params.connected.from && { from: resolveEntityId(params.connected.from) }),
|
||||
...(params.connected.to && { to: resolveEntityId(params.connected.to) })
|
||||
}
|
||||
}
|
||||
graphFirstIds = await brain.resolveConnectedIds(anchored)
|
||||
if (graphFirstIds!.length > 0 && hasFilter) {
|
||||
preResolvedFilter = brain.buildMetadataFilter(params)
|
||||
graphFirstIds = await brain.filterIdsWithinBelted(preResolvedFilter, graphFirstIds)
|
||||
}
|
||||
if (graphFirstIds!.length === 0) return []
|
||||
preResolvedMetadataIds = graphFirstIds
|
||||
} else if (hasFilter) {
|
||||
preResolvedFilter = brain.buildMetadataFilter(params)
|
||||
preResolvedMetadataIds = await brain.filterIdsBelted(preResolvedFilter)
|
||||
if (preResolvedMetadataIds!.length === 0) return []
|
||||
}
|
||||
|
||||
// Text leg — the whole store, then the top `limit * 4`, hydrated in full.
|
||||
const allTextMatches = await index.getIdsForTextQuery(params.query)
|
||||
const topMatches = allTextMatches.slice(0, limit * 2 * 2)
|
||||
const maxMatches = topMatches[0]?.matchCount || 1
|
||||
const textEntities = await brain.batchGet(topMatches.map((m: any) => m.id))
|
||||
const textResults = topMatches
|
||||
.filter((m: any) => textEntities.has(m.id))
|
||||
.map((m: any) => ({ id: m.id, score: m.matchCount / maxMatches }))
|
||||
|
||||
// Semantic leg — the beam walk over the universe, hydrated in full.
|
||||
const vector = await brain.embed(params.query)
|
||||
const searchOptions = preResolvedMetadataIds ? { candidateIds: preResolvedMetadataIds } : undefined
|
||||
const searchResults: [string, number][] = await brain.index.search(
|
||||
vector,
|
||||
limit * 2,
|
||||
undefined,
|
||||
searchOptions
|
||||
)
|
||||
const semanticEntities = await brain.batchGet(searchResults.map(([id]) => id))
|
||||
const semanticResults = searchResults
|
||||
.filter(([id]) => semanticEntities.has(id))
|
||||
.map(([id, distance]) => ({ id, score: Math.max(0, Math.min(1, 1 / (1 + distance))) }))
|
||||
|
||||
// RRF fusion, with the match visibility the rows carried.
|
||||
const alpha = params.hybridAlpha ?? brain.autoAlpha(params.query)
|
||||
const k = 60
|
||||
const matchData = new Map<string, any>()
|
||||
const textWeight = 1 - alpha
|
||||
textResults.forEach((r: any, rank: number) => {
|
||||
const existing = matchData.get(r.id) || { rrf: 0, hasText: false, hasSemantic: false }
|
||||
existing.rrf += textWeight * (1 / (k + rank + 1))
|
||||
existing.textScore = r.score
|
||||
existing.hasText = true
|
||||
matchData.set(r.id, existing)
|
||||
})
|
||||
semanticResults.forEach((r: any, rank: number) => {
|
||||
const existing = matchData.get(r.id) || { rrf: 0, hasText: false, hasSemantic: false }
|
||||
existing.rrf += alpha * (1 / (k + rank + 1))
|
||||
existing.semanticScore = r.score
|
||||
existing.hasSemantic = true
|
||||
matchData.set(r.id, existing)
|
||||
})
|
||||
|
||||
const queryWords: string[] = index.tokenize(params.query)
|
||||
const textResultIds = new Set(textResults.map((r: any) => r.id))
|
||||
const fusedIds = Array.from(matchData.entries())
|
||||
.sort((a, b) => b[1].rrf - a[1].rrf)
|
||||
.map(([id, data]) => ({ id, data }))
|
||||
|
||||
const allEntities = await brain.batchGet(fusedIds.map((f) => f.id))
|
||||
let rows: any[] = []
|
||||
for (const { id, data } of fusedIds) {
|
||||
const entity = allEntities.get(id)
|
||||
if (!entity) continue
|
||||
const textContent = textResultIds.has(id)
|
||||
? index.extractTextContent({ data: entity.data, metadata: entity.metadata }).toLowerCase()
|
||||
: null
|
||||
rows.push({
|
||||
id,
|
||||
score: data.rrf,
|
||||
type: entity.type,
|
||||
metadata: entity.metadata,
|
||||
textMatches:
|
||||
textContent === null ? [] : queryWords.filter((w) => textContent.includes(w.toLowerCase())),
|
||||
textScore: data.textScore,
|
||||
semanticScore: data.semanticScore,
|
||||
matchSource: data.hasText && data.hasSemantic ? 'both' : data.hasText ? 'text' : 'semantic'
|
||||
})
|
||||
}
|
||||
|
||||
// The metadata intersection — after the legs, as it was.
|
||||
if (preResolvedMetadataIds && preResolvedFilter) {
|
||||
const filteredIdSet = new Set(preResolvedMetadataIds)
|
||||
rows = rows.filter((r) => filteredIdSet.has(r.id))
|
||||
}
|
||||
if (graphFirstIds !== null) {
|
||||
const neighbourSet = new Set(graphFirstIds)
|
||||
rows = rows.filter((r) => neighbourSet.has(r.id))
|
||||
}
|
||||
|
||||
// Rank to the page, then cut it.
|
||||
const order = rankIndicesByScore(
|
||||
rows.map((r) => r.score),
|
||||
offset + limit,
|
||||
true
|
||||
)
|
||||
return reorderByIndices(rows, order).slice(offset, offset + limit)
|
||||
}
|
||||
|
||||
/**
|
||||
* FIXTURE A — the filter's universe covers every text match, so the two orders
|
||||
* rank exactly the same rows and the answers must be identical.
|
||||
*/
|
||||
describe('hybrid find: filter before hydrate — the answer is unchanged', () => {
|
||||
let brain: Brainy<any>
|
||||
const QUERY = 'orbital telemetry'
|
||||
const MATCHES = 24
|
||||
const FILLER = 120
|
||||
const OUTSIDE = 30
|
||||
const VFS = 10
|
||||
const RETRACTED = 6
|
||||
const anchor = 'array-anchor'
|
||||
const matchIds: string[] = []
|
||||
|
||||
beforeAll(async () => {
|
||||
brain = new Brainy({ requireSubtype: false, storage: { type: 'memory' } })
|
||||
await brain.init()
|
||||
|
||||
let seed = 1
|
||||
await brain.add({
|
||||
id: anchor,
|
||||
data: 'ground station anchor record',
|
||||
type: NounType.Thing,
|
||||
metadata: { lane: 'alpha', role: 'anchor' },
|
||||
vector: seededVector(seed++)
|
||||
})
|
||||
|
||||
// Rows the query's words actually match — all inside every filter below.
|
||||
for (let i = 0; i < MATCHES; i++) {
|
||||
const id = `match-${i}`
|
||||
await brain.add({
|
||||
id,
|
||||
data: `orbital telemetry packet ${i} recorded downlink`,
|
||||
type: NounType.Document,
|
||||
metadata: { lane: 'alpha', rank: i },
|
||||
vector: seededVector(seed++)
|
||||
})
|
||||
matchIds.push(resolveEntityId(id))
|
||||
await brain.relate({ from: anchor, to: id, type: VerbType.RelatedTo })
|
||||
}
|
||||
// Rows inside the universe that the query's words do NOT match.
|
||||
for (let i = 0; i < FILLER; i++) {
|
||||
await brain.add({
|
||||
id: `filler-${i}`,
|
||||
data: `cistern ledger entry ${i} archived`,
|
||||
type: NounType.Document,
|
||||
metadata: { lane: 'alpha', rank: 1000 + i },
|
||||
vector: seededVector(seed++)
|
||||
})
|
||||
}
|
||||
// Rows outside the universe.
|
||||
for (let i = 0; i < OUTSIDE; i++) {
|
||||
await brain.add({
|
||||
id: `outside-${i}`,
|
||||
data: `unrelated dossier ${i}`,
|
||||
type: NounType.Person,
|
||||
metadata: { lane: 'beta' },
|
||||
vector: seededVector(seed++)
|
||||
})
|
||||
}
|
||||
// VFS infrastructure rows — excluded by excludeVFS.
|
||||
for (let i = 0; i < VFS; i++) {
|
||||
await brain.add({
|
||||
id: `vfs-${i}`,
|
||||
data: `mounted path ${i}`,
|
||||
type: NounType.Document,
|
||||
metadata: { lane: 'alpha', vfsType: 'file' },
|
||||
vector: seededVector(seed++)
|
||||
})
|
||||
}
|
||||
// Retracted rows — excluded by a `missing` negation.
|
||||
for (let i = 0; i < RETRACTED; i++) {
|
||||
await brain.add({
|
||||
id: `retracted-${i}`,
|
||||
data: `withdrawn note ${i}`,
|
||||
type: NounType.Document,
|
||||
metadata: { lane: 'alpha', retracted: true },
|
||||
vector: seededVector(seed++)
|
||||
})
|
||||
}
|
||||
|
||||
// The reference index has no opaque-set door, so the pipeline and the
|
||||
// oracle both restrict the beam walk with the materialized candidate ids.
|
||||
expect(typeof (brain as any).metadataIndex.getIdSetForFilter).not.toBe('function')
|
||||
})
|
||||
|
||||
it('the fixture does not truncate the text leg — the universe covers every text match', async () => {
|
||||
const index = (brain as any).metadataIndex
|
||||
const textMatches = await index.getIdsForTextQuery(QUERY)
|
||||
expect(textMatches).toHaveLength(MATCHES)
|
||||
const universe = await (brain as any).filterIdsBelted({ lane: 'alpha' })
|
||||
const inUniverse = new Set(universe)
|
||||
for (const m of textMatches) expect(inUniverse.has(m.id)).toBe(true)
|
||||
})
|
||||
|
||||
it('hybrid + where: identical rows, identical order, identical scores', async () => {
|
||||
const params = { query: QUERY, where: { lane: 'alpha' }, limit: 8 }
|
||||
const expected = await legacyHybridFind(brain as any, params)
|
||||
const actual = await brain.find(params as any)
|
||||
expect(actual.length).toBe(expected.length)
|
||||
expect(project(actual)).toEqual(expected)
|
||||
})
|
||||
|
||||
it('hybrid + where + offset: identical page two', async () => {
|
||||
const params = { query: QUERY, where: { lane: 'alpha' }, limit: 6, offset: 6 }
|
||||
const expected = await legacyHybridFind(brain as any, params)
|
||||
const actual = await brain.find(params as any)
|
||||
expect(actual.length).toBe(expected.length)
|
||||
expect(project(actual)).toEqual(expected)
|
||||
})
|
||||
|
||||
it('hybrid + type list + excludeVFS + a `missing` negation: identical', async () => {
|
||||
const params = {
|
||||
query: QUERY,
|
||||
type: [NounType.Document, NounType.Person],
|
||||
excludeVFS: true,
|
||||
where: { lane: 'alpha', retracted: { missing: true } },
|
||||
limit: 8
|
||||
}
|
||||
const expected = await legacyHybridFind(brain as any, params)
|
||||
const actual = await brain.find(params as any)
|
||||
expect(actual.length).toBe(expected.length)
|
||||
expect(project(actual)).toEqual(expected)
|
||||
for (const r of actual) {
|
||||
expect(r.metadata.retracted).toBeUndefined()
|
||||
expect(r.metadata.vfsType).toBeUndefined()
|
||||
}
|
||||
})
|
||||
|
||||
it('hybrid + type list + excludeVFS + a `missing` negation, offset: identical', async () => {
|
||||
const params = {
|
||||
query: QUERY,
|
||||
type: [NounType.Document, NounType.Person],
|
||||
excludeVFS: true,
|
||||
where: { lane: 'alpha', retracted: { missing: true } },
|
||||
limit: 5,
|
||||
offset: 5
|
||||
}
|
||||
const expected = await legacyHybridFind(brain as any, params)
|
||||
const actual = await brain.find(params as any)
|
||||
expect(actual.length).toBe(expected.length)
|
||||
expect(project(actual)).toEqual(expected)
|
||||
})
|
||||
|
||||
it('hybrid + connected: identical, and never a non-neighbour', async () => {
|
||||
const params = {
|
||||
query: QUERY,
|
||||
connected: { from: anchor, direction: 'out' as const },
|
||||
where: { lane: 'alpha' },
|
||||
limit: 8
|
||||
}
|
||||
const expected = await legacyHybridFind(brain as any, params)
|
||||
const actual = await brain.find(params as any)
|
||||
expect(actual.length).toBe(expected.length)
|
||||
expect(project(actual)).toEqual(expected)
|
||||
const neighbours = new Set(matchIds)
|
||||
for (const r of actual) expect(neighbours.has(r.id)).toBe(true)
|
||||
})
|
||||
|
||||
it('a hydrated hybrid row is shaped exactly as an eagerly-built one', async () => {
|
||||
const rows = await brain.find({ query: QUERY, where: { lane: 'alpha' }, limit: 8 } as any)
|
||||
const row = rows[0]
|
||||
expect(Object.keys(row)).toEqual([
|
||||
'id',
|
||||
'score',
|
||||
'type',
|
||||
'subtype',
|
||||
'visibility',
|
||||
'metadata',
|
||||
'data',
|
||||
'confidence',
|
||||
'weight',
|
||||
'_rev',
|
||||
'entity',
|
||||
'textMatches',
|
||||
'textScore',
|
||||
'semanticScore',
|
||||
'matchSource'
|
||||
])
|
||||
// The flattened fields are projections of the entity, as always.
|
||||
expect(row.entity).toBeDefined()
|
||||
expect(row.type).toBe(row.entity.type)
|
||||
expect(row.metadata).toBe(row.entity.metadata)
|
||||
expect(row.data).toBe(row.entity.data)
|
||||
expect(row._rev).toBe(row.entity._rev)
|
||||
// The match visibility survives the deferral — every leg's fields, on the
|
||||
// rows that leg contributed, exactly as the eager pipeline set them.
|
||||
expect(['text', 'semantic', 'both']).toContain(row.matchSource)
|
||||
for (const r of rows) {
|
||||
if (r.matchSource === 'semantic') {
|
||||
expect(r.textMatches).toEqual([])
|
||||
expect(r.textScore).toBeUndefined()
|
||||
} else {
|
||||
expect(r.textMatches).toEqual(['orbital', 'telemetry'])
|
||||
expect(typeof r.textScore).toBe('number')
|
||||
}
|
||||
if (r.matchSource === 'text') {
|
||||
expect(r.semanticScore).toBeUndefined()
|
||||
} else {
|
||||
expect(typeof r.semanticScore).toBe('number')
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
it('reads canonical for the page only — one batch, `limit` rows', async () => {
|
||||
// Warm any first-read verification before the counters are read.
|
||||
await brain.find({ query: QUERY, where: { lane: 'alpha' }, limit: 1 } as any)
|
||||
|
||||
const hydrate = vi.spyOn(brain as any, 'batchGet')
|
||||
try {
|
||||
const results = await brain.find({ query: QUERY, where: { lane: 'alpha' }, limit: 10 } as any)
|
||||
expect(results).toHaveLength(10)
|
||||
expect(hydrate).toHaveBeenCalledTimes(1)
|
||||
expect((hydrate.mock.calls[0][0] as string[]).length).toBe(10)
|
||||
} finally {
|
||||
hydrate.mockRestore()
|
||||
}
|
||||
})
|
||||
|
||||
it('asks the text index about the universe only, never the whole store', async () => {
|
||||
const index = (brain as any).metadataIndex
|
||||
const wholeStore = vi.spyOn(index, 'getIdsForTextQuery')
|
||||
const within = vi.spyOn(index, 'getIdsForTextQueryWithin')
|
||||
try {
|
||||
await brain.find({ query: QUERY, where: { lane: 'alpha' }, limit: 10 } as any)
|
||||
expect(wholeStore).not.toHaveBeenCalled()
|
||||
expect(within).toHaveBeenCalledTimes(1)
|
||||
|
||||
const askedIds = within.mock.calls[0][1] as string[]
|
||||
const universe = await (brain as any).filterIdsBelted({ lane: 'alpha' })
|
||||
expect(askedIds).toHaveLength(universe.length)
|
||||
|
||||
// What the text leg marshals is bounded by the universe, not the store.
|
||||
const marshalled = (await within.mock.results[0].value) as unknown[]
|
||||
expect(marshalled.length).toBeLessThanOrEqual(universe.length)
|
||||
expect(marshalled).toHaveLength(MATCHES)
|
||||
} finally {
|
||||
wholeStore.mockRestore()
|
||||
within.mockRestore()
|
||||
}
|
||||
})
|
||||
|
||||
it('the two text doors agree: within is the whole-store answer restricted', async () => {
|
||||
const index = (brain as any).metadataIndex
|
||||
const universe: string[] = await (brain as any).filterIdsBelted({
|
||||
lane: 'alpha',
|
||||
retracted: { missing: true }
|
||||
})
|
||||
const inUniverse = new Set(universe)
|
||||
const whole = await index.getIdsForTextQuery(QUERY)
|
||||
const within = await index.getIdsForTextQueryWithin(QUERY, universe)
|
||||
expect(within).toEqual(whole.filter((m: any) => inUniverse.has(m.id)))
|
||||
expect(await index.getIdsForTextQueryWithin(QUERY, [])).toEqual([])
|
||||
})
|
||||
})
|
||||
|
||||
/**
|
||||
* FIXTURE B — the query's words are common OUTSIDE the universe, so the old
|
||||
* order's text leg was entirely consumed by rows the filter then discarded.
|
||||
* This is the corrected behaviour, held by name.
|
||||
*/
|
||||
describe('hybrid find: the text leg ranks inside the filter, not around it', () => {
|
||||
let brain: Brainy<any>
|
||||
const QUERY = 'orbital telemetry drift'
|
||||
const NOISE = 150
|
||||
const KEEP = 15
|
||||
const keepIds: string[] = []
|
||||
|
||||
beforeAll(async () => {
|
||||
brain = new Brainy({ requireSubtype: false, storage: { type: 'memory' } })
|
||||
await brain.init()
|
||||
|
||||
let seed = 5000
|
||||
// Added FIRST and matching one more query word, so they lead the
|
||||
// store-wide text ranking outright — and none of them pass the filter.
|
||||
for (let i = 0; i < NOISE; i++) {
|
||||
await brain.add({
|
||||
id: `noise-${i}`,
|
||||
data: `orbital telemetry drift report ${i}`,
|
||||
type: NounType.Document,
|
||||
metadata: { lane: 'beta' },
|
||||
vector: seededVector(seed++)
|
||||
})
|
||||
}
|
||||
for (let i = 0; i < KEEP; i++) {
|
||||
const id = `keep-${i}`
|
||||
await brain.add({
|
||||
id,
|
||||
data: `orbital telemetry summary ${i}`,
|
||||
type: NounType.Document,
|
||||
metadata: { lane: 'alpha' },
|
||||
vector: seededVector(seed++)
|
||||
})
|
||||
keepIds.push(resolveEntityId(id))
|
||||
}
|
||||
})
|
||||
|
||||
it('the old order let the filter consume the whole text leg', async () => {
|
||||
const index = (brain as any).metadataIndex
|
||||
const universe: string[] = await (brain as any).filterIdsBelted({ lane: 'alpha' })
|
||||
expect(universe).toHaveLength(KEEP)
|
||||
const inUniverse = new Set(universe)
|
||||
|
||||
// The store-wide prefix the old text leg took (limit 10 → limit * 4).
|
||||
const prefix = (await index.getIdsForTextQuery(QUERY)).slice(0, 40)
|
||||
expect(prefix).toHaveLength(40)
|
||||
expect(prefix.filter((m: any) => inUniverse.has(m.id))).toHaveLength(0)
|
||||
|
||||
// Every row the old text leg ranked was then discarded by the filter, so
|
||||
// the old answer carried NO text contribution at all — fifteen rows that
|
||||
// match the query's words exactly, and not one of them reached the page
|
||||
// through the text leg. What the old order returned was whatever the
|
||||
// semantic leg alone happened to reach.
|
||||
const legacy = await legacyHybridFind(brain as any, {
|
||||
query: QUERY,
|
||||
where: { lane: 'alpha' },
|
||||
limit: 10
|
||||
})
|
||||
for (const r of legacy) {
|
||||
expect(r.matchSource).toBe('semantic')
|
||||
expect(r.textScore).toBeUndefined()
|
||||
expect(r.textMatches).toEqual([])
|
||||
}
|
||||
})
|
||||
|
||||
it('the new order ranks the text leg inside the universe', async () => {
|
||||
const results = await brain.find({
|
||||
query: QUERY,
|
||||
where: { lane: 'alpha' },
|
||||
limit: 10
|
||||
} as any)
|
||||
|
||||
expect(results).toHaveLength(10)
|
||||
const keeps = new Set(keepIds)
|
||||
for (const r of results) {
|
||||
expect(keeps.has(r.id)).toBe(true)
|
||||
expect(r.metadata.lane).toBe('alpha')
|
||||
// The text leg is the contributor the old order threw away.
|
||||
expect(['text', 'both']).toContain(r.matchSource)
|
||||
expect(r.textScore).toBe(1)
|
||||
expect(r.textMatches).toEqual(['orbital', 'telemetry'])
|
||||
}
|
||||
})
|
||||
|
||||
it('paging reaches every matching row the old order could not see', async () => {
|
||||
const seen = new Set<string>()
|
||||
for (let offset = 0; offset < KEEP; offset += 5) {
|
||||
const page = await brain.find({
|
||||
query: QUERY,
|
||||
where: { lane: 'alpha' },
|
||||
limit: 5,
|
||||
offset
|
||||
} as any)
|
||||
expect(page).toHaveLength(5)
|
||||
for (const r of page) {
|
||||
expect(seen.has(r.id)).toBe(false)
|
||||
seen.add(r.id)
|
||||
}
|
||||
}
|
||||
expect(seen.size).toBe(KEEP)
|
||||
expect([...seen].sort()).toEqual([...keepIds].sort())
|
||||
})
|
||||
|
||||
it('reads canonical for the page only, on the truncating shape too', async () => {
|
||||
await brain.find({ query: QUERY, where: { lane: 'alpha' }, limit: 1 } as any)
|
||||
|
||||
const hydrate = vi.spyOn(brain as any, 'batchGet')
|
||||
try {
|
||||
const results = await brain.find({ query: QUERY, where: { lane: 'alpha' }, limit: 10 } as any)
|
||||
expect(results).toHaveLength(10)
|
||||
expect(hydrate).toHaveBeenCalledTimes(1)
|
||||
expect((hydrate.mock.calls[0][0] as string[]).length).toBe(10)
|
||||
} finally {
|
||||
hydrate.mockRestore()
|
||||
}
|
||||
})
|
||||
})
|
||||
Loading…
Add table
Add a link
Reference in a new issue