Compare commits
No commits in common. "905c267c47a9515abcde4a713b3c575ec730e7b2" and "67ae0046de4063cf74bc9d57eb68adccd575ed5e" have entirely different histories.
905c267c47
...
67ae0046de
4 changed files with 118 additions and 971 deletions
338
src/brainy.ts
338
src/brainy.ts
|
|
@ -7651,18 +7651,6 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
|||
const searchMode = params.searchMode || 'auto'
|
||||
const limit = params.limit || 10
|
||||
|
||||
// HYDRATE LAST (the hybrid path): its legs and its fusion rank IDS, and
|
||||
// canonical is read at the two page exits below — never for a row the
|
||||
// metadata filter is about to discard. This closure re-applies a hybrid
|
||||
// row's match visibility once its entity is in hand; it is set only by
|
||||
// the hybrid branch, so every other path hydrates unchanged.
|
||||
let finishHybridRow: ((row: Result<T>, pending: Result<T>) => void) | undefined
|
||||
|
||||
// Set once the metadata block below has already ranked and CUT the page.
|
||||
// The tail must not cut it a second time: `offset` has been consumed, and
|
||||
// re-slicing a `limit`-long page by `offset` returns nothing at all.
|
||||
let pagedEarly = false
|
||||
|
||||
// Handle text-only query (user explicitly wants text search)
|
||||
if (searchMode === 'text' && params.query && params.query.trim() !== '') {
|
||||
results = await this.executeTextSearch(params.query, limit * 2)
|
||||
|
|
@ -7673,32 +7661,20 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
|||
}
|
||||
// Handle explicit hybrid or auto mode with query
|
||||
else if ((searchMode === 'auto' || searchMode === 'hybrid') && params.query && params.query.trim() !== '' && !params.vector) {
|
||||
// Zero-config hybrid: combine text + semantic search with RRF fusion.
|
||||
// BOTH legs are held to the metadata filter's universe: the vector leg
|
||||
// walks it as its candidate set, and the text leg ranks inside it
|
||||
// instead of ranking the whole store and discarding what the filter
|
||||
// would drop. Neither leg reads canonical — the page does, once.
|
||||
const [textScored, semanticScored] = await Promise.all([
|
||||
this.executeTextSearchScored(params.query, limit * 2, preResolvedMetadataIds ?? undefined),
|
||||
this.executeVectorSearchScored(params, preResolvedMetadataIds ?? undefined, preResolvedAllowedIds)
|
||||
// Zero-config hybrid: combine text + semantic search with RRF fusion
|
||||
const [textResults, semanticResults] = await Promise.all([
|
||||
this.executeTextSearch(params.query, limit * 2),
|
||||
this.executeVectorSearch(params, preResolvedMetadataIds ?? undefined, preResolvedAllowedIds)
|
||||
])
|
||||
|
||||
// Use user-specified alpha or auto-detect based on query length
|
||||
const alpha = params.hybridAlpha ?? this.autoAlpha(params.query)
|
||||
|
||||
// Tokenize query for match visibility. The word list needs the entity,
|
||||
// so it is computed on the page, at hydration.
|
||||
// Tokenize query for match visibility
|
||||
const queryWords = this.metadataIndex.tokenize(params.query)
|
||||
const textResultIds = new Set(textScored.map((r) => r.id))
|
||||
finishHybridRow = (row, pending) => {
|
||||
row.textMatches = this.findMatchingWords(row.entity, queryWords, textResultIds)
|
||||
row.textScore = pending.textScore
|
||||
row.semanticScore = pending.semanticScore
|
||||
row.matchSource = pending.matchSource
|
||||
}
|
||||
|
||||
// RRF fusion combines both ranked id sets with match visibility
|
||||
results = this.rrfFusion(textScored, semanticScored, alpha)
|
||||
// RRF fusion combines both result sets with match visibility
|
||||
results = await this.rrfFusion(textResults, semanticResults, alpha, queryWords)
|
||||
}
|
||||
// Handle direct vector search (no query text) - no hybrid needed
|
||||
else if (params.vector && !params.query) {
|
||||
|
|
@ -7759,13 +7735,20 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
|||
const k = offset + limit
|
||||
const order = rankIndicesByScore(results.map(r => r.score), k, true)
|
||||
results = reorderByIndices(results, order).slice(offset, k)
|
||||
pagedEarly = true
|
||||
|
||||
// Batch-load entities only for the paginated results (10x faster on GCS).
|
||||
// This is the hydrate-last seam for the deferring paths: a row that
|
||||
// arrives as a ranked shell is rebuilt in full here — flattened
|
||||
// fields, entity and match visibility — never `entity` alone.
|
||||
results = await this.hydrateResultPage(results, finishHybridRow)
|
||||
// Batch-load entities only for the paginated results (10x faster on GCS)
|
||||
const idsToLoad = results.filter(r => !r.entity).map(r => r.id)
|
||||
if (idsToLoad.length > 0) {
|
||||
const entitiesMap = await this.batchGet(idsToLoad)
|
||||
for (const result of results) {
|
||||
if (!result.entity) {
|
||||
const entity = entitiesMap.get(result.id)
|
||||
if (entity) {
|
||||
result.entity = entity
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Early return if no other processing needed
|
||||
if (!params.connected && !params.fusion) {
|
||||
|
|
@ -7871,20 +7854,8 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
|||
|
||||
const finalOffset = params.offset || 0
|
||||
|
||||
// Efficient pagination - only slice what we need (limit already defined
|
||||
// above), THEN read canonical for the page. Rows that arrived hydrated
|
||||
// pass straight through; a deferred path reads exactly these rows.
|
||||
//
|
||||
// A page the metadata block already cut is NOT cut again: it holds the
|
||||
// rows at [offset, offset+limit) of the ranking, so slicing it by
|
||||
// `offset` a second time drops the whole page. That is how
|
||||
// `find({ query, connected, where, offset })` — the shapes that reach
|
||||
// here after early paging, `connected` and `fusion` — answered [] for
|
||||
// every page but the first.
|
||||
return await this.hydrateResultPage(
|
||||
pagedEarly ? results : results.slice(finalOffset, finalOffset + limit),
|
||||
finishHybridRow
|
||||
)
|
||||
// Efficient pagination - only slice what we need (limit already defined above)
|
||||
return results.slice(finalOffset, finalOffset + limit)
|
||||
})()
|
||||
|
||||
// Index-integrity guard — applied ONCE here so every find() path (metadata,
|
||||
|
|
@ -12978,31 +12949,6 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
|||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* The text-leg twin of {@link filterIdsWithinBelted}: rank `query` INSIDE the
|
||||
* candidate universe, through the provider's own posting-list merge so the
|
||||
* answer can never drift from `getIdsForTextQuery`'s. A provider without the
|
||||
* door is served by its whole-store answer intersected here — the same rows
|
||||
* in the same order, but it pays the whole-store marshal.
|
||||
*
|
||||
* @param query - The text query.
|
||||
* @param ids - The candidate universe (the metadata filter's ids).
|
||||
* @returns `{ id, matchCount }` rows inside `ids`, ranked by match count.
|
||||
*/
|
||||
private async textIdsWithinBelted(
|
||||
query: string,
|
||||
ids: readonly string[]
|
||||
): Promise<Array<{ id: string; matchCount: number }>> {
|
||||
this.ensureIndexesLoaded(['metadata'])
|
||||
const mip = this.metadataIndex as unknown as MetadataIndexProvider
|
||||
if (typeof mip.getIdsForTextQueryWithin === 'function') {
|
||||
return await mip.getIdsForTextQueryWithin(query, ids)
|
||||
}
|
||||
const within = new Set(ids)
|
||||
const all = await this.metadataIndex.getIdsForTextQuery(query)
|
||||
return all.filter((m) => within.has(m.id))
|
||||
}
|
||||
|
||||
async getIndexStatus(): Promise<{
|
||||
initialized: boolean
|
||||
/** `true` once open()'s index-build-if-needed step has run. Named for API
|
||||
|
|
@ -15895,44 +15841,6 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
|||
candidateIds?: string[],
|
||||
allowedIds?: OpaqueIdSet
|
||||
): Promise<Result<T>[]> {
|
||||
const scored = await this.executeVectorSearchScored(params, candidateIds, allowedIds)
|
||||
|
||||
// Batch-load entities for 10-50x faster cloud storage performance
|
||||
// GCS: 10 results = 1×50ms vs 10×50ms = 500ms (10x faster)
|
||||
const entitiesMap = await this.batchGet(scored.map((s) => s.id))
|
||||
|
||||
const results: Result<T>[] = []
|
||||
for (const { id, score } of scored) {
|
||||
const entity = entitiesMap.get(id)
|
||||
if (entity) {
|
||||
results.push(this.createResult(id, score, entity))
|
||||
}
|
||||
}
|
||||
|
||||
return results
|
||||
}
|
||||
|
||||
/**
|
||||
* The semantic leg WITHOUT hydration — ranked ids and their scores.
|
||||
*
|
||||
* The beam walk is already restricted to the candidate universe (that is what
|
||||
* `candidateIds` / `allowedIds` are for), so the leg's cost is the walk. Its
|
||||
* ROWS, though, are candidates for a fusion that will keep one page of them —
|
||||
* so the hybrid path takes them unhydrated and reads exactly the page it
|
||||
* returns. {@link executeVectorSearch} is the eager form, for the search modes
|
||||
* whose leg output IS the answer.
|
||||
*
|
||||
* @param params - Find parameters (supplies the query/vector and the limit).
|
||||
* @param candidateIds - Optional pre-resolved metadata universe (see
|
||||
* {@link executeVectorSearch}).
|
||||
* @param allowedIds - Optional opaque predicate-pushdown universe.
|
||||
* @returns Ranked `{ id, score }` rows — no entity reads.
|
||||
*/
|
||||
private async executeVectorSearchScored(
|
||||
params: FindParams<T>,
|
||||
candidateIds?: string[],
|
||||
allowedIds?: OpaqueIdSet
|
||||
): Promise<Array<{ id: string; score: number }>> {
|
||||
// Vector cold-read guard: before trusting a semantic/vector result, verify the
|
||||
// vector index actually SERVES a known persisted vector (one-shot per brain).
|
||||
// A pure semantic find({ query }) has no filter, so verifyMetadataLive never
|
||||
|
|
@ -15958,10 +15866,21 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
|||
// HNSW search with optional metadata-first candidate filtering
|
||||
const searchResults: [string, number][] = await this.index.search(vector, limit * 2, undefined, searchOptions)
|
||||
|
||||
return searchResults.map(([id, distance]) => ({
|
||||
id,
|
||||
score: Math.max(0, Math.min(1, 1 / (1 + distance)))
|
||||
}))
|
||||
// Batch-load entities for 10-50x faster cloud storage performance
|
||||
// GCS: 10 results = 1×50ms vs 10×50ms = 500ms (10x faster)
|
||||
const ids = searchResults.map(([id]) => id)
|
||||
const entitiesMap = await this.batchGet(ids)
|
||||
|
||||
const results: Result<T>[] = []
|
||||
for (const [id, distance] of searchResults) {
|
||||
const entity = entitiesMap.get(id)
|
||||
if (entity) {
|
||||
const score = Math.max(0, Math.min(1, 1 / (1 + distance)))
|
||||
results.push(this.createResult(id, score, entity))
|
||||
}
|
||||
}
|
||||
|
||||
return results
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -16261,64 +16180,30 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
|||
* @returns Array of Results with scores based on match count
|
||||
*/
|
||||
private async executeTextSearch(query: string, limit: number): Promise<Result<T>[]> {
|
||||
const scored = await this.executeTextSearchScored(query, limit)
|
||||
if (scored.length === 0) return []
|
||||
const textMatches = await this.metadataIndex.getIdsForTextQuery(query)
|
||||
if (textMatches.length === 0) return []
|
||||
|
||||
// Batch-load entities for the whole leg — this is the eager form, kept for
|
||||
// the text-only search mode whose results ARE the answer.
|
||||
const entitiesMap = await this.batchGet(scored.map((s) => s.id))
|
||||
// Take top matches and load entities
|
||||
const topMatches = textMatches.slice(0, limit * 2) // Get more for filtering
|
||||
const ids = topMatches.map(m => m.id)
|
||||
const entitiesMap = await this.batchGet(ids)
|
||||
|
||||
// Create results with scores based on match count
|
||||
const maxMatches = topMatches[0]?.matchCount || 1
|
||||
const results: Result<T>[] = []
|
||||
for (const { id, score } of scored) {
|
||||
const entity = entitiesMap.get(id)
|
||||
|
||||
for (const match of topMatches) {
|
||||
const entity = entitiesMap.get(match.id)
|
||||
if (entity) {
|
||||
results.push(this.createResult(id, score, entity))
|
||||
// Normalize score to 0-1 range based on match count
|
||||
const score = match.matchCount / maxMatches
|
||||
results.push(this.createResult(match.id, score, entity))
|
||||
}
|
||||
}
|
||||
|
||||
return results
|
||||
}
|
||||
|
||||
/**
|
||||
* The text leg WITHOUT hydration — ranked ids and their scores.
|
||||
*
|
||||
* FILTER BEFORE HYDRATE: when the caller already knows the candidate
|
||||
* universe (the metadata filter's ids in a hybrid `find({ query, where })`),
|
||||
* it is passed here and the word index ranks INSIDE that universe. The
|
||||
* earlier order ranked the whole store, took the top `limit * 2`, hydrated
|
||||
* every one of them, and only then intersected with the filter — so a
|
||||
* filtered hybrid find on a large store hydrated hundreds of rows to return
|
||||
* a handful, and a matching row outside the store-wide text prefix was
|
||||
* silently dropped (the same defect `find({ connected })` had before the
|
||||
* graph-first law).
|
||||
*
|
||||
* The score is the match count normalized against the top row's, so a
|
||||
* restricted call normalizes against the top row IN THE UNIVERSE — the same
|
||||
* rule applied to the set actually being ranked.
|
||||
*
|
||||
* @param query - Text query to search for.
|
||||
* @param limit - Result budget; the leg keeps `limit * 2` for the fusion.
|
||||
* @param candidateIds - Optional candidate universe to rank inside.
|
||||
* @returns Ranked `{ id, score }` rows — no entity reads.
|
||||
*/
|
||||
private async executeTextSearchScored(
|
||||
query: string,
|
||||
limit: number,
|
||||
candidateIds?: readonly string[]
|
||||
): Promise<Array<{ id: string; score: number }>> {
|
||||
const textMatches = candidateIds
|
||||
? await this.textIdsWithinBelted(query, candidateIds)
|
||||
: await this.metadataIndex.getIdsForTextQuery(query)
|
||||
if (textMatches.length === 0) return []
|
||||
|
||||
// Take top matches (more than the page, for the fusion to rank)
|
||||
const topMatches = textMatches.slice(0, limit * 2)
|
||||
|
||||
// Normalize score to 0-1 range based on match count
|
||||
const maxMatches = topMatches[0]?.matchCount || 1
|
||||
return topMatches.map((m) => ({ id: m.id, score: m.matchCount / maxMatches }))
|
||||
}
|
||||
|
||||
/**
|
||||
* Auto-detect optimal alpha for hybrid search
|
||||
*
|
||||
|
|
@ -16345,56 +16230,55 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
|||
*
|
||||
* Formula: score(d) = sum(1 / (k + rank(d))) for each list
|
||||
*
|
||||
* Now includes match visibility (textScore, semanticScore, matchSource; the
|
||||
* `textMatches` word list needs the entity and is filled at hydration).
|
||||
* Now includes match visibility (textMatches, textScore, semanticScore, matchSource)
|
||||
*
|
||||
* HYDRATE LAST: both legs arrive as ranked ids + scores, and the fusion ranks
|
||||
* ids — no entity is read here. The rows it returns are ranked SHELLS; the
|
||||
* page is cut from them and only that page is read from canonical (see
|
||||
* {@link hydrateResultPage}). The earlier order hydrated both legs in full —
|
||||
* hundreds of rows — to return one page of them.
|
||||
*
|
||||
* @param textResults - Ranked ids + scores from text search
|
||||
* @param semanticResults - Ranked ids + scores from semantic search
|
||||
* @param textResults - Results from text search
|
||||
* @param semanticResults - Results from semantic search
|
||||
* @param alpha - Weight for semantic (0=text only, 1=semantic only)
|
||||
* @param queryWords - Original query words for match tracking
|
||||
* @param k - RRF constant (default: 60, standard in literature)
|
||||
* @returns Fused result shells sorted by combined score with match visibility
|
||||
* @returns Fused results sorted by combined score with match visibility
|
||||
*/
|
||||
private rrfFusion(
|
||||
textResults: ReadonlyArray<{ id: string; score: number }>,
|
||||
semanticResults: ReadonlyArray<{ id: string; score: number }>,
|
||||
private async rrfFusion(
|
||||
textResults: Result<T>[],
|
||||
semanticResults: Result<T>[],
|
||||
alpha: number,
|
||||
queryWords: string[],
|
||||
k: number = 60
|
||||
): Result<T>[] {
|
||||
): Promise<Result<T>[]> {
|
||||
// Track scores and match details per entity
|
||||
interface MatchData {
|
||||
rrf: number
|
||||
textScore?: number
|
||||
semanticScore?: number
|
||||
textMatches: string[]
|
||||
hasText: boolean
|
||||
hasSemantic: boolean
|
||||
}
|
||||
const matchData = new Map<string, MatchData>()
|
||||
const entityMap = new Map<string, Entity<T>>()
|
||||
|
||||
// Text contribution (1 - alpha weight)
|
||||
const textWeight = 1 - alpha
|
||||
textResults.forEach((r, rank) => {
|
||||
const rrfScore = textWeight * (1 / (k + rank + 1))
|
||||
const existing = matchData.get(r.id) || { rrf: 0, hasText: false, hasSemantic: false }
|
||||
const existing = matchData.get(r.id) || { rrf: 0, textMatches: [], hasText: false, hasSemantic: false }
|
||||
existing.rrf += rrfScore
|
||||
existing.textScore = r.score // Original text search score (0-1)
|
||||
existing.hasText = true
|
||||
matchData.set(r.id, existing)
|
||||
if (r.entity) entityMap.set(r.id, r.entity)
|
||||
})
|
||||
|
||||
// Semantic contribution (alpha weight)
|
||||
semanticResults.forEach((r, rank) => {
|
||||
const rrfScore = alpha * (1 / (k + rank + 1))
|
||||
const existing = matchData.get(r.id) || { rrf: 0, hasText: false, hasSemantic: false }
|
||||
const existing = matchData.get(r.id) || { rrf: 0, textMatches: [], hasText: false, hasSemantic: false }
|
||||
existing.rrf += rrfScore
|
||||
existing.semanticScore = r.score // Original semantic search score (0-1)
|
||||
existing.hasSemantic = true
|
||||
matchData.set(r.id, existing)
|
||||
if (r.entity) entityMap.set(r.id, r.entity)
|
||||
})
|
||||
|
||||
// Sort by fused score
|
||||
|
|
@ -16402,9 +16286,27 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
|||
.sort((a, b) => b[1].rrf - a[1].rrf)
|
||||
.map(([id, data]) => ({ id, data }))
|
||||
|
||||
// Create ranked shells with match visibility
|
||||
// Build results - need to load any missing entities
|
||||
const missingIds = sortedIds.filter(s => !entityMap.has(s.id)).map(s => s.id)
|
||||
if (missingIds.length > 0) {
|
||||
const loaded = await this.batchGet(missingIds)
|
||||
for (const [id, entity] of loaded) {
|
||||
entityMap.set(id, entity)
|
||||
}
|
||||
}
|
||||
|
||||
// Performance: Build set of text result IDs for O(1) lookup
|
||||
// This avoids re-extracting text for entities that weren't in text results
|
||||
const textResultIds = new Set(textResults.map(r => r.id))
|
||||
|
||||
// Create final results with match visibility
|
||||
const results: Result<T>[] = []
|
||||
for (const { id, data } of sortedIds) {
|
||||
const entity = entityMap.get(id)
|
||||
if (entity) {
|
||||
// Find which query words matched - uses fast path if entity wasn't in text results
|
||||
const textMatches = this.findMatchingWords(entity, queryWords, textResultIds)
|
||||
|
||||
// Determine match source
|
||||
let matchSource: 'text' | 'semantic' | 'both'
|
||||
if (data.hasText && data.hasSemantic) {
|
||||
|
|
@ -16415,80 +16317,20 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
|||
matchSource = 'semantic'
|
||||
}
|
||||
|
||||
const result = this.pendingResult(id, data.rrf)
|
||||
// Create result with match visibility
|
||||
const result = this.createResult(id, data.rrf, entity)
|
||||
result.textMatches = textMatches
|
||||
result.textScore = data.textScore
|
||||
result.semanticScore = data.semanticScore
|
||||
result.matchSource = matchSource
|
||||
|
||||
results.push(result)
|
||||
}
|
||||
}
|
||||
|
||||
return results
|
||||
}
|
||||
|
||||
/**
|
||||
* A ranked candidate whose entity has NOT been read yet.
|
||||
*
|
||||
* The shell carries everything the ranking tail needs — the id, the score,
|
||||
* and the match-visibility fields — and nothing that requires canonical. It
|
||||
* is typed `Result` so it flows through the shared dedupe / visibility /
|
||||
* filter / rank / page tail unchanged; {@link hydrateResultPage} turns the
|
||||
* survivors into real results before any caller sees them, and find()'s
|
||||
* index-integrity guard drops any row that never gained an entity.
|
||||
*
|
||||
* @param id - The candidate's canonical id.
|
||||
* @param score - Its rank score.
|
||||
*/
|
||||
private pendingResult(id: string, score: number): Result<T> {
|
||||
return { id, score } as Result<T>
|
||||
}
|
||||
|
||||
/**
|
||||
* Read canonical for exactly the rows that need it — the hydrate-last seam.
|
||||
*
|
||||
* Rows that already carry an entity (the eager legs: metadata, text-only,
|
||||
* semantic-only, proximity, graph) pass through untouched, so this is a no-op
|
||||
* for every path that has not deferred. Rows that are shells are read in ONE
|
||||
* batch and rebuilt through {@link createResult}, so a hydrated row is
|
||||
* indistinguishable from an eagerly-built one — same flattened fields, same
|
||||
* `entity`, same key order — with `finish` re-applying the fields only the
|
||||
* deferring path knows about (a hybrid row's match visibility).
|
||||
*
|
||||
* A shell whose id has no canonical row is dropped, exactly as the eager legs
|
||||
* dropped it; find()'s index-integrity guard makes the same judgement on the
|
||||
* page it returns.
|
||||
*
|
||||
* @param rows - The page's rows, ranked and paged already.
|
||||
* @param finish - Applied to each rebuilt row, with its shell, after the
|
||||
* flattened fields are set.
|
||||
* @returns The page with every surviving row hydrated.
|
||||
*/
|
||||
private async hydrateResultPage(
|
||||
rows: Result<T>[],
|
||||
finish?: (row: Result<T>, pending: Result<T>) => void
|
||||
): Promise<Result<T>[]> {
|
||||
const pendingIds: string[] = []
|
||||
for (const row of rows) {
|
||||
if (!row.entity) pendingIds.push(row.id)
|
||||
}
|
||||
if (pendingIds.length === 0) return rows
|
||||
|
||||
const entitiesMap = await this.batchGet(pendingIds)
|
||||
const hydrated: Result<T>[] = []
|
||||
for (const row of rows) {
|
||||
if (row.entity) {
|
||||
hydrated.push(row)
|
||||
continue
|
||||
}
|
||||
const entity = entitiesMap.get(row.id)
|
||||
if (!entity) continue
|
||||
const filled = this.createResult(row.id, row.score, entity, row.explanation)
|
||||
finish?.(filled, row)
|
||||
hydrated.push(filled)
|
||||
}
|
||||
return hydrated
|
||||
}
|
||||
|
||||
/**
|
||||
* Find which query words match in an entity's text content
|
||||
*
|
||||
|
|
|
|||
|
|
@ -473,28 +473,6 @@ export interface MetadataIndexProvider {
|
|||
graphIndex: unknown
|
||||
): Promise<{ ids: string[]; emptyAt: 'graph' | 'filter' | 'visibility' | 'none' } | null>
|
||||
getIdsForTextQuery(query: string): Promise<Array<{ id: string; matchCount: number }>>
|
||||
/**
|
||||
* @description OPTIONAL: score `query` over `ids` ONLY — the text-leg twin of
|
||||
* {@link filterIdsWithin}, and the door a hybrid `find({ query, where })`
|
||||
* walks. The metadata filter's universe is the candidate set there, so the
|
||||
* text leg must cost O(|ids|) membership checks and marshal at most `|ids|`
|
||||
* rows, never the whole posting list of every query word. A native index
|
||||
* intersects its own postings with the candidate set (membership by entity
|
||||
* int) before any string crosses the boundary; the reference index answers
|
||||
* from its own `getIdsForTextQuery`, so the two doors can never disagree.
|
||||
* Absent → Brainy intersects `getIdsForTextQuery`'s answer with `ids` itself
|
||||
* (correct, and still hydrate-last, but it marshals the whole answer).
|
||||
*
|
||||
* The answer keeps `getIdsForTextQuery`'s contract: `{ id, matchCount }`
|
||||
* sorted by `matchCount` descending, ties in the order the whole-store answer
|
||||
* would have produced. Only rows in `ids` may appear.
|
||||
* @param query - The same text query accepted by `getIdsForTextQuery`.
|
||||
* @param ids - The candidate ids (canonical). The answer is a subset.
|
||||
*/
|
||||
getIdsForTextQueryWithin?(
|
||||
query: string,
|
||||
ids: readonly string[]
|
||||
): Promise<Array<{ id: string; matchCount: number }>>
|
||||
getSortedIdsForFilter(filter: any, orderBy: string, order?: 'asc' | 'desc', topK?: number): Promise<string[]>
|
||||
getFilterValues(field: string): Promise<string[]>
|
||||
getFilterFields(): Promise<string[]>
|
||||
|
|
|
|||
|
|
@ -1509,56 +1509,11 @@ export class MetadataIndexManager implements MetadataIndexProvider {
|
|||
* @returns Array of { id, matchCount } sorted by matchCount descending
|
||||
*/
|
||||
async getIdsForTextQuery(query: string): Promise<Array<{ id: string; matchCount: number }>> {
|
||||
return this.scoreTextQuery(query)
|
||||
}
|
||||
|
||||
/**
|
||||
* Score a text query over `ids` ONLY — the reference implementation of the
|
||||
* optional `getIdsForTextQueryWithin` door (see
|
||||
* {@link import('../plugin.js').MetadataIndexProvider}). The hybrid
|
||||
* `find({ query, where })` path passes the metadata filter's universe here so
|
||||
* the text leg ranks INSIDE that universe instead of ranking the whole store
|
||||
* and discarding the rows the filter would have dropped.
|
||||
*
|
||||
* It answers from the same posting-list merge as {@link getIdsForTextQuery},
|
||||
* with the candidate membership applied as each word's postings are counted,
|
||||
* so the two doors can never disagree: the answer is exactly the whole-store
|
||||
* answer restricted to `ids`, in the same order.
|
||||
*
|
||||
* @param query - Text query to search for.
|
||||
* @param ids - Candidate entity ids; only these may appear in the answer.
|
||||
* @returns Array of { id, matchCount } sorted by matchCount descending.
|
||||
*/
|
||||
async getIdsForTextQueryWithin(
|
||||
query: string,
|
||||
ids: readonly string[]
|
||||
): Promise<Array<{ id: string; matchCount: number }>> {
|
||||
if (ids.length === 0) return []
|
||||
return this.scoreTextQuery(query, new Set(ids))
|
||||
}
|
||||
|
||||
/**
|
||||
* The one posting-list merge behind both text doors.
|
||||
*
|
||||
* Each query word contributes AT MOST one match per entity (a posting list
|
||||
* can name an id more than once), and entities are ranked by how many of the
|
||||
* query's words they matched. `within`, when given, restricts the count to
|
||||
* those candidates — applied during the merge, so a restricted call never
|
||||
* materializes a whole-store match map.
|
||||
*
|
||||
* @param query - Text query to search for.
|
||||
* @param within - Optional candidate universe; absent = the whole store.
|
||||
* @returns Array of { id, matchCount } sorted by matchCount descending.
|
||||
*/
|
||||
private async scoreTextQuery(
|
||||
query: string,
|
||||
within?: ReadonlySet<string>
|
||||
): Promise<Array<{ id: string; matchCount: number }>> {
|
||||
const queryWords = this.tokenize(query)
|
||||
if (queryWords.length === 0) return []
|
||||
|
||||
// Count matches per entity, one word's postings at a time.
|
||||
const matchCounts = new Map<string, number>()
|
||||
// Get IDs for each word hash
|
||||
const wordIdSets: Map<string, number>[] = []
|
||||
for (const word of queryWords) {
|
||||
const wordHash = this.hashWord(word)
|
||||
let ids: string[]
|
||||
|
|
@ -1574,12 +1529,19 @@ export class MetadataIndexManager implements MetadataIndexProvider {
|
|||
throw err
|
||||
}
|
||||
}
|
||||
// One count per (word, entity) — dedupe this word's postings first.
|
||||
const counted = new Set<string>()
|
||||
const idSet = new Map<string, number>()
|
||||
for (const id of ids) {
|
||||
if (counted.has(id)) continue
|
||||
counted.add(id)
|
||||
if (within && !within.has(id)) continue
|
||||
idSet.set(id, 1)
|
||||
}
|
||||
wordIdSets.push(idSet)
|
||||
}
|
||||
|
||||
if (wordIdSets.length === 0) return []
|
||||
|
||||
// Count matches per entity
|
||||
const matchCounts = new Map<string, number>()
|
||||
for (const idSet of wordIdSets) {
|
||||
for (const [id] of idSet) {
|
||||
matchCounts.set(id, (matchCounts.get(id) || 0) + 1)
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,635 +0,0 @@
|
|||
/**
|
||||
* @module tests/integration/find-hybrid-filter-before-hydrate
|
||||
* @description FILTER BEFORE HYDRATE, applied to the hybrid `find({ query })` path.
|
||||
*
|
||||
* A hybrid find fuses two legs. The semantic leg already walked only the
|
||||
* metadata filter's universe (`candidateIds` / `allowedIds`). The TEXT leg did
|
||||
* not: it ranked the WHOLE store, took the top `limit * 4`, read every one of
|
||||
* those rows from canonical, and only then intersected with the filter — so a
|
||||
* filtered hybrid find on a large store read hundreds of rows to return a
|
||||
* handful of them, and a matching row outside the store-wide text prefix was
|
||||
* silently dropped. That is the same defect `find({ connected })` carried
|
||||
* before the graph-first law, one leg over.
|
||||
*
|
||||
* Both halves are pinned here.
|
||||
*
|
||||
* THE ANSWER. Where the filter did not truncate the text leg — the universe
|
||||
* covers every text match, so both orders rank the same rows — the new
|
||||
* pipeline's answer is IDENTICAL to the old one's: same rows, same order, same
|
||||
* scores, same match visibility, same row shape. The oracle below is the
|
||||
* pre-change pipeline itself, replayed on the same brain through the same
|
||||
* doors, so the comparison is against what actually ran, not a remembered
|
||||
* expectation.
|
||||
*
|
||||
* THE CORRECTION. Where the filter DID truncate it — the query's words are
|
||||
* common outside the universe — the old order let the text leg contribute
|
||||
* nothing at all: every row it ranked was discarded by the filter, and the
|
||||
* answer came from the semantic leg alone. The new order ranks inside the
|
||||
* universe, so the text leg contributes the rows it always should have.
|
||||
*
|
||||
* THE COST. Canonical is read for exactly the page: one batch, `limit` rows,
|
||||
* never the legs. And the text leg is asked about the universe's ids only —
|
||||
* what it marshals is bounded by the universe, not by the store.
|
||||
*/
|
||||
import { describe, it, expect, beforeAll, vi } from 'vitest'
|
||||
import { Brainy } from '../../src/brainy'
|
||||
import { NounType, VerbType } from '../../src/types/graphTypes'
|
||||
import { rankIndicesByScore, reorderByIndices } from '../../src/utils/resultRanking'
|
||||
import { resolveEntityId } from '../../src/utils/idNormalization'
|
||||
|
||||
/** Embedding width of the default model — the row vectors must match it. */
|
||||
const DIM = 384
|
||||
|
||||
/**
|
||||
* A deterministic, per-row-distinct unit vector. Distinct so the semantic leg
|
||||
* has a real ranking to produce (identical vectors would make its order a tie
|
||||
* break), deterministic so the oracle and the pipeline see the same one.
|
||||
*/
|
||||
function seededVector(seed: number): number[] {
|
||||
const v = new Array<number>(DIM)
|
||||
for (let i = 0; i < DIM; i++) {
|
||||
v[i] = Math.sin((i + 1) * 0.11 + seed * 0.37) * 0.5 + Math.cos((i + 1) * 0.05 + seed * 0.13) * 0.3
|
||||
}
|
||||
const magnitude = Math.sqrt(v.reduce((sum, x) => sum + x * x, 0))
|
||||
return v.map((x) => x / magnitude)
|
||||
}
|
||||
|
||||
/** The fields a caller reads off a hybrid row — the whole comparable surface. */
|
||||
function project(rows: any[]): any[] {
|
||||
return rows.map((r) => ({
|
||||
id: r.id,
|
||||
score: r.score,
|
||||
type: r.type,
|
||||
metadata: r.metadata,
|
||||
textMatches: r.textMatches,
|
||||
textScore: r.textScore,
|
||||
semanticScore: r.semanticScore,
|
||||
matchSource: r.matchSource
|
||||
}))
|
||||
}
|
||||
|
||||
/**
|
||||
* The PRE-CHANGE hybrid pipeline, replayed on a live brain through the same
|
||||
* provider doors it used: whole-store text ranking with both legs hydrated in
|
||||
* full, RRF fusion, then the metadata intersection, then the page.
|
||||
*
|
||||
* Supports the shapes these pins exercise (query + where/type/excludeVFS +
|
||||
* connected + offset); `orderBy`, `fusion` and `near` are not replayed.
|
||||
*/
|
||||
async function legacyHybridFind(brain: any, params: any): Promise<any[]> {
|
||||
const index = brain.metadataIndex
|
||||
const limit = params.limit ?? 10
|
||||
const offset = params.offset ?? 0
|
||||
const hasFilter = Boolean(
|
||||
params.where || params.type || params.subtype || params.service || params.excludeVFS
|
||||
)
|
||||
|
||||
let preResolvedMetadataIds: string[] | null = null
|
||||
let preResolvedFilter: any = null
|
||||
let graphFirstIds: string[] | null = null
|
||||
|
||||
if (params.connected) {
|
||||
// find() normalizes the anchors to canonical ids before this stage runs.
|
||||
const anchored = {
|
||||
...params,
|
||||
connected: {
|
||||
...params.connected,
|
||||
...(params.connected.from && { from: resolveEntityId(params.connected.from) }),
|
||||
...(params.connected.to && { to: resolveEntityId(params.connected.to) })
|
||||
}
|
||||
}
|
||||
graphFirstIds = await brain.resolveConnectedIds(anchored)
|
||||
if (graphFirstIds!.length > 0 && hasFilter) {
|
||||
preResolvedFilter = brain.buildMetadataFilter(params)
|
||||
graphFirstIds = await brain.filterIdsWithinBelted(preResolvedFilter, graphFirstIds)
|
||||
}
|
||||
if (graphFirstIds!.length === 0) return []
|
||||
preResolvedMetadataIds = graphFirstIds
|
||||
} else if (hasFilter) {
|
||||
preResolvedFilter = brain.buildMetadataFilter(params)
|
||||
preResolvedMetadataIds = await brain.filterIdsBelted(preResolvedFilter)
|
||||
if (preResolvedMetadataIds!.length === 0) return []
|
||||
}
|
||||
|
||||
// Text leg — the whole store, then the top `limit * 4`, hydrated in full.
|
||||
const allTextMatches = await index.getIdsForTextQuery(params.query)
|
||||
const topMatches = allTextMatches.slice(0, limit * 2 * 2)
|
||||
const maxMatches = topMatches[0]?.matchCount || 1
|
||||
const textEntities = await brain.batchGet(topMatches.map((m: any) => m.id))
|
||||
const textResults = topMatches
|
||||
.filter((m: any) => textEntities.has(m.id))
|
||||
.map((m: any) => ({ id: m.id, score: m.matchCount / maxMatches }))
|
||||
|
||||
// Semantic leg — the beam walk over the universe, hydrated in full.
|
||||
const vector = await brain.embed(params.query)
|
||||
const searchOptions = preResolvedMetadataIds ? { candidateIds: preResolvedMetadataIds } : undefined
|
||||
const searchResults: [string, number][] = await brain.index.search(
|
||||
vector,
|
||||
limit * 2,
|
||||
undefined,
|
||||
searchOptions
|
||||
)
|
||||
const semanticEntities = await brain.batchGet(searchResults.map(([id]) => id))
|
||||
const semanticResults = searchResults
|
||||
.filter(([id]) => semanticEntities.has(id))
|
||||
.map(([id, distance]) => ({ id, score: Math.max(0, Math.min(1, 1 / (1 + distance))) }))
|
||||
|
||||
// RRF fusion, with the match visibility the rows carried.
|
||||
const alpha = params.hybridAlpha ?? brain.autoAlpha(params.query)
|
||||
const k = 60
|
||||
const matchData = new Map<string, any>()
|
||||
const textWeight = 1 - alpha
|
||||
textResults.forEach((r: any, rank: number) => {
|
||||
const existing = matchData.get(r.id) || { rrf: 0, hasText: false, hasSemantic: false }
|
||||
existing.rrf += textWeight * (1 / (k + rank + 1))
|
||||
existing.textScore = r.score
|
||||
existing.hasText = true
|
||||
matchData.set(r.id, existing)
|
||||
})
|
||||
semanticResults.forEach((r: any, rank: number) => {
|
||||
const existing = matchData.get(r.id) || { rrf: 0, hasText: false, hasSemantic: false }
|
||||
existing.rrf += alpha * (1 / (k + rank + 1))
|
||||
existing.semanticScore = r.score
|
||||
existing.hasSemantic = true
|
||||
matchData.set(r.id, existing)
|
||||
})
|
||||
|
||||
const queryWords: string[] = index.tokenize(params.query)
|
||||
const textResultIds = new Set(textResults.map((r: any) => r.id))
|
||||
const fusedIds = Array.from(matchData.entries())
|
||||
.sort((a, b) => b[1].rrf - a[1].rrf)
|
||||
.map(([id, data]) => ({ id, data }))
|
||||
|
||||
const allEntities = await brain.batchGet(fusedIds.map((f) => f.id))
|
||||
let rows: any[] = []
|
||||
for (const { id, data } of fusedIds) {
|
||||
const entity = allEntities.get(id)
|
||||
if (!entity) continue
|
||||
const textContent = textResultIds.has(id)
|
||||
? index.extractTextContent({ data: entity.data, metadata: entity.metadata }).toLowerCase()
|
||||
: null
|
||||
rows.push({
|
||||
id,
|
||||
score: data.rrf,
|
||||
type: entity.type,
|
||||
metadata: entity.metadata,
|
||||
textMatches:
|
||||
textContent === null ? [] : queryWords.filter((w) => textContent.includes(w.toLowerCase())),
|
||||
textScore: data.textScore,
|
||||
semanticScore: data.semanticScore,
|
||||
matchSource: data.hasText && data.hasSemantic ? 'both' : data.hasText ? 'text' : 'semantic'
|
||||
})
|
||||
}
|
||||
|
||||
// The metadata intersection — after the legs, as it was.
|
||||
if (preResolvedMetadataIds && preResolvedFilter) {
|
||||
const filteredIdSet = new Set(preResolvedMetadataIds)
|
||||
rows = rows.filter((r) => filteredIdSet.has(r.id))
|
||||
}
|
||||
if (graphFirstIds !== null) {
|
||||
const neighbourSet = new Set(graphFirstIds)
|
||||
rows = rows.filter((r) => neighbourSet.has(r.id))
|
||||
}
|
||||
|
||||
// Rank to the page, then cut it.
|
||||
const order = rankIndicesByScore(
|
||||
rows.map((r) => r.score),
|
||||
offset + limit,
|
||||
true
|
||||
)
|
||||
return reorderByIndices(rows, order).slice(offset, offset + limit)
|
||||
}
|
||||
|
||||
/**
|
||||
* FIXTURE A — the filter's universe covers every text match, so the two orders
|
||||
* rank exactly the same rows and the answers must be identical.
|
||||
*/
|
||||
describe('hybrid find: filter before hydrate — the answer is unchanged', () => {
|
||||
let brain: Brainy<any>
|
||||
const QUERY = 'orbital telemetry'
|
||||
const MATCHES = 24
|
||||
const FILLER = 120
|
||||
const OUTSIDE = 30
|
||||
const VFS = 10
|
||||
const RETRACTED = 6
|
||||
const anchor = 'array-anchor'
|
||||
const matchIds: string[] = []
|
||||
|
||||
beforeAll(async () => {
|
||||
brain = new Brainy({ requireSubtype: false, storage: { type: 'memory' } })
|
||||
await brain.init()
|
||||
|
||||
let seed = 1
|
||||
await brain.add({
|
||||
id: anchor,
|
||||
data: 'ground station anchor record',
|
||||
type: NounType.Thing,
|
||||
metadata: { lane: 'alpha', role: 'anchor' },
|
||||
vector: seededVector(seed++)
|
||||
})
|
||||
|
||||
// Rows the query's words actually match — all inside every filter below.
|
||||
for (let i = 0; i < MATCHES; i++) {
|
||||
const id = `match-${i}`
|
||||
await brain.add({
|
||||
id,
|
||||
data: `orbital telemetry packet ${i} recorded downlink`,
|
||||
type: NounType.Document,
|
||||
metadata: { lane: 'alpha', rank: i },
|
||||
vector: seededVector(seed++)
|
||||
})
|
||||
matchIds.push(resolveEntityId(id))
|
||||
await brain.relate({ from: anchor, to: id, type: VerbType.RelatedTo })
|
||||
}
|
||||
// Rows inside the universe that the query's words do NOT match.
|
||||
for (let i = 0; i < FILLER; i++) {
|
||||
await brain.add({
|
||||
id: `filler-${i}`,
|
||||
data: `cistern ledger entry ${i} archived`,
|
||||
type: NounType.Document,
|
||||
metadata: { lane: 'alpha', rank: 1000 + i },
|
||||
vector: seededVector(seed++)
|
||||
})
|
||||
}
|
||||
// Rows outside the universe.
|
||||
for (let i = 0; i < OUTSIDE; i++) {
|
||||
await brain.add({
|
||||
id: `outside-${i}`,
|
||||
data: `unrelated dossier ${i}`,
|
||||
type: NounType.Person,
|
||||
metadata: { lane: 'beta' },
|
||||
vector: seededVector(seed++)
|
||||
})
|
||||
}
|
||||
// VFS infrastructure rows — excluded by excludeVFS.
|
||||
for (let i = 0; i < VFS; i++) {
|
||||
await brain.add({
|
||||
id: `vfs-${i}`,
|
||||
data: `mounted path ${i}`,
|
||||
type: NounType.Document,
|
||||
metadata: { lane: 'alpha', vfsType: 'file' },
|
||||
vector: seededVector(seed++)
|
||||
})
|
||||
}
|
||||
// Retracted rows — excluded by a `missing` negation.
|
||||
for (let i = 0; i < RETRACTED; i++) {
|
||||
await brain.add({
|
||||
id: `retracted-${i}`,
|
||||
data: `withdrawn note ${i}`,
|
||||
type: NounType.Document,
|
||||
metadata: { lane: 'alpha', retracted: true },
|
||||
vector: seededVector(seed++)
|
||||
})
|
||||
}
|
||||
|
||||
// The reference index has no opaque-set door, so the pipeline and the
|
||||
// oracle both restrict the beam walk with the materialized candidate ids.
|
||||
expect(typeof (brain as any).metadataIndex.getIdSetForFilter).not.toBe('function')
|
||||
})
|
||||
|
||||
it('the fixture does not truncate the text leg — the universe covers every text match', async () => {
|
||||
const index = (brain as any).metadataIndex
|
||||
const textMatches = await index.getIdsForTextQuery(QUERY)
|
||||
expect(textMatches).toHaveLength(MATCHES)
|
||||
const universe = await (brain as any).filterIdsBelted({ lane: 'alpha' })
|
||||
const inUniverse = new Set(universe)
|
||||
for (const m of textMatches) expect(inUniverse.has(m.id)).toBe(true)
|
||||
})
|
||||
|
||||
it('hybrid + where: identical rows, identical order, identical scores', async () => {
|
||||
const params = { query: QUERY, where: { lane: 'alpha' }, limit: 8 }
|
||||
const expected = await legacyHybridFind(brain as any, params)
|
||||
const actual = await brain.find(params as any)
|
||||
expect(actual.length).toBe(expected.length)
|
||||
expect(project(actual)).toEqual(expected)
|
||||
})
|
||||
|
||||
it('hybrid + where + offset: identical page two', async () => {
|
||||
const params = { query: QUERY, where: { lane: 'alpha' }, limit: 6, offset: 6 }
|
||||
const expected = await legacyHybridFind(brain as any, params)
|
||||
const actual = await brain.find(params as any)
|
||||
expect(actual.length).toBe(expected.length)
|
||||
expect(project(actual)).toEqual(expected)
|
||||
})
|
||||
|
||||
it('hybrid + type list + excludeVFS + a `missing` negation: identical', async () => {
|
||||
const params = {
|
||||
query: QUERY,
|
||||
type: [NounType.Document, NounType.Person],
|
||||
excludeVFS: true,
|
||||
where: { lane: 'alpha', retracted: { missing: true } },
|
||||
limit: 8
|
||||
}
|
||||
const expected = await legacyHybridFind(brain as any, params)
|
||||
const actual = await brain.find(params as any)
|
||||
expect(actual.length).toBe(expected.length)
|
||||
expect(project(actual)).toEqual(expected)
|
||||
for (const r of actual) {
|
||||
expect(r.metadata.retracted).toBeUndefined()
|
||||
expect(r.metadata.vfsType).toBeUndefined()
|
||||
}
|
||||
})
|
||||
|
||||
it('hybrid + type list + excludeVFS + a `missing` negation, offset: identical', async () => {
|
||||
const params = {
|
||||
query: QUERY,
|
||||
type: [NounType.Document, NounType.Person],
|
||||
excludeVFS: true,
|
||||
where: { lane: 'alpha', retracted: { missing: true } },
|
||||
limit: 5,
|
||||
offset: 5
|
||||
}
|
||||
const expected = await legacyHybridFind(brain as any, params)
|
||||
const actual = await brain.find(params as any)
|
||||
expect(actual.length).toBe(expected.length)
|
||||
expect(project(actual)).toEqual(expected)
|
||||
})
|
||||
|
||||
it('hybrid + connected: identical, and never a non-neighbour', async () => {
|
||||
const params = {
|
||||
query: QUERY,
|
||||
connected: { from: anchor, direction: 'out' as const },
|
||||
where: { lane: 'alpha' },
|
||||
limit: 8
|
||||
}
|
||||
const expected = await legacyHybridFind(brain as any, params)
|
||||
const actual = await brain.find(params as any)
|
||||
expect(actual.length).toBe(expected.length)
|
||||
expect(project(actual)).toEqual(expected)
|
||||
const neighbours = new Set(matchIds)
|
||||
for (const r of actual) expect(neighbours.has(r.id)).toBe(true)
|
||||
})
|
||||
|
||||
it('hybrid + connected + offset: page two is the page, not an empty answer', async () => {
|
||||
const params = {
|
||||
query: QUERY,
|
||||
connected: { from: anchor, direction: 'out' as const },
|
||||
where: { lane: 'alpha' },
|
||||
limit: 5,
|
||||
offset: 5
|
||||
}
|
||||
const expected = await legacyHybridFind(brain as any, params)
|
||||
expect(expected).toHaveLength(5)
|
||||
const actual = await brain.find(params as any)
|
||||
expect(actual.length).toBe(expected.length)
|
||||
expect(project(actual)).toEqual(expected)
|
||||
})
|
||||
|
||||
it('hybrid + connected: paging reaches every matching neighbour exactly once', async () => {
|
||||
const seen = new Set<string>()
|
||||
for (let offset = 0; offset < MATCHES; offset += 6) {
|
||||
const page = await brain.find({
|
||||
query: QUERY,
|
||||
connected: { from: anchor, direction: 'out' as const },
|
||||
where: { lane: 'alpha' },
|
||||
limit: 6,
|
||||
offset
|
||||
} as any)
|
||||
for (const r of page) {
|
||||
expect(seen.has(r.id)).toBe(false)
|
||||
seen.add(r.id)
|
||||
}
|
||||
}
|
||||
// Every row the fused candidate set holds is reachable by paging, and the
|
||||
// neighbour set is the ceiling.
|
||||
expect(seen.size).toBeGreaterThanOrEqual(MATCHES)
|
||||
const neighbours = new Set(matchIds)
|
||||
for (const id of seen) expect(neighbours.has(id)).toBe(true)
|
||||
})
|
||||
|
||||
it('hybrid + fusion + offset: page two is the page', async () => {
|
||||
const plain = await brain.find({
|
||||
query: QUERY,
|
||||
where: { lane: 'alpha' },
|
||||
limit: 5,
|
||||
offset: 5
|
||||
} as any)
|
||||
const fused = await brain.find({
|
||||
query: QUERY,
|
||||
where: { lane: 'alpha' },
|
||||
fusion: 'weighted',
|
||||
limit: 5,
|
||||
offset: 5
|
||||
} as any)
|
||||
expect(fused).toHaveLength(plain.length)
|
||||
expect(fused.map((r) => r.id)).toEqual(plain.map((r) => r.id))
|
||||
})
|
||||
|
||||
it('a hydrated hybrid row is shaped exactly as an eagerly-built one', async () => {
|
||||
const rows = await brain.find({ query: QUERY, where: { lane: 'alpha' }, limit: 8 } as any)
|
||||
const row = rows[0]
|
||||
expect(Object.keys(row)).toEqual([
|
||||
'id',
|
||||
'score',
|
||||
'type',
|
||||
'subtype',
|
||||
'visibility',
|
||||
'metadata',
|
||||
'data',
|
||||
'confidence',
|
||||
'weight',
|
||||
'_rev',
|
||||
'entity',
|
||||
'textMatches',
|
||||
'textScore',
|
||||
'semanticScore',
|
||||
'matchSource'
|
||||
])
|
||||
// The flattened fields are projections of the entity, as always.
|
||||
expect(row.entity).toBeDefined()
|
||||
expect(row.type).toBe(row.entity.type)
|
||||
expect(row.metadata).toBe(row.entity.metadata)
|
||||
expect(row.data).toBe(row.entity.data)
|
||||
expect(row._rev).toBe(row.entity._rev)
|
||||
// The match visibility survives the deferral — every leg's fields, on the
|
||||
// rows that leg contributed, exactly as the eager pipeline set them.
|
||||
expect(['text', 'semantic', 'both']).toContain(row.matchSource)
|
||||
for (const r of rows) {
|
||||
if (r.matchSource === 'semantic') {
|
||||
expect(r.textMatches).toEqual([])
|
||||
expect(r.textScore).toBeUndefined()
|
||||
} else {
|
||||
expect(r.textMatches).toEqual(['orbital', 'telemetry'])
|
||||
expect(typeof r.textScore).toBe('number')
|
||||
}
|
||||
if (r.matchSource === 'text') {
|
||||
expect(r.semanticScore).toBeUndefined()
|
||||
} else {
|
||||
expect(typeof r.semanticScore).toBe('number')
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
it('reads canonical for the page only — one batch, `limit` rows', async () => {
|
||||
// Warm any first-read verification before the counters are read.
|
||||
await brain.find({ query: QUERY, where: { lane: 'alpha' }, limit: 1 } as any)
|
||||
|
||||
const hydrate = vi.spyOn(brain as any, 'batchGet')
|
||||
try {
|
||||
const results = await brain.find({ query: QUERY, where: { lane: 'alpha' }, limit: 10 } as any)
|
||||
expect(results).toHaveLength(10)
|
||||
expect(hydrate).toHaveBeenCalledTimes(1)
|
||||
expect((hydrate.mock.calls[0][0] as string[]).length).toBe(10)
|
||||
} finally {
|
||||
hydrate.mockRestore()
|
||||
}
|
||||
})
|
||||
|
||||
it('asks the text index about the universe only, never the whole store', async () => {
|
||||
const index = (brain as any).metadataIndex
|
||||
const wholeStore = vi.spyOn(index, 'getIdsForTextQuery')
|
||||
const within = vi.spyOn(index, 'getIdsForTextQueryWithin')
|
||||
try {
|
||||
await brain.find({ query: QUERY, where: { lane: 'alpha' }, limit: 10 } as any)
|
||||
expect(wholeStore).not.toHaveBeenCalled()
|
||||
expect(within).toHaveBeenCalledTimes(1)
|
||||
|
||||
const askedIds = within.mock.calls[0][1] as string[]
|
||||
const universe = await (brain as any).filterIdsBelted({ lane: 'alpha' })
|
||||
expect(askedIds).toHaveLength(universe.length)
|
||||
|
||||
// What the text leg marshals is bounded by the universe, not the store.
|
||||
const marshalled = (await within.mock.results[0].value) as unknown[]
|
||||
expect(marshalled.length).toBeLessThanOrEqual(universe.length)
|
||||
expect(marshalled).toHaveLength(MATCHES)
|
||||
} finally {
|
||||
wholeStore.mockRestore()
|
||||
within.mockRestore()
|
||||
}
|
||||
})
|
||||
|
||||
it('the two text doors agree: within is the whole-store answer restricted', async () => {
|
||||
const index = (brain as any).metadataIndex
|
||||
const universe: string[] = await (brain as any).filterIdsBelted({
|
||||
lane: 'alpha',
|
||||
retracted: { missing: true }
|
||||
})
|
||||
const inUniverse = new Set(universe)
|
||||
const whole = await index.getIdsForTextQuery(QUERY)
|
||||
const within = await index.getIdsForTextQueryWithin(QUERY, universe)
|
||||
expect(within).toEqual(whole.filter((m: any) => inUniverse.has(m.id)))
|
||||
expect(await index.getIdsForTextQueryWithin(QUERY, [])).toEqual([])
|
||||
})
|
||||
})
|
||||
|
||||
/**
|
||||
* FIXTURE B — the query's words are common OUTSIDE the universe, so the old
|
||||
* order's text leg was entirely consumed by rows the filter then discarded.
|
||||
* This is the corrected behaviour, held by name.
|
||||
*/
|
||||
describe('hybrid find: the text leg ranks inside the filter, not around it', () => {
|
||||
let brain: Brainy<any>
|
||||
const QUERY = 'orbital telemetry drift'
|
||||
const NOISE = 150
|
||||
const KEEP = 15
|
||||
const keepIds: string[] = []
|
||||
|
||||
beforeAll(async () => {
|
||||
brain = new Brainy({ requireSubtype: false, storage: { type: 'memory' } })
|
||||
await brain.init()
|
||||
|
||||
let seed = 5000
|
||||
// Added FIRST and matching one more query word, so they lead the
|
||||
// store-wide text ranking outright — and none of them pass the filter.
|
||||
for (let i = 0; i < NOISE; i++) {
|
||||
await brain.add({
|
||||
id: `noise-${i}`,
|
||||
data: `orbital telemetry drift report ${i}`,
|
||||
type: NounType.Document,
|
||||
metadata: { lane: 'beta' },
|
||||
vector: seededVector(seed++)
|
||||
})
|
||||
}
|
||||
for (let i = 0; i < KEEP; i++) {
|
||||
const id = `keep-${i}`
|
||||
await brain.add({
|
||||
id,
|
||||
data: `orbital telemetry summary ${i}`,
|
||||
type: NounType.Document,
|
||||
metadata: { lane: 'alpha' },
|
||||
vector: seededVector(seed++)
|
||||
})
|
||||
keepIds.push(resolveEntityId(id))
|
||||
}
|
||||
})
|
||||
|
||||
it('the old order let the filter consume the whole text leg', async () => {
|
||||
const index = (brain as any).metadataIndex
|
||||
const universe: string[] = await (brain as any).filterIdsBelted({ lane: 'alpha' })
|
||||
expect(universe).toHaveLength(KEEP)
|
||||
const inUniverse = new Set(universe)
|
||||
|
||||
// The store-wide prefix the old text leg took (limit 10 → limit * 4).
|
||||
const prefix = (await index.getIdsForTextQuery(QUERY)).slice(0, 40)
|
||||
expect(prefix).toHaveLength(40)
|
||||
expect(prefix.filter((m: any) => inUniverse.has(m.id))).toHaveLength(0)
|
||||
|
||||
// Every row the old text leg ranked was then discarded by the filter, so
|
||||
// the old answer carried NO text contribution at all — fifteen rows that
|
||||
// match the query's words exactly, and not one of them reached the page
|
||||
// through the text leg. What the old order returned was whatever the
|
||||
// semantic leg alone happened to reach.
|
||||
const legacy = await legacyHybridFind(brain as any, {
|
||||
query: QUERY,
|
||||
where: { lane: 'alpha' },
|
||||
limit: 10
|
||||
})
|
||||
for (const r of legacy) {
|
||||
expect(r.matchSource).toBe('semantic')
|
||||
expect(r.textScore).toBeUndefined()
|
||||
expect(r.textMatches).toEqual([])
|
||||
}
|
||||
})
|
||||
|
||||
it('the new order ranks the text leg inside the universe', async () => {
|
||||
const results = await brain.find({
|
||||
query: QUERY,
|
||||
where: { lane: 'alpha' },
|
||||
limit: 10
|
||||
} as any)
|
||||
|
||||
expect(results).toHaveLength(10)
|
||||
const keeps = new Set(keepIds)
|
||||
for (const r of results) {
|
||||
expect(keeps.has(r.id)).toBe(true)
|
||||
expect(r.metadata.lane).toBe('alpha')
|
||||
// The text leg is the contributor the old order threw away.
|
||||
expect(['text', 'both']).toContain(r.matchSource)
|
||||
expect(r.textScore).toBe(1)
|
||||
expect(r.textMatches).toEqual(['orbital', 'telemetry'])
|
||||
}
|
||||
})
|
||||
|
||||
it('paging reaches every matching row the old order could not see', async () => {
|
||||
const seen = new Set<string>()
|
||||
for (let offset = 0; offset < KEEP; offset += 5) {
|
||||
const page = await brain.find({
|
||||
query: QUERY,
|
||||
where: { lane: 'alpha' },
|
||||
limit: 5,
|
||||
offset
|
||||
} as any)
|
||||
expect(page).toHaveLength(5)
|
||||
for (const r of page) {
|
||||
expect(seen.has(r.id)).toBe(false)
|
||||
seen.add(r.id)
|
||||
}
|
||||
}
|
||||
expect(seen.size).toBe(KEEP)
|
||||
expect([...seen].sort()).toEqual([...keepIds].sort())
|
||||
})
|
||||
|
||||
it('reads canonical for the page only, on the truncating shape too', async () => {
|
||||
await brain.find({ query: QUERY, where: { lane: 'alpha' }, limit: 1 } as any)
|
||||
|
||||
const hydrate = vi.spyOn(brain as any, 'batchGet')
|
||||
try {
|
||||
const results = await brain.find({ query: QUERY, where: { lane: 'alpha' }, limit: 10 } as any)
|
||||
expect(results).toHaveLength(10)
|
||||
expect(hydrate).toHaveBeenCalledTimes(1)
|
||||
expect((hydrate.mock.calls[0][0] as string[]).length).toBe(10)
|
||||
} finally {
|
||||
hydrate.mockRestore()
|
||||
}
|
||||
})
|
||||
})
|
||||
Reference in a new issue