Compare commits
2 commits
67ae0046de
...
905c267c47
| Author | SHA1 | Date | |
|---|---|---|---|
| 905c267c47 | |||
| b1c7054467 |
4 changed files with 971 additions and 118 deletions
366
src/brainy.ts
366
src/brainy.ts
|
|
@ -7651,6 +7651,18 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
||||||
const searchMode = params.searchMode || 'auto'
|
const searchMode = params.searchMode || 'auto'
|
||||||
const limit = params.limit || 10
|
const limit = params.limit || 10
|
||||||
|
|
||||||
|
// HYDRATE LAST (the hybrid path): its legs and its fusion rank IDS, and
|
||||||
|
// canonical is read at the two page exits below — never for a row the
|
||||||
|
// metadata filter is about to discard. This closure re-applies a hybrid
|
||||||
|
// row's match visibility once its entity is in hand; it is set only by
|
||||||
|
// the hybrid branch, so every other path hydrates unchanged.
|
||||||
|
let finishHybridRow: ((row: Result<T>, pending: Result<T>) => void) | undefined
|
||||||
|
|
||||||
|
// Set once the metadata block below has already ranked and CUT the page.
|
||||||
|
// The tail must not cut it a second time: `offset` has been consumed, and
|
||||||
|
// re-slicing a `limit`-long page by `offset` returns nothing at all.
|
||||||
|
let pagedEarly = false
|
||||||
|
|
||||||
// Handle text-only query (user explicitly wants text search)
|
// Handle text-only query (user explicitly wants text search)
|
||||||
if (searchMode === 'text' && params.query && params.query.trim() !== '') {
|
if (searchMode === 'text' && params.query && params.query.trim() !== '') {
|
||||||
results = await this.executeTextSearch(params.query, limit * 2)
|
results = await this.executeTextSearch(params.query, limit * 2)
|
||||||
|
|
@ -7661,20 +7673,32 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
||||||
}
|
}
|
||||||
// Handle explicit hybrid or auto mode with query
|
// Handle explicit hybrid or auto mode with query
|
||||||
else if ((searchMode === 'auto' || searchMode === 'hybrid') && params.query && params.query.trim() !== '' && !params.vector) {
|
else if ((searchMode === 'auto' || searchMode === 'hybrid') && params.query && params.query.trim() !== '' && !params.vector) {
|
||||||
// Zero-config hybrid: combine text + semantic search with RRF fusion
|
// Zero-config hybrid: combine text + semantic search with RRF fusion.
|
||||||
const [textResults, semanticResults] = await Promise.all([
|
// BOTH legs are held to the metadata filter's universe: the vector leg
|
||||||
this.executeTextSearch(params.query, limit * 2),
|
// walks it as its candidate set, and the text leg ranks inside it
|
||||||
this.executeVectorSearch(params, preResolvedMetadataIds ?? undefined, preResolvedAllowedIds)
|
// instead of ranking the whole store and discarding what the filter
|
||||||
|
// would drop. Neither leg reads canonical — the page does, once.
|
||||||
|
const [textScored, semanticScored] = await Promise.all([
|
||||||
|
this.executeTextSearchScored(params.query, limit * 2, preResolvedMetadataIds ?? undefined),
|
||||||
|
this.executeVectorSearchScored(params, preResolvedMetadataIds ?? undefined, preResolvedAllowedIds)
|
||||||
])
|
])
|
||||||
|
|
||||||
// Use user-specified alpha or auto-detect based on query length
|
// Use user-specified alpha or auto-detect based on query length
|
||||||
const alpha = params.hybridAlpha ?? this.autoAlpha(params.query)
|
const alpha = params.hybridAlpha ?? this.autoAlpha(params.query)
|
||||||
|
|
||||||
// Tokenize query for match visibility
|
// Tokenize query for match visibility. The word list needs the entity,
|
||||||
|
// so it is computed on the page, at hydration.
|
||||||
const queryWords = this.metadataIndex.tokenize(params.query)
|
const queryWords = this.metadataIndex.tokenize(params.query)
|
||||||
|
const textResultIds = new Set(textScored.map((r) => r.id))
|
||||||
|
finishHybridRow = (row, pending) => {
|
||||||
|
row.textMatches = this.findMatchingWords(row.entity, queryWords, textResultIds)
|
||||||
|
row.textScore = pending.textScore
|
||||||
|
row.semanticScore = pending.semanticScore
|
||||||
|
row.matchSource = pending.matchSource
|
||||||
|
}
|
||||||
|
|
||||||
// RRF fusion combines both result sets with match visibility
|
// RRF fusion combines both ranked id sets with match visibility
|
||||||
results = await this.rrfFusion(textResults, semanticResults, alpha, queryWords)
|
results = this.rrfFusion(textScored, semanticScored, alpha)
|
||||||
}
|
}
|
||||||
// Handle direct vector search (no query text) - no hybrid needed
|
// Handle direct vector search (no query text) - no hybrid needed
|
||||||
else if (params.vector && !params.query) {
|
else if (params.vector && !params.query) {
|
||||||
|
|
@ -7735,20 +7759,13 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
||||||
const k = offset + limit
|
const k = offset + limit
|
||||||
const order = rankIndicesByScore(results.map(r => r.score), k, true)
|
const order = rankIndicesByScore(results.map(r => r.score), k, true)
|
||||||
results = reorderByIndices(results, order).slice(offset, k)
|
results = reorderByIndices(results, order).slice(offset, k)
|
||||||
|
pagedEarly = true
|
||||||
|
|
||||||
// Batch-load entities only for the paginated results (10x faster on GCS)
|
// Batch-load entities only for the paginated results (10x faster on GCS).
|
||||||
const idsToLoad = results.filter(r => !r.entity).map(r => r.id)
|
// This is the hydrate-last seam for the deferring paths: a row that
|
||||||
if (idsToLoad.length > 0) {
|
// arrives as a ranked shell is rebuilt in full here — flattened
|
||||||
const entitiesMap = await this.batchGet(idsToLoad)
|
// fields, entity and match visibility — never `entity` alone.
|
||||||
for (const result of results) {
|
results = await this.hydrateResultPage(results, finishHybridRow)
|
||||||
if (!result.entity) {
|
|
||||||
const entity = entitiesMap.get(result.id)
|
|
||||||
if (entity) {
|
|
||||||
result.entity = entity
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Early return if no other processing needed
|
// Early return if no other processing needed
|
||||||
if (!params.connected && !params.fusion) {
|
if (!params.connected && !params.fusion) {
|
||||||
|
|
@ -7854,8 +7871,20 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
||||||
|
|
||||||
const finalOffset = params.offset || 0
|
const finalOffset = params.offset || 0
|
||||||
|
|
||||||
// Efficient pagination - only slice what we need (limit already defined above)
|
// Efficient pagination - only slice what we need (limit already defined
|
||||||
return results.slice(finalOffset, finalOffset + limit)
|
// above), THEN read canonical for the page. Rows that arrived hydrated
|
||||||
|
// pass straight through; a deferred path reads exactly these rows.
|
||||||
|
//
|
||||||
|
// A page the metadata block already cut is NOT cut again: it holds the
|
||||||
|
// rows at [offset, offset+limit) of the ranking, so slicing it by
|
||||||
|
// `offset` a second time drops the whole page. That is how
|
||||||
|
// `find({ query, connected, where, offset })` — the shapes that reach
|
||||||
|
// here after early paging, `connected` and `fusion` — answered [] for
|
||||||
|
// every page but the first.
|
||||||
|
return await this.hydrateResultPage(
|
||||||
|
pagedEarly ? results : results.slice(finalOffset, finalOffset + limit),
|
||||||
|
finishHybridRow
|
||||||
|
)
|
||||||
})()
|
})()
|
||||||
|
|
||||||
// Index-integrity guard — applied ONCE here so every find() path (metadata,
|
// Index-integrity guard — applied ONCE here so every find() path (metadata,
|
||||||
|
|
@ -12949,6 +12978,31 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* The text-leg twin of {@link filterIdsWithinBelted}: rank `query` INSIDE the
|
||||||
|
* candidate universe, through the provider's own posting-list merge so the
|
||||||
|
* answer can never drift from `getIdsForTextQuery`'s. A provider without the
|
||||||
|
* door is served by its whole-store answer intersected here — the same rows
|
||||||
|
* in the same order, but it pays the whole-store marshal.
|
||||||
|
*
|
||||||
|
* @param query - The text query.
|
||||||
|
* @param ids - The candidate universe (the metadata filter's ids).
|
||||||
|
* @returns `{ id, matchCount }` rows inside `ids`, ranked by match count.
|
||||||
|
*/
|
||||||
|
private async textIdsWithinBelted(
|
||||||
|
query: string,
|
||||||
|
ids: readonly string[]
|
||||||
|
): Promise<Array<{ id: string; matchCount: number }>> {
|
||||||
|
this.ensureIndexesLoaded(['metadata'])
|
||||||
|
const mip = this.metadataIndex as unknown as MetadataIndexProvider
|
||||||
|
if (typeof mip.getIdsForTextQueryWithin === 'function') {
|
||||||
|
return await mip.getIdsForTextQueryWithin(query, ids)
|
||||||
|
}
|
||||||
|
const within = new Set(ids)
|
||||||
|
const all = await this.metadataIndex.getIdsForTextQuery(query)
|
||||||
|
return all.filter((m) => within.has(m.id))
|
||||||
|
}
|
||||||
|
|
||||||
async getIndexStatus(): Promise<{
|
async getIndexStatus(): Promise<{
|
||||||
initialized: boolean
|
initialized: boolean
|
||||||
/** `true` once open()'s index-build-if-needed step has run. Named for API
|
/** `true` once open()'s index-build-if-needed step has run. Named for API
|
||||||
|
|
@ -15841,6 +15895,44 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
||||||
candidateIds?: string[],
|
candidateIds?: string[],
|
||||||
allowedIds?: OpaqueIdSet
|
allowedIds?: OpaqueIdSet
|
||||||
): Promise<Result<T>[]> {
|
): Promise<Result<T>[]> {
|
||||||
|
const scored = await this.executeVectorSearchScored(params, candidateIds, allowedIds)
|
||||||
|
|
||||||
|
// Batch-load entities for 10-50x faster cloud storage performance
|
||||||
|
// GCS: 10 results = 1×50ms vs 10×50ms = 500ms (10x faster)
|
||||||
|
const entitiesMap = await this.batchGet(scored.map((s) => s.id))
|
||||||
|
|
||||||
|
const results: Result<T>[] = []
|
||||||
|
for (const { id, score } of scored) {
|
||||||
|
const entity = entitiesMap.get(id)
|
||||||
|
if (entity) {
|
||||||
|
results.push(this.createResult(id, score, entity))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return results
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* The semantic leg WITHOUT hydration — ranked ids and their scores.
|
||||||
|
*
|
||||||
|
* The beam walk is already restricted to the candidate universe (that is what
|
||||||
|
* `candidateIds` / `allowedIds` are for), so the leg's cost is the walk. Its
|
||||||
|
* ROWS, though, are candidates for a fusion that will keep one page of them —
|
||||||
|
* so the hybrid path takes them unhydrated and reads exactly the page it
|
||||||
|
* returns. {@link executeVectorSearch} is the eager form, for the search modes
|
||||||
|
* whose leg output IS the answer.
|
||||||
|
*
|
||||||
|
* @param params - Find parameters (supplies the query/vector and the limit).
|
||||||
|
* @param candidateIds - Optional pre-resolved metadata universe (see
|
||||||
|
* {@link executeVectorSearch}).
|
||||||
|
* @param allowedIds - Optional opaque predicate-pushdown universe.
|
||||||
|
* @returns Ranked `{ id, score }` rows — no entity reads.
|
||||||
|
*/
|
||||||
|
private async executeVectorSearchScored(
|
||||||
|
params: FindParams<T>,
|
||||||
|
candidateIds?: string[],
|
||||||
|
allowedIds?: OpaqueIdSet
|
||||||
|
): Promise<Array<{ id: string; score: number }>> {
|
||||||
// Vector cold-read guard: before trusting a semantic/vector result, verify the
|
// Vector cold-read guard: before trusting a semantic/vector result, verify the
|
||||||
// vector index actually SERVES a known persisted vector (one-shot per brain).
|
// vector index actually SERVES a known persisted vector (one-shot per brain).
|
||||||
// A pure semantic find({ query }) has no filter, so verifyMetadataLive never
|
// A pure semantic find({ query }) has no filter, so verifyMetadataLive never
|
||||||
|
|
@ -15866,21 +15958,10 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
||||||
// HNSW search with optional metadata-first candidate filtering
|
// HNSW search with optional metadata-first candidate filtering
|
||||||
const searchResults: [string, number][] = await this.index.search(vector, limit * 2, undefined, searchOptions)
|
const searchResults: [string, number][] = await this.index.search(vector, limit * 2, undefined, searchOptions)
|
||||||
|
|
||||||
// Batch-load entities for 10-50x faster cloud storage performance
|
return searchResults.map(([id, distance]) => ({
|
||||||
// GCS: 10 results = 1×50ms vs 10×50ms = 500ms (10x faster)
|
id,
|
||||||
const ids = searchResults.map(([id]) => id)
|
score: Math.max(0, Math.min(1, 1 / (1 + distance)))
|
||||||
const entitiesMap = await this.batchGet(ids)
|
}))
|
||||||
|
|
||||||
const results: Result<T>[] = []
|
|
||||||
for (const [id, distance] of searchResults) {
|
|
||||||
const entity = entitiesMap.get(id)
|
|
||||||
if (entity) {
|
|
||||||
const score = Math.max(0, Math.min(1, 1 / (1 + distance)))
|
|
||||||
results.push(this.createResult(id, score, entity))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return results
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
|
@ -16180,30 +16261,64 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
||||||
* @returns Array of Results with scores based on match count
|
* @returns Array of Results with scores based on match count
|
||||||
*/
|
*/
|
||||||
private async executeTextSearch(query: string, limit: number): Promise<Result<T>[]> {
|
private async executeTextSearch(query: string, limit: number): Promise<Result<T>[]> {
|
||||||
const textMatches = await this.metadataIndex.getIdsForTextQuery(query)
|
const scored = await this.executeTextSearchScored(query, limit)
|
||||||
if (textMatches.length === 0) return []
|
if (scored.length === 0) return []
|
||||||
|
|
||||||
// Take top matches and load entities
|
// Batch-load entities for the whole leg — this is the eager form, kept for
|
||||||
const topMatches = textMatches.slice(0, limit * 2) // Get more for filtering
|
// the text-only search mode whose results ARE the answer.
|
||||||
const ids = topMatches.map(m => m.id)
|
const entitiesMap = await this.batchGet(scored.map((s) => s.id))
|
||||||
const entitiesMap = await this.batchGet(ids)
|
|
||||||
|
|
||||||
// Create results with scores based on match count
|
|
||||||
const maxMatches = topMatches[0]?.matchCount || 1
|
|
||||||
const results: Result<T>[] = []
|
const results: Result<T>[] = []
|
||||||
|
for (const { id, score } of scored) {
|
||||||
for (const match of topMatches) {
|
const entity = entitiesMap.get(id)
|
||||||
const entity = entitiesMap.get(match.id)
|
|
||||||
if (entity) {
|
if (entity) {
|
||||||
// Normalize score to 0-1 range based on match count
|
results.push(this.createResult(id, score, entity))
|
||||||
const score = match.matchCount / maxMatches
|
|
||||||
results.push(this.createResult(match.id, score, entity))
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
return results
|
return results
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* The text leg WITHOUT hydration — ranked ids and their scores.
|
||||||
|
*
|
||||||
|
* FILTER BEFORE HYDRATE: when the caller already knows the candidate
|
||||||
|
* universe (the metadata filter's ids in a hybrid `find({ query, where })`),
|
||||||
|
* it is passed here and the word index ranks INSIDE that universe. The
|
||||||
|
* earlier order ranked the whole store, took the top `limit * 2`, hydrated
|
||||||
|
* every one of them, and only then intersected with the filter — so a
|
||||||
|
* filtered hybrid find on a large store hydrated hundreds of rows to return
|
||||||
|
* a handful, and a matching row outside the store-wide text prefix was
|
||||||
|
* silently dropped (the same defect `find({ connected })` had before the
|
||||||
|
* graph-first law).
|
||||||
|
*
|
||||||
|
* The score is the match count normalized against the top row's, so a
|
||||||
|
* restricted call normalizes against the top row IN THE UNIVERSE — the same
|
||||||
|
* rule applied to the set actually being ranked.
|
||||||
|
*
|
||||||
|
* @param query - Text query to search for.
|
||||||
|
* @param limit - Result budget; the leg keeps `limit * 2` for the fusion.
|
||||||
|
* @param candidateIds - Optional candidate universe to rank inside.
|
||||||
|
* @returns Ranked `{ id, score }` rows — no entity reads.
|
||||||
|
*/
|
||||||
|
private async executeTextSearchScored(
|
||||||
|
query: string,
|
||||||
|
limit: number,
|
||||||
|
candidateIds?: readonly string[]
|
||||||
|
): Promise<Array<{ id: string; score: number }>> {
|
||||||
|
const textMatches = candidateIds
|
||||||
|
? await this.textIdsWithinBelted(query, candidateIds)
|
||||||
|
: await this.metadataIndex.getIdsForTextQuery(query)
|
||||||
|
if (textMatches.length === 0) return []
|
||||||
|
|
||||||
|
// Take top matches (more than the page, for the fusion to rank)
|
||||||
|
const topMatches = textMatches.slice(0, limit * 2)
|
||||||
|
|
||||||
|
// Normalize score to 0-1 range based on match count
|
||||||
|
const maxMatches = topMatches[0]?.matchCount || 1
|
||||||
|
return topMatches.map((m) => ({ id: m.id, score: m.matchCount / maxMatches }))
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Auto-detect optimal alpha for hybrid search
|
* Auto-detect optimal alpha for hybrid search
|
||||||
*
|
*
|
||||||
|
|
@ -16230,55 +16345,56 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
||||||
*
|
*
|
||||||
* Formula: score(d) = sum(1 / (k + rank(d))) for each list
|
* Formula: score(d) = sum(1 / (k + rank(d))) for each list
|
||||||
*
|
*
|
||||||
* Now includes match visibility (textMatches, textScore, semanticScore, matchSource)
|
* Now includes match visibility (textScore, semanticScore, matchSource; the
|
||||||
|
* `textMatches` word list needs the entity and is filled at hydration).
|
||||||
*
|
*
|
||||||
* @param textResults - Results from text search
|
* HYDRATE LAST: both legs arrive as ranked ids + scores, and the fusion ranks
|
||||||
* @param semanticResults - Results from semantic search
|
* ids — no entity is read here. The rows it returns are ranked SHELLS; the
|
||||||
|
* page is cut from them and only that page is read from canonical (see
|
||||||
|
* {@link hydrateResultPage}). The earlier order hydrated both legs in full —
|
||||||
|
* hundreds of rows — to return one page of them.
|
||||||
|
*
|
||||||
|
* @param textResults - Ranked ids + scores from text search
|
||||||
|
* @param semanticResults - Ranked ids + scores from semantic search
|
||||||
* @param alpha - Weight for semantic (0=text only, 1=semantic only)
|
* @param alpha - Weight for semantic (0=text only, 1=semantic only)
|
||||||
* @param queryWords - Original query words for match tracking
|
|
||||||
* @param k - RRF constant (default: 60, standard in literature)
|
* @param k - RRF constant (default: 60, standard in literature)
|
||||||
* @returns Fused results sorted by combined score with match visibility
|
* @returns Fused result shells sorted by combined score with match visibility
|
||||||
*/
|
*/
|
||||||
private async rrfFusion(
|
private rrfFusion(
|
||||||
textResults: Result<T>[],
|
textResults: ReadonlyArray<{ id: string; score: number }>,
|
||||||
semanticResults: Result<T>[],
|
semanticResults: ReadonlyArray<{ id: string; score: number }>,
|
||||||
alpha: number,
|
alpha: number,
|
||||||
queryWords: string[],
|
|
||||||
k: number = 60
|
k: number = 60
|
||||||
): Promise<Result<T>[]> {
|
): Result<T>[] {
|
||||||
// Track scores and match details per entity
|
// Track scores and match details per entity
|
||||||
interface MatchData {
|
interface MatchData {
|
||||||
rrf: number
|
rrf: number
|
||||||
textScore?: number
|
textScore?: number
|
||||||
semanticScore?: number
|
semanticScore?: number
|
||||||
textMatches: string[]
|
|
||||||
hasText: boolean
|
hasText: boolean
|
||||||
hasSemantic: boolean
|
hasSemantic: boolean
|
||||||
}
|
}
|
||||||
const matchData = new Map<string, MatchData>()
|
const matchData = new Map<string, MatchData>()
|
||||||
const entityMap = new Map<string, Entity<T>>()
|
|
||||||
|
|
||||||
// Text contribution (1 - alpha weight)
|
// Text contribution (1 - alpha weight)
|
||||||
const textWeight = 1 - alpha
|
const textWeight = 1 - alpha
|
||||||
textResults.forEach((r, rank) => {
|
textResults.forEach((r, rank) => {
|
||||||
const rrfScore = textWeight * (1 / (k + rank + 1))
|
const rrfScore = textWeight * (1 / (k + rank + 1))
|
||||||
const existing = matchData.get(r.id) || { rrf: 0, textMatches: [], hasText: false, hasSemantic: false }
|
const existing = matchData.get(r.id) || { rrf: 0, hasText: false, hasSemantic: false }
|
||||||
existing.rrf += rrfScore
|
existing.rrf += rrfScore
|
||||||
existing.textScore = r.score // Original text search score (0-1)
|
existing.textScore = r.score // Original text search score (0-1)
|
||||||
existing.hasText = true
|
existing.hasText = true
|
||||||
matchData.set(r.id, existing)
|
matchData.set(r.id, existing)
|
||||||
if (r.entity) entityMap.set(r.id, r.entity)
|
|
||||||
})
|
})
|
||||||
|
|
||||||
// Semantic contribution (alpha weight)
|
// Semantic contribution (alpha weight)
|
||||||
semanticResults.forEach((r, rank) => {
|
semanticResults.forEach((r, rank) => {
|
||||||
const rrfScore = alpha * (1 / (k + rank + 1))
|
const rrfScore = alpha * (1 / (k + rank + 1))
|
||||||
const existing = matchData.get(r.id) || { rrf: 0, textMatches: [], hasText: false, hasSemantic: false }
|
const existing = matchData.get(r.id) || { rrf: 0, hasText: false, hasSemantic: false }
|
||||||
existing.rrf += rrfScore
|
existing.rrf += rrfScore
|
||||||
existing.semanticScore = r.score // Original semantic search score (0-1)
|
existing.semanticScore = r.score // Original semantic search score (0-1)
|
||||||
existing.hasSemantic = true
|
existing.hasSemantic = true
|
||||||
matchData.set(r.id, existing)
|
matchData.set(r.id, existing)
|
||||||
if (r.entity) entityMap.set(r.id, r.entity)
|
|
||||||
})
|
})
|
||||||
|
|
||||||
// Sort by fused score
|
// Sort by fused score
|
||||||
|
|
@ -16286,51 +16402,93 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
||||||
.sort((a, b) => b[1].rrf - a[1].rrf)
|
.sort((a, b) => b[1].rrf - a[1].rrf)
|
||||||
.map(([id, data]) => ({ id, data }))
|
.map(([id, data]) => ({ id, data }))
|
||||||
|
|
||||||
// Build results - need to load any missing entities
|
// Create ranked shells with match visibility
|
||||||
const missingIds = sortedIds.filter(s => !entityMap.has(s.id)).map(s => s.id)
|
|
||||||
if (missingIds.length > 0) {
|
|
||||||
const loaded = await this.batchGet(missingIds)
|
|
||||||
for (const [id, entity] of loaded) {
|
|
||||||
entityMap.set(id, entity)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Performance: Build set of text result IDs for O(1) lookup
|
|
||||||
// This avoids re-extracting text for entities that weren't in text results
|
|
||||||
const textResultIds = new Set(textResults.map(r => r.id))
|
|
||||||
|
|
||||||
// Create final results with match visibility
|
|
||||||
const results: Result<T>[] = []
|
const results: Result<T>[] = []
|
||||||
for (const { id, data } of sortedIds) {
|
for (const { id, data } of sortedIds) {
|
||||||
const entity = entityMap.get(id)
|
// Determine match source
|
||||||
if (entity) {
|
let matchSource: 'text' | 'semantic' | 'both'
|
||||||
// Find which query words matched - uses fast path if entity wasn't in text results
|
if (data.hasText && data.hasSemantic) {
|
||||||
const textMatches = this.findMatchingWords(entity, queryWords, textResultIds)
|
matchSource = 'both'
|
||||||
|
} else if (data.hasText) {
|
||||||
// Determine match source
|
matchSource = 'text'
|
||||||
let matchSource: 'text' | 'semantic' | 'both'
|
} else {
|
||||||
if (data.hasText && data.hasSemantic) {
|
matchSource = 'semantic'
|
||||||
matchSource = 'both'
|
|
||||||
} else if (data.hasText) {
|
|
||||||
matchSource = 'text'
|
|
||||||
} else {
|
|
||||||
matchSource = 'semantic'
|
|
||||||
}
|
|
||||||
|
|
||||||
// Create result with match visibility
|
|
||||||
const result = this.createResult(id, data.rrf, entity)
|
|
||||||
result.textMatches = textMatches
|
|
||||||
result.textScore = data.textScore
|
|
||||||
result.semanticScore = data.semanticScore
|
|
||||||
result.matchSource = matchSource
|
|
||||||
|
|
||||||
results.push(result)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const result = this.pendingResult(id, data.rrf)
|
||||||
|
result.textScore = data.textScore
|
||||||
|
result.semanticScore = data.semanticScore
|
||||||
|
result.matchSource = matchSource
|
||||||
|
|
||||||
|
results.push(result)
|
||||||
}
|
}
|
||||||
|
|
||||||
return results
|
return results
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* A ranked candidate whose entity has NOT been read yet.
|
||||||
|
*
|
||||||
|
* The shell carries everything the ranking tail needs — the id, the score,
|
||||||
|
* and the match-visibility fields — and nothing that requires canonical. It
|
||||||
|
* is typed `Result` so it flows through the shared dedupe / visibility /
|
||||||
|
* filter / rank / page tail unchanged; {@link hydrateResultPage} turns the
|
||||||
|
* survivors into real results before any caller sees them, and find()'s
|
||||||
|
* index-integrity guard drops any row that never gained an entity.
|
||||||
|
*
|
||||||
|
* @param id - The candidate's canonical id.
|
||||||
|
* @param score - Its rank score.
|
||||||
|
*/
|
||||||
|
private pendingResult(id: string, score: number): Result<T> {
|
||||||
|
return { id, score } as Result<T>
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Read canonical for exactly the rows that need it — the hydrate-last seam.
|
||||||
|
*
|
||||||
|
* Rows that already carry an entity (the eager legs: metadata, text-only,
|
||||||
|
* semantic-only, proximity, graph) pass through untouched, so this is a no-op
|
||||||
|
* for every path that has not deferred. Rows that are shells are read in ONE
|
||||||
|
* batch and rebuilt through {@link createResult}, so a hydrated row is
|
||||||
|
* indistinguishable from an eagerly-built one — same flattened fields, same
|
||||||
|
* `entity`, same key order — with `finish` re-applying the fields only the
|
||||||
|
* deferring path knows about (a hybrid row's match visibility).
|
||||||
|
*
|
||||||
|
* A shell whose id has no canonical row is dropped, exactly as the eager legs
|
||||||
|
* dropped it; find()'s index-integrity guard makes the same judgement on the
|
||||||
|
* page it returns.
|
||||||
|
*
|
||||||
|
* @param rows - The page's rows, ranked and paged already.
|
||||||
|
* @param finish - Applied to each rebuilt row, with its shell, after the
|
||||||
|
* flattened fields are set.
|
||||||
|
* @returns The page with every surviving row hydrated.
|
||||||
|
*/
|
||||||
|
private async hydrateResultPage(
|
||||||
|
rows: Result<T>[],
|
||||||
|
finish?: (row: Result<T>, pending: Result<T>) => void
|
||||||
|
): Promise<Result<T>[]> {
|
||||||
|
const pendingIds: string[] = []
|
||||||
|
for (const row of rows) {
|
||||||
|
if (!row.entity) pendingIds.push(row.id)
|
||||||
|
}
|
||||||
|
if (pendingIds.length === 0) return rows
|
||||||
|
|
||||||
|
const entitiesMap = await this.batchGet(pendingIds)
|
||||||
|
const hydrated: Result<T>[] = []
|
||||||
|
for (const row of rows) {
|
||||||
|
if (row.entity) {
|
||||||
|
hydrated.push(row)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
const entity = entitiesMap.get(row.id)
|
||||||
|
if (!entity) continue
|
||||||
|
const filled = this.createResult(row.id, row.score, entity, row.explanation)
|
||||||
|
finish?.(filled, row)
|
||||||
|
hydrated.push(filled)
|
||||||
|
}
|
||||||
|
return hydrated
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Find which query words match in an entity's text content
|
* Find which query words match in an entity's text content
|
||||||
*
|
*
|
||||||
|
|
|
||||||
|
|
@ -473,6 +473,28 @@ export interface MetadataIndexProvider {
|
||||||
graphIndex: unknown
|
graphIndex: unknown
|
||||||
): Promise<{ ids: string[]; emptyAt: 'graph' | 'filter' | 'visibility' | 'none' } | null>
|
): Promise<{ ids: string[]; emptyAt: 'graph' | 'filter' | 'visibility' | 'none' } | null>
|
||||||
getIdsForTextQuery(query: string): Promise<Array<{ id: string; matchCount: number }>>
|
getIdsForTextQuery(query: string): Promise<Array<{ id: string; matchCount: number }>>
|
||||||
|
/**
|
||||||
|
* @description OPTIONAL: score `query` over `ids` ONLY — the text-leg twin of
|
||||||
|
* {@link filterIdsWithin}, and the door a hybrid `find({ query, where })`
|
||||||
|
* walks. The metadata filter's universe is the candidate set there, so the
|
||||||
|
* text leg must cost O(|ids|) membership checks and marshal at most `|ids|`
|
||||||
|
* rows, never the whole posting list of every query word. A native index
|
||||||
|
* intersects its own postings with the candidate set (membership by entity
|
||||||
|
* int) before any string crosses the boundary; the reference index answers
|
||||||
|
* from its own `getIdsForTextQuery`, so the two doors can never disagree.
|
||||||
|
* Absent → Brainy intersects `getIdsForTextQuery`'s answer with `ids` itself
|
||||||
|
* (correct, and still hydrate-last, but it marshals the whole answer).
|
||||||
|
*
|
||||||
|
* The answer keeps `getIdsForTextQuery`'s contract: `{ id, matchCount }`
|
||||||
|
* sorted by `matchCount` descending, ties in the order the whole-store answer
|
||||||
|
* would have produced. Only rows in `ids` may appear.
|
||||||
|
* @param query - The same text query accepted by `getIdsForTextQuery`.
|
||||||
|
* @param ids - The candidate ids (canonical). The answer is a subset.
|
||||||
|
*/
|
||||||
|
getIdsForTextQueryWithin?(
|
||||||
|
query: string,
|
||||||
|
ids: readonly string[]
|
||||||
|
): Promise<Array<{ id: string; matchCount: number }>>
|
||||||
getSortedIdsForFilter(filter: any, orderBy: string, order?: 'asc' | 'desc', topK?: number): Promise<string[]>
|
getSortedIdsForFilter(filter: any, orderBy: string, order?: 'asc' | 'desc', topK?: number): Promise<string[]>
|
||||||
getFilterValues(field: string): Promise<string[]>
|
getFilterValues(field: string): Promise<string[]>
|
||||||
getFilterFields(): Promise<string[]>
|
getFilterFields(): Promise<string[]>
|
||||||
|
|
|
||||||
|
|
@ -1509,11 +1509,56 @@ export class MetadataIndexManager implements MetadataIndexProvider {
|
||||||
* @returns Array of { id, matchCount } sorted by matchCount descending
|
* @returns Array of { id, matchCount } sorted by matchCount descending
|
||||||
*/
|
*/
|
||||||
async getIdsForTextQuery(query: string): Promise<Array<{ id: string; matchCount: number }>> {
|
async getIdsForTextQuery(query: string): Promise<Array<{ id: string; matchCount: number }>> {
|
||||||
|
return this.scoreTextQuery(query)
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Score a text query over `ids` ONLY — the reference implementation of the
|
||||||
|
* optional `getIdsForTextQueryWithin` door (see
|
||||||
|
* {@link import('../plugin.js').MetadataIndexProvider}). The hybrid
|
||||||
|
* `find({ query, where })` path passes the metadata filter's universe here so
|
||||||
|
* the text leg ranks INSIDE that universe instead of ranking the whole store
|
||||||
|
* and discarding the rows the filter would have dropped.
|
||||||
|
*
|
||||||
|
* It answers from the same posting-list merge as {@link getIdsForTextQuery},
|
||||||
|
* with the candidate membership applied as each word's postings are counted,
|
||||||
|
* so the two doors can never disagree: the answer is exactly the whole-store
|
||||||
|
* answer restricted to `ids`, in the same order.
|
||||||
|
*
|
||||||
|
* @param query - Text query to search for.
|
||||||
|
* @param ids - Candidate entity ids; only these may appear in the answer.
|
||||||
|
* @returns Array of { id, matchCount } sorted by matchCount descending.
|
||||||
|
*/
|
||||||
|
async getIdsForTextQueryWithin(
|
||||||
|
query: string,
|
||||||
|
ids: readonly string[]
|
||||||
|
): Promise<Array<{ id: string; matchCount: number }>> {
|
||||||
|
if (ids.length === 0) return []
|
||||||
|
return this.scoreTextQuery(query, new Set(ids))
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* The one posting-list merge behind both text doors.
|
||||||
|
*
|
||||||
|
* Each query word contributes AT MOST one match per entity (a posting list
|
||||||
|
* can name an id more than once), and entities are ranked by how many of the
|
||||||
|
* query's words they matched. `within`, when given, restricts the count to
|
||||||
|
* those candidates — applied during the merge, so a restricted call never
|
||||||
|
* materializes a whole-store match map.
|
||||||
|
*
|
||||||
|
* @param query - Text query to search for.
|
||||||
|
* @param within - Optional candidate universe; absent = the whole store.
|
||||||
|
* @returns Array of { id, matchCount } sorted by matchCount descending.
|
||||||
|
*/
|
||||||
|
private async scoreTextQuery(
|
||||||
|
query: string,
|
||||||
|
within?: ReadonlySet<string>
|
||||||
|
): Promise<Array<{ id: string; matchCount: number }>> {
|
||||||
const queryWords = this.tokenize(query)
|
const queryWords = this.tokenize(query)
|
||||||
if (queryWords.length === 0) return []
|
if (queryWords.length === 0) return []
|
||||||
|
|
||||||
// Get IDs for each word hash
|
// Count matches per entity, one word's postings at a time.
|
||||||
const wordIdSets: Map<string, number>[] = []
|
const matchCounts = new Map<string, number>()
|
||||||
for (const word of queryWords) {
|
for (const word of queryWords) {
|
||||||
const wordHash = this.hashWord(word)
|
const wordHash = this.hashWord(word)
|
||||||
let ids: string[]
|
let ids: string[]
|
||||||
|
|
@ -1529,19 +1574,12 @@ export class MetadataIndexManager implements MetadataIndexProvider {
|
||||||
throw err
|
throw err
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
const idSet = new Map<string, number>()
|
// One count per (word, entity) — dedupe this word's postings first.
|
||||||
|
const counted = new Set<string>()
|
||||||
for (const id of ids) {
|
for (const id of ids) {
|
||||||
idSet.set(id, 1)
|
if (counted.has(id)) continue
|
||||||
}
|
counted.add(id)
|
||||||
wordIdSets.push(idSet)
|
if (within && !within.has(id)) continue
|
||||||
}
|
|
||||||
|
|
||||||
if (wordIdSets.length === 0) return []
|
|
||||||
|
|
||||||
// Count matches per entity
|
|
||||||
const matchCounts = new Map<string, number>()
|
|
||||||
for (const idSet of wordIdSets) {
|
|
||||||
for (const [id] of idSet) {
|
|
||||||
matchCounts.set(id, (matchCounts.get(id) || 0) + 1)
|
matchCounts.set(id, (matchCounts.get(id) || 0) + 1)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
||||||
635
tests/integration/find-hybrid-filter-before-hydrate.test.ts
Normal file
635
tests/integration/find-hybrid-filter-before-hydrate.test.ts
Normal file
|
|
@ -0,0 +1,635 @@
|
||||||
|
/**
|
||||||
|
* @module tests/integration/find-hybrid-filter-before-hydrate
|
||||||
|
* @description FILTER BEFORE HYDRATE, applied to the hybrid `find({ query })` path.
|
||||||
|
*
|
||||||
|
* A hybrid find fuses two legs. The semantic leg already walked only the
|
||||||
|
* metadata filter's universe (`candidateIds` / `allowedIds`). The TEXT leg did
|
||||||
|
* not: it ranked the WHOLE store, took the top `limit * 4`, read every one of
|
||||||
|
* those rows from canonical, and only then intersected with the filter — so a
|
||||||
|
* filtered hybrid find on a large store read hundreds of rows to return a
|
||||||
|
* handful of them, and a matching row outside the store-wide text prefix was
|
||||||
|
* silently dropped. That is the same defect `find({ connected })` carried
|
||||||
|
* before the graph-first law, one leg over.
|
||||||
|
*
|
||||||
|
* Both halves are pinned here.
|
||||||
|
*
|
||||||
|
* THE ANSWER. Where the filter did not truncate the text leg — the universe
|
||||||
|
* covers every text match, so both orders rank the same rows — the new
|
||||||
|
* pipeline's answer is IDENTICAL to the old one's: same rows, same order, same
|
||||||
|
* scores, same match visibility, same row shape. The oracle below is the
|
||||||
|
* pre-change pipeline itself, replayed on the same brain through the same
|
||||||
|
* doors, so the comparison is against what actually ran, not a remembered
|
||||||
|
* expectation.
|
||||||
|
*
|
||||||
|
* THE CORRECTION. Where the filter DID truncate it — the query's words are
|
||||||
|
* common outside the universe — the old order let the text leg contribute
|
||||||
|
* nothing at all: every row it ranked was discarded by the filter, and the
|
||||||
|
* answer came from the semantic leg alone. The new order ranks inside the
|
||||||
|
* universe, so the text leg contributes the rows it always should have.
|
||||||
|
*
|
||||||
|
* THE COST. Canonical is read for exactly the page: one batch, `limit` rows,
|
||||||
|
* never the legs. And the text leg is asked about the universe's ids only —
|
||||||
|
* what it marshals is bounded by the universe, not by the store.
|
||||||
|
*/
|
||||||
|
import { describe, it, expect, beforeAll, vi } from 'vitest'
|
||||||
|
import { Brainy } from '../../src/brainy'
|
||||||
|
import { NounType, VerbType } from '../../src/types/graphTypes'
|
||||||
|
import { rankIndicesByScore, reorderByIndices } from '../../src/utils/resultRanking'
|
||||||
|
import { resolveEntityId } from '../../src/utils/idNormalization'
|
||||||
|
|
||||||
|
/** Embedding width of the default model — the row vectors must match it. */
|
||||||
|
const DIM = 384
|
||||||
|
|
||||||
|
/**
|
||||||
|
* A deterministic, per-row-distinct unit vector. Distinct so the semantic leg
|
||||||
|
* has a real ranking to produce (identical vectors would make its order a tie
|
||||||
|
* break), deterministic so the oracle and the pipeline see the same one.
|
||||||
|
*/
|
||||||
|
function seededVector(seed: number): number[] {
|
||||||
|
const v = new Array<number>(DIM)
|
||||||
|
for (let i = 0; i < DIM; i++) {
|
||||||
|
v[i] = Math.sin((i + 1) * 0.11 + seed * 0.37) * 0.5 + Math.cos((i + 1) * 0.05 + seed * 0.13) * 0.3
|
||||||
|
}
|
||||||
|
const magnitude = Math.sqrt(v.reduce((sum, x) => sum + x * x, 0))
|
||||||
|
return v.map((x) => x / magnitude)
|
||||||
|
}
|
||||||
|
|
||||||
|
/** The fields a caller reads off a hybrid row — the whole comparable surface. */
|
||||||
|
function project(rows: any[]): any[] {
|
||||||
|
return rows.map((r) => ({
|
||||||
|
id: r.id,
|
||||||
|
score: r.score,
|
||||||
|
type: r.type,
|
||||||
|
metadata: r.metadata,
|
||||||
|
textMatches: r.textMatches,
|
||||||
|
textScore: r.textScore,
|
||||||
|
semanticScore: r.semanticScore,
|
||||||
|
matchSource: r.matchSource
|
||||||
|
}))
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* The PRE-CHANGE hybrid pipeline, replayed on a live brain through the same
|
||||||
|
* provider doors it used: whole-store text ranking with both legs hydrated in
|
||||||
|
* full, RRF fusion, then the metadata intersection, then the page.
|
||||||
|
*
|
||||||
|
* Supports the shapes these pins exercise (query + where/type/excludeVFS +
|
||||||
|
* connected + offset); `orderBy`, `fusion` and `near` are not replayed.
|
||||||
|
*/
|
||||||
|
async function legacyHybridFind(brain: any, params: any): Promise<any[]> {
|
||||||
|
const index = brain.metadataIndex
|
||||||
|
const limit = params.limit ?? 10
|
||||||
|
const offset = params.offset ?? 0
|
||||||
|
const hasFilter = Boolean(
|
||||||
|
params.where || params.type || params.subtype || params.service || params.excludeVFS
|
||||||
|
)
|
||||||
|
|
||||||
|
let preResolvedMetadataIds: string[] | null = null
|
||||||
|
let preResolvedFilter: any = null
|
||||||
|
let graphFirstIds: string[] | null = null
|
||||||
|
|
||||||
|
if (params.connected) {
|
||||||
|
// find() normalizes the anchors to canonical ids before this stage runs.
|
||||||
|
const anchored = {
|
||||||
|
...params,
|
||||||
|
connected: {
|
||||||
|
...params.connected,
|
||||||
|
...(params.connected.from && { from: resolveEntityId(params.connected.from) }),
|
||||||
|
...(params.connected.to && { to: resolveEntityId(params.connected.to) })
|
||||||
|
}
|
||||||
|
}
|
||||||
|
graphFirstIds = await brain.resolveConnectedIds(anchored)
|
||||||
|
if (graphFirstIds!.length > 0 && hasFilter) {
|
||||||
|
preResolvedFilter = brain.buildMetadataFilter(params)
|
||||||
|
graphFirstIds = await brain.filterIdsWithinBelted(preResolvedFilter, graphFirstIds)
|
||||||
|
}
|
||||||
|
if (graphFirstIds!.length === 0) return []
|
||||||
|
preResolvedMetadataIds = graphFirstIds
|
||||||
|
} else if (hasFilter) {
|
||||||
|
preResolvedFilter = brain.buildMetadataFilter(params)
|
||||||
|
preResolvedMetadataIds = await brain.filterIdsBelted(preResolvedFilter)
|
||||||
|
if (preResolvedMetadataIds!.length === 0) return []
|
||||||
|
}
|
||||||
|
|
||||||
|
// Text leg — the whole store, then the top `limit * 4`, hydrated in full.
|
||||||
|
const allTextMatches = await index.getIdsForTextQuery(params.query)
|
||||||
|
const topMatches = allTextMatches.slice(0, limit * 2 * 2)
|
||||||
|
const maxMatches = topMatches[0]?.matchCount || 1
|
||||||
|
const textEntities = await brain.batchGet(topMatches.map((m: any) => m.id))
|
||||||
|
const textResults = topMatches
|
||||||
|
.filter((m: any) => textEntities.has(m.id))
|
||||||
|
.map((m: any) => ({ id: m.id, score: m.matchCount / maxMatches }))
|
||||||
|
|
||||||
|
// Semantic leg — the beam walk over the universe, hydrated in full.
|
||||||
|
const vector = await brain.embed(params.query)
|
||||||
|
const searchOptions = preResolvedMetadataIds ? { candidateIds: preResolvedMetadataIds } : undefined
|
||||||
|
const searchResults: [string, number][] = await brain.index.search(
|
||||||
|
vector,
|
||||||
|
limit * 2,
|
||||||
|
undefined,
|
||||||
|
searchOptions
|
||||||
|
)
|
||||||
|
const semanticEntities = await brain.batchGet(searchResults.map(([id]) => id))
|
||||||
|
const semanticResults = searchResults
|
||||||
|
.filter(([id]) => semanticEntities.has(id))
|
||||||
|
.map(([id, distance]) => ({ id, score: Math.max(0, Math.min(1, 1 / (1 + distance))) }))
|
||||||
|
|
||||||
|
// RRF fusion, with the match visibility the rows carried.
|
||||||
|
const alpha = params.hybridAlpha ?? brain.autoAlpha(params.query)
|
||||||
|
const k = 60
|
||||||
|
const matchData = new Map<string, any>()
|
||||||
|
const textWeight = 1 - alpha
|
||||||
|
textResults.forEach((r: any, rank: number) => {
|
||||||
|
const existing = matchData.get(r.id) || { rrf: 0, hasText: false, hasSemantic: false }
|
||||||
|
existing.rrf += textWeight * (1 / (k + rank + 1))
|
||||||
|
existing.textScore = r.score
|
||||||
|
existing.hasText = true
|
||||||
|
matchData.set(r.id, existing)
|
||||||
|
})
|
||||||
|
semanticResults.forEach((r: any, rank: number) => {
|
||||||
|
const existing = matchData.get(r.id) || { rrf: 0, hasText: false, hasSemantic: false }
|
||||||
|
existing.rrf += alpha * (1 / (k + rank + 1))
|
||||||
|
existing.semanticScore = r.score
|
||||||
|
existing.hasSemantic = true
|
||||||
|
matchData.set(r.id, existing)
|
||||||
|
})
|
||||||
|
|
||||||
|
const queryWords: string[] = index.tokenize(params.query)
|
||||||
|
const textResultIds = new Set(textResults.map((r: any) => r.id))
|
||||||
|
const fusedIds = Array.from(matchData.entries())
|
||||||
|
.sort((a, b) => b[1].rrf - a[1].rrf)
|
||||||
|
.map(([id, data]) => ({ id, data }))
|
||||||
|
|
||||||
|
const allEntities = await brain.batchGet(fusedIds.map((f) => f.id))
|
||||||
|
let rows: any[] = []
|
||||||
|
for (const { id, data } of fusedIds) {
|
||||||
|
const entity = allEntities.get(id)
|
||||||
|
if (!entity) continue
|
||||||
|
const textContent = textResultIds.has(id)
|
||||||
|
? index.extractTextContent({ data: entity.data, metadata: entity.metadata }).toLowerCase()
|
||||||
|
: null
|
||||||
|
rows.push({
|
||||||
|
id,
|
||||||
|
score: data.rrf,
|
||||||
|
type: entity.type,
|
||||||
|
metadata: entity.metadata,
|
||||||
|
textMatches:
|
||||||
|
textContent === null ? [] : queryWords.filter((w) => textContent.includes(w.toLowerCase())),
|
||||||
|
textScore: data.textScore,
|
||||||
|
semanticScore: data.semanticScore,
|
||||||
|
matchSource: data.hasText && data.hasSemantic ? 'both' : data.hasText ? 'text' : 'semantic'
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
// The metadata intersection — after the legs, as it was.
|
||||||
|
if (preResolvedMetadataIds && preResolvedFilter) {
|
||||||
|
const filteredIdSet = new Set(preResolvedMetadataIds)
|
||||||
|
rows = rows.filter((r) => filteredIdSet.has(r.id))
|
||||||
|
}
|
||||||
|
if (graphFirstIds !== null) {
|
||||||
|
const neighbourSet = new Set(graphFirstIds)
|
||||||
|
rows = rows.filter((r) => neighbourSet.has(r.id))
|
||||||
|
}
|
||||||
|
|
||||||
|
// Rank to the page, then cut it.
|
||||||
|
const order = rankIndicesByScore(
|
||||||
|
rows.map((r) => r.score),
|
||||||
|
offset + limit,
|
||||||
|
true
|
||||||
|
)
|
||||||
|
return reorderByIndices(rows, order).slice(offset, offset + limit)
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* FIXTURE A — the filter's universe covers every text match, so the two orders
|
||||||
|
* rank exactly the same rows and the answers must be identical.
|
||||||
|
*/
|
||||||
|
describe('hybrid find: filter before hydrate — the answer is unchanged', () => {
|
||||||
|
let brain: Brainy<any>
|
||||||
|
const QUERY = 'orbital telemetry'
|
||||||
|
const MATCHES = 24
|
||||||
|
const FILLER = 120
|
||||||
|
const OUTSIDE = 30
|
||||||
|
const VFS = 10
|
||||||
|
const RETRACTED = 6
|
||||||
|
const anchor = 'array-anchor'
|
||||||
|
const matchIds: string[] = []
|
||||||
|
|
||||||
|
beforeAll(async () => {
|
||||||
|
brain = new Brainy({ requireSubtype: false, storage: { type: 'memory' } })
|
||||||
|
await brain.init()
|
||||||
|
|
||||||
|
let seed = 1
|
||||||
|
await brain.add({
|
||||||
|
id: anchor,
|
||||||
|
data: 'ground station anchor record',
|
||||||
|
type: NounType.Thing,
|
||||||
|
metadata: { lane: 'alpha', role: 'anchor' },
|
||||||
|
vector: seededVector(seed++)
|
||||||
|
})
|
||||||
|
|
||||||
|
// Rows the query's words actually match — all inside every filter below.
|
||||||
|
for (let i = 0; i < MATCHES; i++) {
|
||||||
|
const id = `match-${i}`
|
||||||
|
await brain.add({
|
||||||
|
id,
|
||||||
|
data: `orbital telemetry packet ${i} recorded downlink`,
|
||||||
|
type: NounType.Document,
|
||||||
|
metadata: { lane: 'alpha', rank: i },
|
||||||
|
vector: seededVector(seed++)
|
||||||
|
})
|
||||||
|
matchIds.push(resolveEntityId(id))
|
||||||
|
await brain.relate({ from: anchor, to: id, type: VerbType.RelatedTo })
|
||||||
|
}
|
||||||
|
// Rows inside the universe that the query's words do NOT match.
|
||||||
|
for (let i = 0; i < FILLER; i++) {
|
||||||
|
await brain.add({
|
||||||
|
id: `filler-${i}`,
|
||||||
|
data: `cistern ledger entry ${i} archived`,
|
||||||
|
type: NounType.Document,
|
||||||
|
metadata: { lane: 'alpha', rank: 1000 + i },
|
||||||
|
vector: seededVector(seed++)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
// Rows outside the universe.
|
||||||
|
for (let i = 0; i < OUTSIDE; i++) {
|
||||||
|
await brain.add({
|
||||||
|
id: `outside-${i}`,
|
||||||
|
data: `unrelated dossier ${i}`,
|
||||||
|
type: NounType.Person,
|
||||||
|
metadata: { lane: 'beta' },
|
||||||
|
vector: seededVector(seed++)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
// VFS infrastructure rows — excluded by excludeVFS.
|
||||||
|
for (let i = 0; i < VFS; i++) {
|
||||||
|
await brain.add({
|
||||||
|
id: `vfs-${i}`,
|
||||||
|
data: `mounted path ${i}`,
|
||||||
|
type: NounType.Document,
|
||||||
|
metadata: { lane: 'alpha', vfsType: 'file' },
|
||||||
|
vector: seededVector(seed++)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
// Retracted rows — excluded by a `missing` negation.
|
||||||
|
for (let i = 0; i < RETRACTED; i++) {
|
||||||
|
await brain.add({
|
||||||
|
id: `retracted-${i}`,
|
||||||
|
data: `withdrawn note ${i}`,
|
||||||
|
type: NounType.Document,
|
||||||
|
metadata: { lane: 'alpha', retracted: true },
|
||||||
|
vector: seededVector(seed++)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
// The reference index has no opaque-set door, so the pipeline and the
|
||||||
|
// oracle both restrict the beam walk with the materialized candidate ids.
|
||||||
|
expect(typeof (brain as any).metadataIndex.getIdSetForFilter).not.toBe('function')
|
||||||
|
})
|
||||||
|
|
||||||
|
it('the fixture does not truncate the text leg — the universe covers every text match', async () => {
|
||||||
|
const index = (brain as any).metadataIndex
|
||||||
|
const textMatches = await index.getIdsForTextQuery(QUERY)
|
||||||
|
expect(textMatches).toHaveLength(MATCHES)
|
||||||
|
const universe = await (brain as any).filterIdsBelted({ lane: 'alpha' })
|
||||||
|
const inUniverse = new Set(universe)
|
||||||
|
for (const m of textMatches) expect(inUniverse.has(m.id)).toBe(true)
|
||||||
|
})
|
||||||
|
|
||||||
|
it('hybrid + where: identical rows, identical order, identical scores', async () => {
|
||||||
|
const params = { query: QUERY, where: { lane: 'alpha' }, limit: 8 }
|
||||||
|
const expected = await legacyHybridFind(brain as any, params)
|
||||||
|
const actual = await brain.find(params as any)
|
||||||
|
expect(actual.length).toBe(expected.length)
|
||||||
|
expect(project(actual)).toEqual(expected)
|
||||||
|
})
|
||||||
|
|
||||||
|
it('hybrid + where + offset: identical page two', async () => {
|
||||||
|
const params = { query: QUERY, where: { lane: 'alpha' }, limit: 6, offset: 6 }
|
||||||
|
const expected = await legacyHybridFind(brain as any, params)
|
||||||
|
const actual = await brain.find(params as any)
|
||||||
|
expect(actual.length).toBe(expected.length)
|
||||||
|
expect(project(actual)).toEqual(expected)
|
||||||
|
})
|
||||||
|
|
||||||
|
it('hybrid + type list + excludeVFS + a `missing` negation: identical', async () => {
|
||||||
|
const params = {
|
||||||
|
query: QUERY,
|
||||||
|
type: [NounType.Document, NounType.Person],
|
||||||
|
excludeVFS: true,
|
||||||
|
where: { lane: 'alpha', retracted: { missing: true } },
|
||||||
|
limit: 8
|
||||||
|
}
|
||||||
|
const expected = await legacyHybridFind(brain as any, params)
|
||||||
|
const actual = await brain.find(params as any)
|
||||||
|
expect(actual.length).toBe(expected.length)
|
||||||
|
expect(project(actual)).toEqual(expected)
|
||||||
|
for (const r of actual) {
|
||||||
|
expect(r.metadata.retracted).toBeUndefined()
|
||||||
|
expect(r.metadata.vfsType).toBeUndefined()
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
it('hybrid + type list + excludeVFS + a `missing` negation, offset: identical', async () => {
|
||||||
|
const params = {
|
||||||
|
query: QUERY,
|
||||||
|
type: [NounType.Document, NounType.Person],
|
||||||
|
excludeVFS: true,
|
||||||
|
where: { lane: 'alpha', retracted: { missing: true } },
|
||||||
|
limit: 5,
|
||||||
|
offset: 5
|
||||||
|
}
|
||||||
|
const expected = await legacyHybridFind(brain as any, params)
|
||||||
|
const actual = await brain.find(params as any)
|
||||||
|
expect(actual.length).toBe(expected.length)
|
||||||
|
expect(project(actual)).toEqual(expected)
|
||||||
|
})
|
||||||
|
|
||||||
|
it('hybrid + connected: identical, and never a non-neighbour', async () => {
|
||||||
|
const params = {
|
||||||
|
query: QUERY,
|
||||||
|
connected: { from: anchor, direction: 'out' as const },
|
||||||
|
where: { lane: 'alpha' },
|
||||||
|
limit: 8
|
||||||
|
}
|
||||||
|
const expected = await legacyHybridFind(brain as any, params)
|
||||||
|
const actual = await brain.find(params as any)
|
||||||
|
expect(actual.length).toBe(expected.length)
|
||||||
|
expect(project(actual)).toEqual(expected)
|
||||||
|
const neighbours = new Set(matchIds)
|
||||||
|
for (const r of actual) expect(neighbours.has(r.id)).toBe(true)
|
||||||
|
})
|
||||||
|
|
||||||
|
it('hybrid + connected + offset: page two is the page, not an empty answer', async () => {
|
||||||
|
const params = {
|
||||||
|
query: QUERY,
|
||||||
|
connected: { from: anchor, direction: 'out' as const },
|
||||||
|
where: { lane: 'alpha' },
|
||||||
|
limit: 5,
|
||||||
|
offset: 5
|
||||||
|
}
|
||||||
|
const expected = await legacyHybridFind(brain as any, params)
|
||||||
|
expect(expected).toHaveLength(5)
|
||||||
|
const actual = await brain.find(params as any)
|
||||||
|
expect(actual.length).toBe(expected.length)
|
||||||
|
expect(project(actual)).toEqual(expected)
|
||||||
|
})
|
||||||
|
|
||||||
|
it('hybrid + connected: paging reaches every matching neighbour exactly once', async () => {
|
||||||
|
const seen = new Set<string>()
|
||||||
|
for (let offset = 0; offset < MATCHES; offset += 6) {
|
||||||
|
const page = await brain.find({
|
||||||
|
query: QUERY,
|
||||||
|
connected: { from: anchor, direction: 'out' as const },
|
||||||
|
where: { lane: 'alpha' },
|
||||||
|
limit: 6,
|
||||||
|
offset
|
||||||
|
} as any)
|
||||||
|
for (const r of page) {
|
||||||
|
expect(seen.has(r.id)).toBe(false)
|
||||||
|
seen.add(r.id)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// Every row the fused candidate set holds is reachable by paging, and the
|
||||||
|
// neighbour set is the ceiling.
|
||||||
|
expect(seen.size).toBeGreaterThanOrEqual(MATCHES)
|
||||||
|
const neighbours = new Set(matchIds)
|
||||||
|
for (const id of seen) expect(neighbours.has(id)).toBe(true)
|
||||||
|
})
|
||||||
|
|
||||||
|
it('hybrid + fusion + offset: page two is the page', async () => {
|
||||||
|
const plain = await brain.find({
|
||||||
|
query: QUERY,
|
||||||
|
where: { lane: 'alpha' },
|
||||||
|
limit: 5,
|
||||||
|
offset: 5
|
||||||
|
} as any)
|
||||||
|
const fused = await brain.find({
|
||||||
|
query: QUERY,
|
||||||
|
where: { lane: 'alpha' },
|
||||||
|
fusion: 'weighted',
|
||||||
|
limit: 5,
|
||||||
|
offset: 5
|
||||||
|
} as any)
|
||||||
|
expect(fused).toHaveLength(plain.length)
|
||||||
|
expect(fused.map((r) => r.id)).toEqual(plain.map((r) => r.id))
|
||||||
|
})
|
||||||
|
|
||||||
|
it('a hydrated hybrid row is shaped exactly as an eagerly-built one', async () => {
|
||||||
|
const rows = await brain.find({ query: QUERY, where: { lane: 'alpha' }, limit: 8 } as any)
|
||||||
|
const row = rows[0]
|
||||||
|
expect(Object.keys(row)).toEqual([
|
||||||
|
'id',
|
||||||
|
'score',
|
||||||
|
'type',
|
||||||
|
'subtype',
|
||||||
|
'visibility',
|
||||||
|
'metadata',
|
||||||
|
'data',
|
||||||
|
'confidence',
|
||||||
|
'weight',
|
||||||
|
'_rev',
|
||||||
|
'entity',
|
||||||
|
'textMatches',
|
||||||
|
'textScore',
|
||||||
|
'semanticScore',
|
||||||
|
'matchSource'
|
||||||
|
])
|
||||||
|
// The flattened fields are projections of the entity, as always.
|
||||||
|
expect(row.entity).toBeDefined()
|
||||||
|
expect(row.type).toBe(row.entity.type)
|
||||||
|
expect(row.metadata).toBe(row.entity.metadata)
|
||||||
|
expect(row.data).toBe(row.entity.data)
|
||||||
|
expect(row._rev).toBe(row.entity._rev)
|
||||||
|
// The match visibility survives the deferral — every leg's fields, on the
|
||||||
|
// rows that leg contributed, exactly as the eager pipeline set them.
|
||||||
|
expect(['text', 'semantic', 'both']).toContain(row.matchSource)
|
||||||
|
for (const r of rows) {
|
||||||
|
if (r.matchSource === 'semantic') {
|
||||||
|
expect(r.textMatches).toEqual([])
|
||||||
|
expect(r.textScore).toBeUndefined()
|
||||||
|
} else {
|
||||||
|
expect(r.textMatches).toEqual(['orbital', 'telemetry'])
|
||||||
|
expect(typeof r.textScore).toBe('number')
|
||||||
|
}
|
||||||
|
if (r.matchSource === 'text') {
|
||||||
|
expect(r.semanticScore).toBeUndefined()
|
||||||
|
} else {
|
||||||
|
expect(typeof r.semanticScore).toBe('number')
|
||||||
|
}
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
it('reads canonical for the page only — one batch, `limit` rows', async () => {
|
||||||
|
// Warm any first-read verification before the counters are read.
|
||||||
|
await brain.find({ query: QUERY, where: { lane: 'alpha' }, limit: 1 } as any)
|
||||||
|
|
||||||
|
const hydrate = vi.spyOn(brain as any, 'batchGet')
|
||||||
|
try {
|
||||||
|
const results = await brain.find({ query: QUERY, where: { lane: 'alpha' }, limit: 10 } as any)
|
||||||
|
expect(results).toHaveLength(10)
|
||||||
|
expect(hydrate).toHaveBeenCalledTimes(1)
|
||||||
|
expect((hydrate.mock.calls[0][0] as string[]).length).toBe(10)
|
||||||
|
} finally {
|
||||||
|
hydrate.mockRestore()
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
it('asks the text index about the universe only, never the whole store', async () => {
|
||||||
|
const index = (brain as any).metadataIndex
|
||||||
|
const wholeStore = vi.spyOn(index, 'getIdsForTextQuery')
|
||||||
|
const within = vi.spyOn(index, 'getIdsForTextQueryWithin')
|
||||||
|
try {
|
||||||
|
await brain.find({ query: QUERY, where: { lane: 'alpha' }, limit: 10 } as any)
|
||||||
|
expect(wholeStore).not.toHaveBeenCalled()
|
||||||
|
expect(within).toHaveBeenCalledTimes(1)
|
||||||
|
|
||||||
|
const askedIds = within.mock.calls[0][1] as string[]
|
||||||
|
const universe = await (brain as any).filterIdsBelted({ lane: 'alpha' })
|
||||||
|
expect(askedIds).toHaveLength(universe.length)
|
||||||
|
|
||||||
|
// What the text leg marshals is bounded by the universe, not the store.
|
||||||
|
const marshalled = (await within.mock.results[0].value) as unknown[]
|
||||||
|
expect(marshalled.length).toBeLessThanOrEqual(universe.length)
|
||||||
|
expect(marshalled).toHaveLength(MATCHES)
|
||||||
|
} finally {
|
||||||
|
wholeStore.mockRestore()
|
||||||
|
within.mockRestore()
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
it('the two text doors agree: within is the whole-store answer restricted', async () => {
|
||||||
|
const index = (brain as any).metadataIndex
|
||||||
|
const universe: string[] = await (brain as any).filterIdsBelted({
|
||||||
|
lane: 'alpha',
|
||||||
|
retracted: { missing: true }
|
||||||
|
})
|
||||||
|
const inUniverse = new Set(universe)
|
||||||
|
const whole = await index.getIdsForTextQuery(QUERY)
|
||||||
|
const within = await index.getIdsForTextQueryWithin(QUERY, universe)
|
||||||
|
expect(within).toEqual(whole.filter((m: any) => inUniverse.has(m.id)))
|
||||||
|
expect(await index.getIdsForTextQueryWithin(QUERY, [])).toEqual([])
|
||||||
|
})
|
||||||
|
})
|
||||||
|
|
||||||
|
/**
|
||||||
|
* FIXTURE B — the query's words are common OUTSIDE the universe, so the old
|
||||||
|
* order's text leg was entirely consumed by rows the filter then discarded.
|
||||||
|
* This is the corrected behaviour, held by name.
|
||||||
|
*/
|
||||||
|
describe('hybrid find: the text leg ranks inside the filter, not around it', () => {
|
||||||
|
let brain: Brainy<any>
|
||||||
|
const QUERY = 'orbital telemetry drift'
|
||||||
|
const NOISE = 150
|
||||||
|
const KEEP = 15
|
||||||
|
const keepIds: string[] = []
|
||||||
|
|
||||||
|
beforeAll(async () => {
|
||||||
|
brain = new Brainy({ requireSubtype: false, storage: { type: 'memory' } })
|
||||||
|
await brain.init()
|
||||||
|
|
||||||
|
let seed = 5000
|
||||||
|
// Added FIRST and matching one more query word, so they lead the
|
||||||
|
// store-wide text ranking outright — and none of them pass the filter.
|
||||||
|
for (let i = 0; i < NOISE; i++) {
|
||||||
|
await brain.add({
|
||||||
|
id: `noise-${i}`,
|
||||||
|
data: `orbital telemetry drift report ${i}`,
|
||||||
|
type: NounType.Document,
|
||||||
|
metadata: { lane: 'beta' },
|
||||||
|
vector: seededVector(seed++)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
for (let i = 0; i < KEEP; i++) {
|
||||||
|
const id = `keep-${i}`
|
||||||
|
await brain.add({
|
||||||
|
id,
|
||||||
|
data: `orbital telemetry summary ${i}`,
|
||||||
|
type: NounType.Document,
|
||||||
|
metadata: { lane: 'alpha' },
|
||||||
|
vector: seededVector(seed++)
|
||||||
|
})
|
||||||
|
keepIds.push(resolveEntityId(id))
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
it('the old order let the filter consume the whole text leg', async () => {
|
||||||
|
const index = (brain as any).metadataIndex
|
||||||
|
const universe: string[] = await (brain as any).filterIdsBelted({ lane: 'alpha' })
|
||||||
|
expect(universe).toHaveLength(KEEP)
|
||||||
|
const inUniverse = new Set(universe)
|
||||||
|
|
||||||
|
// The store-wide prefix the old text leg took (limit 10 → limit * 4).
|
||||||
|
const prefix = (await index.getIdsForTextQuery(QUERY)).slice(0, 40)
|
||||||
|
expect(prefix).toHaveLength(40)
|
||||||
|
expect(prefix.filter((m: any) => inUniverse.has(m.id))).toHaveLength(0)
|
||||||
|
|
||||||
|
// Every row the old text leg ranked was then discarded by the filter, so
|
||||||
|
// the old answer carried NO text contribution at all — fifteen rows that
|
||||||
|
// match the query's words exactly, and not one of them reached the page
|
||||||
|
// through the text leg. What the old order returned was whatever the
|
||||||
|
// semantic leg alone happened to reach.
|
||||||
|
const legacy = await legacyHybridFind(brain as any, {
|
||||||
|
query: QUERY,
|
||||||
|
where: { lane: 'alpha' },
|
||||||
|
limit: 10
|
||||||
|
})
|
||||||
|
for (const r of legacy) {
|
||||||
|
expect(r.matchSource).toBe('semantic')
|
||||||
|
expect(r.textScore).toBeUndefined()
|
||||||
|
expect(r.textMatches).toEqual([])
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
it('the new order ranks the text leg inside the universe', async () => {
|
||||||
|
const results = await brain.find({
|
||||||
|
query: QUERY,
|
||||||
|
where: { lane: 'alpha' },
|
||||||
|
limit: 10
|
||||||
|
} as any)
|
||||||
|
|
||||||
|
expect(results).toHaveLength(10)
|
||||||
|
const keeps = new Set(keepIds)
|
||||||
|
for (const r of results) {
|
||||||
|
expect(keeps.has(r.id)).toBe(true)
|
||||||
|
expect(r.metadata.lane).toBe('alpha')
|
||||||
|
// The text leg is the contributor the old order threw away.
|
||||||
|
expect(['text', 'both']).toContain(r.matchSource)
|
||||||
|
expect(r.textScore).toBe(1)
|
||||||
|
expect(r.textMatches).toEqual(['orbital', 'telemetry'])
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
it('paging reaches every matching row the old order could not see', async () => {
|
||||||
|
const seen = new Set<string>()
|
||||||
|
for (let offset = 0; offset < KEEP; offset += 5) {
|
||||||
|
const page = await brain.find({
|
||||||
|
query: QUERY,
|
||||||
|
where: { lane: 'alpha' },
|
||||||
|
limit: 5,
|
||||||
|
offset
|
||||||
|
} as any)
|
||||||
|
expect(page).toHaveLength(5)
|
||||||
|
for (const r of page) {
|
||||||
|
expect(seen.has(r.id)).toBe(false)
|
||||||
|
seen.add(r.id)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
expect(seen.size).toBe(KEEP)
|
||||||
|
expect([...seen].sort()).toEqual([...keepIds].sort())
|
||||||
|
})
|
||||||
|
|
||||||
|
it('reads canonical for the page only, on the truncating shape too', async () => {
|
||||||
|
await brain.find({ query: QUERY, where: { lane: 'alpha' }, limit: 1 } as any)
|
||||||
|
|
||||||
|
const hydrate = vi.spyOn(brain as any, 'batchGet')
|
||||||
|
try {
|
||||||
|
const results = await brain.find({ query: QUERY, where: { lane: 'alpha' }, limit: 10 } as any)
|
||||||
|
expect(results).toHaveLength(10)
|
||||||
|
expect(hydrate).toHaveBeenCalledTimes(1)
|
||||||
|
expect((hydrate.mock.calls[0][0] as string[]).length).toBe(10)
|
||||||
|
} finally {
|
||||||
|
hydrate.mockRestore()
|
||||||
|
}
|
||||||
|
})
|
||||||
|
})
|
||||||
Reference in a new issue