Compare commits

..

No commits in common. "905c267c47a9515abcde4a713b3c575ec730e7b2" and "67ae0046de4063cf74bc9d57eb68adccd575ed5e" have entirely different histories.

4 changed files with 118 additions and 971 deletions

View file

@ -7651,18 +7651,6 @@ export class Brainy<T = any> implements BrainyInterface<T> {
const searchMode = params.searchMode || 'auto'
const limit = params.limit || 10
// HYDRATE LAST (the hybrid path): its legs and its fusion rank IDS, and
// canonical is read at the two page exits below — never for a row the
// metadata filter is about to discard. This closure re-applies a hybrid
// row's match visibility once its entity is in hand; it is set only by
// the hybrid branch, so every other path hydrates unchanged.
let finishHybridRow: ((row: Result<T>, pending: Result<T>) => void) | undefined
// Set once the metadata block below has already ranked and CUT the page.
// The tail must not cut it a second time: `offset` has been consumed, and
// re-slicing a `limit`-long page by `offset` returns nothing at all.
let pagedEarly = false
// Handle text-only query (user explicitly wants text search)
if (searchMode === 'text' && params.query && params.query.trim() !== '') {
results = await this.executeTextSearch(params.query, limit * 2)
@ -7673,32 +7661,20 @@ export class Brainy<T = any> implements BrainyInterface<T> {
}
// Handle explicit hybrid or auto mode with query
else if ((searchMode === 'auto' || searchMode === 'hybrid') && params.query && params.query.trim() !== '' && !params.vector) {
// Zero-config hybrid: combine text + semantic search with RRF fusion.
// BOTH legs are held to the metadata filter's universe: the vector leg
// walks it as its candidate set, and the text leg ranks inside it
// instead of ranking the whole store and discarding what the filter
// would drop. Neither leg reads canonical — the page does, once.
const [textScored, semanticScored] = await Promise.all([
this.executeTextSearchScored(params.query, limit * 2, preResolvedMetadataIds ?? undefined),
this.executeVectorSearchScored(params, preResolvedMetadataIds ?? undefined, preResolvedAllowedIds)
// Zero-config hybrid: combine text + semantic search with RRF fusion
const [textResults, semanticResults] = await Promise.all([
this.executeTextSearch(params.query, limit * 2),
this.executeVectorSearch(params, preResolvedMetadataIds ?? undefined, preResolvedAllowedIds)
])
// Use user-specified alpha or auto-detect based on query length
const alpha = params.hybridAlpha ?? this.autoAlpha(params.query)
// Tokenize query for match visibility. The word list needs the entity,
// so it is computed on the page, at hydration.
// Tokenize query for match visibility
const queryWords = this.metadataIndex.tokenize(params.query)
const textResultIds = new Set(textScored.map((r) => r.id))
finishHybridRow = (row, pending) => {
row.textMatches = this.findMatchingWords(row.entity, queryWords, textResultIds)
row.textScore = pending.textScore
row.semanticScore = pending.semanticScore
row.matchSource = pending.matchSource
}
// RRF fusion combines both ranked id sets with match visibility
results = this.rrfFusion(textScored, semanticScored, alpha)
// RRF fusion combines both result sets with match visibility
results = await this.rrfFusion(textResults, semanticResults, alpha, queryWords)
}
// Handle direct vector search (no query text) - no hybrid needed
else if (params.vector && !params.query) {
@ -7759,13 +7735,20 @@ export class Brainy<T = any> implements BrainyInterface<T> {
const k = offset + limit
const order = rankIndicesByScore(results.map(r => r.score), k, true)
results = reorderByIndices(results, order).slice(offset, k)
pagedEarly = true
// Batch-load entities only for the paginated results (10x faster on GCS).
// This is the hydrate-last seam for the deferring paths: a row that
// arrives as a ranked shell is rebuilt in full here — flattened
// fields, entity and match visibility — never `entity` alone.
results = await this.hydrateResultPage(results, finishHybridRow)
// Batch-load entities only for the paginated results (10x faster on GCS)
const idsToLoad = results.filter(r => !r.entity).map(r => r.id)
if (idsToLoad.length > 0) {
const entitiesMap = await this.batchGet(idsToLoad)
for (const result of results) {
if (!result.entity) {
const entity = entitiesMap.get(result.id)
if (entity) {
result.entity = entity
}
}
}
}
// Early return if no other processing needed
if (!params.connected && !params.fusion) {
@ -7871,20 +7854,8 @@ export class Brainy<T = any> implements BrainyInterface<T> {
const finalOffset = params.offset || 0
// Efficient pagination - only slice what we need (limit already defined
// above), THEN read canonical for the page. Rows that arrived hydrated
// pass straight through; a deferred path reads exactly these rows.
//
// A page the metadata block already cut is NOT cut again: it holds the
// rows at [offset, offset+limit) of the ranking, so slicing it by
// `offset` a second time drops the whole page. That is how
// `find({ query, connected, where, offset })` — the shapes that reach
// here after early paging, `connected` and `fusion` — answered [] for
// every page but the first.
return await this.hydrateResultPage(
pagedEarly ? results : results.slice(finalOffset, finalOffset + limit),
finishHybridRow
)
// Efficient pagination - only slice what we need (limit already defined above)
return results.slice(finalOffset, finalOffset + limit)
})()
// Index-integrity guard — applied ONCE here so every find() path (metadata,
@ -12978,31 +12949,6 @@ export class Brainy<T = any> implements BrainyInterface<T> {
}
}
/**
* The text-leg twin of {@link filterIdsWithinBelted}: rank `query` INSIDE the
* candidate universe, through the provider's own posting-list merge so the
* answer can never drift from `getIdsForTextQuery`'s. A provider without the
* door is served by its whole-store answer intersected here the same rows
* in the same order, but it pays the whole-store marshal.
*
* @param query - The text query.
* @param ids - The candidate universe (the metadata filter's ids).
* @returns `{ id, matchCount }` rows inside `ids`, ranked by match count.
*/
private async textIdsWithinBelted(
query: string,
ids: readonly string[]
): Promise<Array<{ id: string; matchCount: number }>> {
this.ensureIndexesLoaded(['metadata'])
const mip = this.metadataIndex as unknown as MetadataIndexProvider
if (typeof mip.getIdsForTextQueryWithin === 'function') {
return await mip.getIdsForTextQueryWithin(query, ids)
}
const within = new Set(ids)
const all = await this.metadataIndex.getIdsForTextQuery(query)
return all.filter((m) => within.has(m.id))
}
async getIndexStatus(): Promise<{
initialized: boolean
/** `true` once open()'s index-build-if-needed step has run. Named for API
@ -15895,44 +15841,6 @@ export class Brainy<T = any> implements BrainyInterface<T> {
candidateIds?: string[],
allowedIds?: OpaqueIdSet
): Promise<Result<T>[]> {
const scored = await this.executeVectorSearchScored(params, candidateIds, allowedIds)
// Batch-load entities for 10-50x faster cloud storage performance
// GCS: 10 results = 1×50ms vs 10×50ms = 500ms (10x faster)
const entitiesMap = await this.batchGet(scored.map((s) => s.id))
const results: Result<T>[] = []
for (const { id, score } of scored) {
const entity = entitiesMap.get(id)
if (entity) {
results.push(this.createResult(id, score, entity))
}
}
return results
}
/**
* The semantic leg WITHOUT hydration ranked ids and their scores.
*
* The beam walk is already restricted to the candidate universe (that is what
* `candidateIds` / `allowedIds` are for), so the leg's cost is the walk. Its
* ROWS, though, are candidates for a fusion that will keep one page of them
* so the hybrid path takes them unhydrated and reads exactly the page it
* returns. {@link executeVectorSearch} is the eager form, for the search modes
* whose leg output IS the answer.
*
* @param params - Find parameters (supplies the query/vector and the limit).
* @param candidateIds - Optional pre-resolved metadata universe (see
* {@link executeVectorSearch}).
* @param allowedIds - Optional opaque predicate-pushdown universe.
* @returns Ranked `{ id, score }` rows no entity reads.
*/
private async executeVectorSearchScored(
params: FindParams<T>,
candidateIds?: string[],
allowedIds?: OpaqueIdSet
): Promise<Array<{ id: string; score: number }>> {
// Vector cold-read guard: before trusting a semantic/vector result, verify the
// vector index actually SERVES a known persisted vector (one-shot per brain).
// A pure semantic find({ query }) has no filter, so verifyMetadataLive never
@ -15958,10 +15866,21 @@ export class Brainy<T = any> implements BrainyInterface<T> {
// HNSW search with optional metadata-first candidate filtering
const searchResults: [string, number][] = await this.index.search(vector, limit * 2, undefined, searchOptions)
return searchResults.map(([id, distance]) => ({
id,
score: Math.max(0, Math.min(1, 1 / (1 + distance)))
}))
// Batch-load entities for 10-50x faster cloud storage performance
// GCS: 10 results = 1×50ms vs 10×50ms = 500ms (10x faster)
const ids = searchResults.map(([id]) => id)
const entitiesMap = await this.batchGet(ids)
const results: Result<T>[] = []
for (const [id, distance] of searchResults) {
const entity = entitiesMap.get(id)
if (entity) {
const score = Math.max(0, Math.min(1, 1 / (1 + distance)))
results.push(this.createResult(id, score, entity))
}
}
return results
}
/**
@ -16261,64 +16180,30 @@ export class Brainy<T = any> implements BrainyInterface<T> {
* @returns Array of Results with scores based on match count
*/
private async executeTextSearch(query: string, limit: number): Promise<Result<T>[]> {
const scored = await this.executeTextSearchScored(query, limit)
if (scored.length === 0) return []
const textMatches = await this.metadataIndex.getIdsForTextQuery(query)
if (textMatches.length === 0) return []
// Batch-load entities for the whole leg — this is the eager form, kept for
// the text-only search mode whose results ARE the answer.
const entitiesMap = await this.batchGet(scored.map((s) => s.id))
// Take top matches and load entities
const topMatches = textMatches.slice(0, limit * 2) // Get more for filtering
const ids = topMatches.map(m => m.id)
const entitiesMap = await this.batchGet(ids)
// Create results with scores based on match count
const maxMatches = topMatches[0]?.matchCount || 1
const results: Result<T>[] = []
for (const { id, score } of scored) {
const entity = entitiesMap.get(id)
for (const match of topMatches) {
const entity = entitiesMap.get(match.id)
if (entity) {
results.push(this.createResult(id, score, entity))
// Normalize score to 0-1 range based on match count
const score = match.matchCount / maxMatches
results.push(this.createResult(match.id, score, entity))
}
}
return results
}
/**
* The text leg WITHOUT hydration ranked ids and their scores.
*
* FILTER BEFORE HYDRATE: when the caller already knows the candidate
* universe (the metadata filter's ids in a hybrid `find({ query, where })`),
* it is passed here and the word index ranks INSIDE that universe. The
* earlier order ranked the whole store, took the top `limit * 2`, hydrated
* every one of them, and only then intersected with the filter so a
* filtered hybrid find on a large store hydrated hundreds of rows to return
* a handful, and a matching row outside the store-wide text prefix was
* silently dropped (the same defect `find({ connected })` had before the
* graph-first law).
*
* The score is the match count normalized against the top row's, so a
* restricted call normalizes against the top row IN THE UNIVERSE the same
* rule applied to the set actually being ranked.
*
* @param query - Text query to search for.
* @param limit - Result budget; the leg keeps `limit * 2` for the fusion.
* @param candidateIds - Optional candidate universe to rank inside.
* @returns Ranked `{ id, score }` rows no entity reads.
*/
private async executeTextSearchScored(
query: string,
limit: number,
candidateIds?: readonly string[]
): Promise<Array<{ id: string; score: number }>> {
const textMatches = candidateIds
? await this.textIdsWithinBelted(query, candidateIds)
: await this.metadataIndex.getIdsForTextQuery(query)
if (textMatches.length === 0) return []
// Take top matches (more than the page, for the fusion to rank)
const topMatches = textMatches.slice(0, limit * 2)
// Normalize score to 0-1 range based on match count
const maxMatches = topMatches[0]?.matchCount || 1
return topMatches.map((m) => ({ id: m.id, score: m.matchCount / maxMatches }))
}
/**
* Auto-detect optimal alpha for hybrid search
*
@ -16345,56 +16230,55 @@ export class Brainy<T = any> implements BrainyInterface<T> {
*
* Formula: score(d) = sum(1 / (k + rank(d))) for each list
*
* Now includes match visibility (textScore, semanticScore, matchSource; the
* `textMatches` word list needs the entity and is filled at hydration).
* Now includes match visibility (textMatches, textScore, semanticScore, matchSource)
*
* HYDRATE LAST: both legs arrive as ranked ids + scores, and the fusion ranks
* ids no entity is read here. The rows it returns are ranked SHELLS; the
* page is cut from them and only that page is read from canonical (see
* {@link hydrateResultPage}). The earlier order hydrated both legs in full
* hundreds of rows to return one page of them.
*
* @param textResults - Ranked ids + scores from text search
* @param semanticResults - Ranked ids + scores from semantic search
* @param textResults - Results from text search
* @param semanticResults - Results from semantic search
* @param alpha - Weight for semantic (0=text only, 1=semantic only)
* @param queryWords - Original query words for match tracking
* @param k - RRF constant (default: 60, standard in literature)
* @returns Fused result shells sorted by combined score with match visibility
* @returns Fused results sorted by combined score with match visibility
*/
private rrfFusion(
textResults: ReadonlyArray<{ id: string; score: number }>,
semanticResults: ReadonlyArray<{ id: string; score: number }>,
private async rrfFusion(
textResults: Result<T>[],
semanticResults: Result<T>[],
alpha: number,
queryWords: string[],
k: number = 60
): Result<T>[] {
): Promise<Result<T>[]> {
// Track scores and match details per entity
interface MatchData {
rrf: number
textScore?: number
semanticScore?: number
textMatches: string[]
hasText: boolean
hasSemantic: boolean
}
const matchData = new Map<string, MatchData>()
const entityMap = new Map<string, Entity<T>>()
// Text contribution (1 - alpha weight)
const textWeight = 1 - alpha
textResults.forEach((r, rank) => {
const rrfScore = textWeight * (1 / (k + rank + 1))
const existing = matchData.get(r.id) || { rrf: 0, hasText: false, hasSemantic: false }
const existing = matchData.get(r.id) || { rrf: 0, textMatches: [], hasText: false, hasSemantic: false }
existing.rrf += rrfScore
existing.textScore = r.score // Original text search score (0-1)
existing.hasText = true
matchData.set(r.id, existing)
if (r.entity) entityMap.set(r.id, r.entity)
})
// Semantic contribution (alpha weight)
semanticResults.forEach((r, rank) => {
const rrfScore = alpha * (1 / (k + rank + 1))
const existing = matchData.get(r.id) || { rrf: 0, hasText: false, hasSemantic: false }
const existing = matchData.get(r.id) || { rrf: 0, textMatches: [], hasText: false, hasSemantic: false }
existing.rrf += rrfScore
existing.semanticScore = r.score // Original semantic search score (0-1)
existing.hasSemantic = true
matchData.set(r.id, existing)
if (r.entity) entityMap.set(r.id, r.entity)
})
// Sort by fused score
@ -16402,9 +16286,27 @@ export class Brainy<T = any> implements BrainyInterface<T> {
.sort((a, b) => b[1].rrf - a[1].rrf)
.map(([id, data]) => ({ id, data }))
// Create ranked shells with match visibility
// Build results - need to load any missing entities
const missingIds = sortedIds.filter(s => !entityMap.has(s.id)).map(s => s.id)
if (missingIds.length > 0) {
const loaded = await this.batchGet(missingIds)
for (const [id, entity] of loaded) {
entityMap.set(id, entity)
}
}
// Performance: Build set of text result IDs for O(1) lookup
// This avoids re-extracting text for entities that weren't in text results
const textResultIds = new Set(textResults.map(r => r.id))
// Create final results with match visibility
const results: Result<T>[] = []
for (const { id, data } of sortedIds) {
const entity = entityMap.get(id)
if (entity) {
// Find which query words matched - uses fast path if entity wasn't in text results
const textMatches = this.findMatchingWords(entity, queryWords, textResultIds)
// Determine match source
let matchSource: 'text' | 'semantic' | 'both'
if (data.hasText && data.hasSemantic) {
@ -16415,80 +16317,20 @@ export class Brainy<T = any> implements BrainyInterface<T> {
matchSource = 'semantic'
}
const result = this.pendingResult(id, data.rrf)
// Create result with match visibility
const result = this.createResult(id, data.rrf, entity)
result.textMatches = textMatches
result.textScore = data.textScore
result.semanticScore = data.semanticScore
result.matchSource = matchSource
results.push(result)
}
}
return results
}
/**
* A ranked candidate whose entity has NOT been read yet.
*
* The shell carries everything the ranking tail needs the id, the score,
* and the match-visibility fields and nothing that requires canonical. It
* is typed `Result` so it flows through the shared dedupe / visibility /
* filter / rank / page tail unchanged; {@link hydrateResultPage} turns the
* survivors into real results before any caller sees them, and find()'s
* index-integrity guard drops any row that never gained an entity.
*
* @param id - The candidate's canonical id.
* @param score - Its rank score.
*/
private pendingResult(id: string, score: number): Result<T> {
return { id, score } as Result<T>
}
/**
* Read canonical for exactly the rows that need it the hydrate-last seam.
*
* Rows that already carry an entity (the eager legs: metadata, text-only,
* semantic-only, proximity, graph) pass through untouched, so this is a no-op
* for every path that has not deferred. Rows that are shells are read in ONE
* batch and rebuilt through {@link createResult}, so a hydrated row is
* indistinguishable from an eagerly-built one same flattened fields, same
* `entity`, same key order with `finish` re-applying the fields only the
* deferring path knows about (a hybrid row's match visibility).
*
* A shell whose id has no canonical row is dropped, exactly as the eager legs
* dropped it; find()'s index-integrity guard makes the same judgement on the
* page it returns.
*
* @param rows - The page's rows, ranked and paged already.
* @param finish - Applied to each rebuilt row, with its shell, after the
* flattened fields are set.
* @returns The page with every surviving row hydrated.
*/
private async hydrateResultPage(
rows: Result<T>[],
finish?: (row: Result<T>, pending: Result<T>) => void
): Promise<Result<T>[]> {
const pendingIds: string[] = []
for (const row of rows) {
if (!row.entity) pendingIds.push(row.id)
}
if (pendingIds.length === 0) return rows
const entitiesMap = await this.batchGet(pendingIds)
const hydrated: Result<T>[] = []
for (const row of rows) {
if (row.entity) {
hydrated.push(row)
continue
}
const entity = entitiesMap.get(row.id)
if (!entity) continue
const filled = this.createResult(row.id, row.score, entity, row.explanation)
finish?.(filled, row)
hydrated.push(filled)
}
return hydrated
}
/**
* Find which query words match in an entity's text content
*

View file

@ -473,28 +473,6 @@ export interface MetadataIndexProvider {
graphIndex: unknown
): Promise<{ ids: string[]; emptyAt: 'graph' | 'filter' | 'visibility' | 'none' } | null>
getIdsForTextQuery(query: string): Promise<Array<{ id: string; matchCount: number }>>
/**
* @description OPTIONAL: score `query` over `ids` ONLY the text-leg twin of
* {@link filterIdsWithin}, and the door a hybrid `find({ query, where })`
* walks. The metadata filter's universe is the candidate set there, so the
* text leg must cost O(|ids|) membership checks and marshal at most `|ids|`
* rows, never the whole posting list of every query word. A native index
* intersects its own postings with the candidate set (membership by entity
* int) before any string crosses the boundary; the reference index answers
* from its own `getIdsForTextQuery`, so the two doors can never disagree.
* Absent Brainy intersects `getIdsForTextQuery`'s answer with `ids` itself
* (correct, and still hydrate-last, but it marshals the whole answer).
*
* The answer keeps `getIdsForTextQuery`'s contract: `{ id, matchCount }`
* sorted by `matchCount` descending, ties in the order the whole-store answer
* would have produced. Only rows in `ids` may appear.
* @param query - The same text query accepted by `getIdsForTextQuery`.
* @param ids - The candidate ids (canonical). The answer is a subset.
*/
getIdsForTextQueryWithin?(
query: string,
ids: readonly string[]
): Promise<Array<{ id: string; matchCount: number }>>
getSortedIdsForFilter(filter: any, orderBy: string, order?: 'asc' | 'desc', topK?: number): Promise<string[]>
getFilterValues(field: string): Promise<string[]>
getFilterFields(): Promise<string[]>

View file

@ -1509,56 +1509,11 @@ export class MetadataIndexManager implements MetadataIndexProvider {
* @returns Array of { id, matchCount } sorted by matchCount descending
*/
async getIdsForTextQuery(query: string): Promise<Array<{ id: string; matchCount: number }>> {
return this.scoreTextQuery(query)
}
/**
* Score a text query over `ids` ONLY the reference implementation of the
* optional `getIdsForTextQueryWithin` door (see
* {@link import('../plugin.js').MetadataIndexProvider}). The hybrid
* `find({ query, where })` path passes the metadata filter's universe here so
* the text leg ranks INSIDE that universe instead of ranking the whole store
* and discarding the rows the filter would have dropped.
*
* It answers from the same posting-list merge as {@link getIdsForTextQuery},
* with the candidate membership applied as each word's postings are counted,
* so the two doors can never disagree: the answer is exactly the whole-store
* answer restricted to `ids`, in the same order.
*
* @param query - Text query to search for.
* @param ids - Candidate entity ids; only these may appear in the answer.
* @returns Array of { id, matchCount } sorted by matchCount descending.
*/
async getIdsForTextQueryWithin(
query: string,
ids: readonly string[]
): Promise<Array<{ id: string; matchCount: number }>> {
if (ids.length === 0) return []
return this.scoreTextQuery(query, new Set(ids))
}
/**
* The one posting-list merge behind both text doors.
*
* Each query word contributes AT MOST one match per entity (a posting list
* can name an id more than once), and entities are ranked by how many of the
* query's words they matched. `within`, when given, restricts the count to
* those candidates applied during the merge, so a restricted call never
* materializes a whole-store match map.
*
* @param query - Text query to search for.
* @param within - Optional candidate universe; absent = the whole store.
* @returns Array of { id, matchCount } sorted by matchCount descending.
*/
private async scoreTextQuery(
query: string,
within?: ReadonlySet<string>
): Promise<Array<{ id: string; matchCount: number }>> {
const queryWords = this.tokenize(query)
if (queryWords.length === 0) return []
// Count matches per entity, one word's postings at a time.
const matchCounts = new Map<string, number>()
// Get IDs for each word hash
const wordIdSets: Map<string, number>[] = []
for (const word of queryWords) {
const wordHash = this.hashWord(word)
let ids: string[]
@ -1574,12 +1529,19 @@ export class MetadataIndexManager implements MetadataIndexProvider {
throw err
}
}
// One count per (word, entity) — dedupe this word's postings first.
const counted = new Set<string>()
const idSet = new Map<string, number>()
for (const id of ids) {
if (counted.has(id)) continue
counted.add(id)
if (within && !within.has(id)) continue
idSet.set(id, 1)
}
wordIdSets.push(idSet)
}
if (wordIdSets.length === 0) return []
// Count matches per entity
const matchCounts = new Map<string, number>()
for (const idSet of wordIdSets) {
for (const [id] of idSet) {
matchCounts.set(id, (matchCounts.get(id) || 0) + 1)
}
}

View file

@ -1,635 +0,0 @@
/**
* @module tests/integration/find-hybrid-filter-before-hydrate
* @description FILTER BEFORE HYDRATE, applied to the hybrid `find({ query })` path.
*
* A hybrid find fuses two legs. The semantic leg already walked only the
* metadata filter's universe (`candidateIds` / `allowedIds`). The TEXT leg did
* not: it ranked the WHOLE store, took the top `limit * 4`, read every one of
* those rows from canonical, and only then intersected with the filter so a
* filtered hybrid find on a large store read hundreds of rows to return a
* handful of them, and a matching row outside the store-wide text prefix was
* silently dropped. That is the same defect `find({ connected })` carried
* before the graph-first law, one leg over.
*
* Both halves are pinned here.
*
* THE ANSWER. Where the filter did not truncate the text leg the universe
* covers every text match, so both orders rank the same rows the new
* pipeline's answer is IDENTICAL to the old one's: same rows, same order, same
* scores, same match visibility, same row shape. The oracle below is the
* pre-change pipeline itself, replayed on the same brain through the same
* doors, so the comparison is against what actually ran, not a remembered
* expectation.
*
* THE CORRECTION. Where the filter DID truncate it the query's words are
* common outside the universe the old order let the text leg contribute
* nothing at all: every row it ranked was discarded by the filter, and the
* answer came from the semantic leg alone. The new order ranks inside the
* universe, so the text leg contributes the rows it always should have.
*
* THE COST. Canonical is read for exactly the page: one batch, `limit` rows,
* never the legs. And the text leg is asked about the universe's ids only
* what it marshals is bounded by the universe, not by the store.
*/
import { describe, it, expect, beforeAll, vi } from 'vitest'
import { Brainy } from '../../src/brainy'
import { NounType, VerbType } from '../../src/types/graphTypes'
import { rankIndicesByScore, reorderByIndices } from '../../src/utils/resultRanking'
import { resolveEntityId } from '../../src/utils/idNormalization'
/** Embedding width of the default model — the row vectors must match it. */
const DIM = 384
/**
* A deterministic, per-row-distinct unit vector. Distinct so the semantic leg
* has a real ranking to produce (identical vectors would make its order a tie
* break), deterministic so the oracle and the pipeline see the same one.
*/
function seededVector(seed: number): number[] {
const v = new Array<number>(DIM)
for (let i = 0; i < DIM; i++) {
v[i] = Math.sin((i + 1) * 0.11 + seed * 0.37) * 0.5 + Math.cos((i + 1) * 0.05 + seed * 0.13) * 0.3
}
const magnitude = Math.sqrt(v.reduce((sum, x) => sum + x * x, 0))
return v.map((x) => x / magnitude)
}
/** The fields a caller reads off a hybrid row — the whole comparable surface. */
function project(rows: any[]): any[] {
return rows.map((r) => ({
id: r.id,
score: r.score,
type: r.type,
metadata: r.metadata,
textMatches: r.textMatches,
textScore: r.textScore,
semanticScore: r.semanticScore,
matchSource: r.matchSource
}))
}
/**
* The PRE-CHANGE hybrid pipeline, replayed on a live brain through the same
* provider doors it used: whole-store text ranking with both legs hydrated in
* full, RRF fusion, then the metadata intersection, then the page.
*
* Supports the shapes these pins exercise (query + where/type/excludeVFS +
* connected + offset); `orderBy`, `fusion` and `near` are not replayed.
*/
async function legacyHybridFind(brain: any, params: any): Promise<any[]> {
const index = brain.metadataIndex
const limit = params.limit ?? 10
const offset = params.offset ?? 0
const hasFilter = Boolean(
params.where || params.type || params.subtype || params.service || params.excludeVFS
)
let preResolvedMetadataIds: string[] | null = null
let preResolvedFilter: any = null
let graphFirstIds: string[] | null = null
if (params.connected) {
// find() normalizes the anchors to canonical ids before this stage runs.
const anchored = {
...params,
connected: {
...params.connected,
...(params.connected.from && { from: resolveEntityId(params.connected.from) }),
...(params.connected.to && { to: resolveEntityId(params.connected.to) })
}
}
graphFirstIds = await brain.resolveConnectedIds(anchored)
if (graphFirstIds!.length > 0 && hasFilter) {
preResolvedFilter = brain.buildMetadataFilter(params)
graphFirstIds = await brain.filterIdsWithinBelted(preResolvedFilter, graphFirstIds)
}
if (graphFirstIds!.length === 0) return []
preResolvedMetadataIds = graphFirstIds
} else if (hasFilter) {
preResolvedFilter = brain.buildMetadataFilter(params)
preResolvedMetadataIds = await brain.filterIdsBelted(preResolvedFilter)
if (preResolvedMetadataIds!.length === 0) return []
}
// Text leg — the whole store, then the top `limit * 4`, hydrated in full.
const allTextMatches = await index.getIdsForTextQuery(params.query)
const topMatches = allTextMatches.slice(0, limit * 2 * 2)
const maxMatches = topMatches[0]?.matchCount || 1
const textEntities = await brain.batchGet(topMatches.map((m: any) => m.id))
const textResults = topMatches
.filter((m: any) => textEntities.has(m.id))
.map((m: any) => ({ id: m.id, score: m.matchCount / maxMatches }))
// Semantic leg — the beam walk over the universe, hydrated in full.
const vector = await brain.embed(params.query)
const searchOptions = preResolvedMetadataIds ? { candidateIds: preResolvedMetadataIds } : undefined
const searchResults: [string, number][] = await brain.index.search(
vector,
limit * 2,
undefined,
searchOptions
)
const semanticEntities = await brain.batchGet(searchResults.map(([id]) => id))
const semanticResults = searchResults
.filter(([id]) => semanticEntities.has(id))
.map(([id, distance]) => ({ id, score: Math.max(0, Math.min(1, 1 / (1 + distance))) }))
// RRF fusion, with the match visibility the rows carried.
const alpha = params.hybridAlpha ?? brain.autoAlpha(params.query)
const k = 60
const matchData = new Map<string, any>()
const textWeight = 1 - alpha
textResults.forEach((r: any, rank: number) => {
const existing = matchData.get(r.id) || { rrf: 0, hasText: false, hasSemantic: false }
existing.rrf += textWeight * (1 / (k + rank + 1))
existing.textScore = r.score
existing.hasText = true
matchData.set(r.id, existing)
})
semanticResults.forEach((r: any, rank: number) => {
const existing = matchData.get(r.id) || { rrf: 0, hasText: false, hasSemantic: false }
existing.rrf += alpha * (1 / (k + rank + 1))
existing.semanticScore = r.score
existing.hasSemantic = true
matchData.set(r.id, existing)
})
const queryWords: string[] = index.tokenize(params.query)
const textResultIds = new Set(textResults.map((r: any) => r.id))
const fusedIds = Array.from(matchData.entries())
.sort((a, b) => b[1].rrf - a[1].rrf)
.map(([id, data]) => ({ id, data }))
const allEntities = await brain.batchGet(fusedIds.map((f) => f.id))
let rows: any[] = []
for (const { id, data } of fusedIds) {
const entity = allEntities.get(id)
if (!entity) continue
const textContent = textResultIds.has(id)
? index.extractTextContent({ data: entity.data, metadata: entity.metadata }).toLowerCase()
: null
rows.push({
id,
score: data.rrf,
type: entity.type,
metadata: entity.metadata,
textMatches:
textContent === null ? [] : queryWords.filter((w) => textContent.includes(w.toLowerCase())),
textScore: data.textScore,
semanticScore: data.semanticScore,
matchSource: data.hasText && data.hasSemantic ? 'both' : data.hasText ? 'text' : 'semantic'
})
}
// The metadata intersection — after the legs, as it was.
if (preResolvedMetadataIds && preResolvedFilter) {
const filteredIdSet = new Set(preResolvedMetadataIds)
rows = rows.filter((r) => filteredIdSet.has(r.id))
}
if (graphFirstIds !== null) {
const neighbourSet = new Set(graphFirstIds)
rows = rows.filter((r) => neighbourSet.has(r.id))
}
// Rank to the page, then cut it.
const order = rankIndicesByScore(
rows.map((r) => r.score),
offset + limit,
true
)
return reorderByIndices(rows, order).slice(offset, offset + limit)
}
/**
* FIXTURE A the filter's universe covers every text match, so the two orders
* rank exactly the same rows and the answers must be identical.
*/
describe('hybrid find: filter before hydrate — the answer is unchanged', () => {
let brain: Brainy<any>
const QUERY = 'orbital telemetry'
const MATCHES = 24
const FILLER = 120
const OUTSIDE = 30
const VFS = 10
const RETRACTED = 6
const anchor = 'array-anchor'
const matchIds: string[] = []
beforeAll(async () => {
brain = new Brainy({ requireSubtype: false, storage: { type: 'memory' } })
await brain.init()
let seed = 1
await brain.add({
id: anchor,
data: 'ground station anchor record',
type: NounType.Thing,
metadata: { lane: 'alpha', role: 'anchor' },
vector: seededVector(seed++)
})
// Rows the query's words actually match — all inside every filter below.
for (let i = 0; i < MATCHES; i++) {
const id = `match-${i}`
await brain.add({
id,
data: `orbital telemetry packet ${i} recorded downlink`,
type: NounType.Document,
metadata: { lane: 'alpha', rank: i },
vector: seededVector(seed++)
})
matchIds.push(resolveEntityId(id))
await brain.relate({ from: anchor, to: id, type: VerbType.RelatedTo })
}
// Rows inside the universe that the query's words do NOT match.
for (let i = 0; i < FILLER; i++) {
await brain.add({
id: `filler-${i}`,
data: `cistern ledger entry ${i} archived`,
type: NounType.Document,
metadata: { lane: 'alpha', rank: 1000 + i },
vector: seededVector(seed++)
})
}
// Rows outside the universe.
for (let i = 0; i < OUTSIDE; i++) {
await brain.add({
id: `outside-${i}`,
data: `unrelated dossier ${i}`,
type: NounType.Person,
metadata: { lane: 'beta' },
vector: seededVector(seed++)
})
}
// VFS infrastructure rows — excluded by excludeVFS.
for (let i = 0; i < VFS; i++) {
await brain.add({
id: `vfs-${i}`,
data: `mounted path ${i}`,
type: NounType.Document,
metadata: { lane: 'alpha', vfsType: 'file' },
vector: seededVector(seed++)
})
}
// Retracted rows — excluded by a `missing` negation.
for (let i = 0; i < RETRACTED; i++) {
await brain.add({
id: `retracted-${i}`,
data: `withdrawn note ${i}`,
type: NounType.Document,
metadata: { lane: 'alpha', retracted: true },
vector: seededVector(seed++)
})
}
// The reference index has no opaque-set door, so the pipeline and the
// oracle both restrict the beam walk with the materialized candidate ids.
expect(typeof (brain as any).metadataIndex.getIdSetForFilter).not.toBe('function')
})
it('the fixture does not truncate the text leg — the universe covers every text match', async () => {
const index = (brain as any).metadataIndex
const textMatches = await index.getIdsForTextQuery(QUERY)
expect(textMatches).toHaveLength(MATCHES)
const universe = await (brain as any).filterIdsBelted({ lane: 'alpha' })
const inUniverse = new Set(universe)
for (const m of textMatches) expect(inUniverse.has(m.id)).toBe(true)
})
it('hybrid + where: identical rows, identical order, identical scores', async () => {
const params = { query: QUERY, where: { lane: 'alpha' }, limit: 8 }
const expected = await legacyHybridFind(brain as any, params)
const actual = await brain.find(params as any)
expect(actual.length).toBe(expected.length)
expect(project(actual)).toEqual(expected)
})
it('hybrid + where + offset: identical page two', async () => {
const params = { query: QUERY, where: { lane: 'alpha' }, limit: 6, offset: 6 }
const expected = await legacyHybridFind(brain as any, params)
const actual = await brain.find(params as any)
expect(actual.length).toBe(expected.length)
expect(project(actual)).toEqual(expected)
})
it('hybrid + type list + excludeVFS + a `missing` negation: identical', async () => {
const params = {
query: QUERY,
type: [NounType.Document, NounType.Person],
excludeVFS: true,
where: { lane: 'alpha', retracted: { missing: true } },
limit: 8
}
const expected = await legacyHybridFind(brain as any, params)
const actual = await brain.find(params as any)
expect(actual.length).toBe(expected.length)
expect(project(actual)).toEqual(expected)
for (const r of actual) {
expect(r.metadata.retracted).toBeUndefined()
expect(r.metadata.vfsType).toBeUndefined()
}
})
it('hybrid + type list + excludeVFS + a `missing` negation, offset: identical', async () => {
const params = {
query: QUERY,
type: [NounType.Document, NounType.Person],
excludeVFS: true,
where: { lane: 'alpha', retracted: { missing: true } },
limit: 5,
offset: 5
}
const expected = await legacyHybridFind(brain as any, params)
const actual = await brain.find(params as any)
expect(actual.length).toBe(expected.length)
expect(project(actual)).toEqual(expected)
})
it('hybrid + connected: identical, and never a non-neighbour', async () => {
const params = {
query: QUERY,
connected: { from: anchor, direction: 'out' as const },
where: { lane: 'alpha' },
limit: 8
}
const expected = await legacyHybridFind(brain as any, params)
const actual = await brain.find(params as any)
expect(actual.length).toBe(expected.length)
expect(project(actual)).toEqual(expected)
const neighbours = new Set(matchIds)
for (const r of actual) expect(neighbours.has(r.id)).toBe(true)
})
it('hybrid + connected + offset: page two is the page, not an empty answer', async () => {
const params = {
query: QUERY,
connected: { from: anchor, direction: 'out' as const },
where: { lane: 'alpha' },
limit: 5,
offset: 5
}
const expected = await legacyHybridFind(brain as any, params)
expect(expected).toHaveLength(5)
const actual = await brain.find(params as any)
expect(actual.length).toBe(expected.length)
expect(project(actual)).toEqual(expected)
})
it('hybrid + connected: paging reaches every matching neighbour exactly once', async () => {
const seen = new Set<string>()
for (let offset = 0; offset < MATCHES; offset += 6) {
const page = await brain.find({
query: QUERY,
connected: { from: anchor, direction: 'out' as const },
where: { lane: 'alpha' },
limit: 6,
offset
} as any)
for (const r of page) {
expect(seen.has(r.id)).toBe(false)
seen.add(r.id)
}
}
// Every row the fused candidate set holds is reachable by paging, and the
// neighbour set is the ceiling.
expect(seen.size).toBeGreaterThanOrEqual(MATCHES)
const neighbours = new Set(matchIds)
for (const id of seen) expect(neighbours.has(id)).toBe(true)
})
it('hybrid + fusion + offset: page two is the page', async () => {
const plain = await brain.find({
query: QUERY,
where: { lane: 'alpha' },
limit: 5,
offset: 5
} as any)
const fused = await brain.find({
query: QUERY,
where: { lane: 'alpha' },
fusion: 'weighted',
limit: 5,
offset: 5
} as any)
expect(fused).toHaveLength(plain.length)
expect(fused.map((r) => r.id)).toEqual(plain.map((r) => r.id))
})
it('a hydrated hybrid row is shaped exactly as an eagerly-built one', async () => {
const rows = await brain.find({ query: QUERY, where: { lane: 'alpha' }, limit: 8 } as any)
const row = rows[0]
expect(Object.keys(row)).toEqual([
'id',
'score',
'type',
'subtype',
'visibility',
'metadata',
'data',
'confidence',
'weight',
'_rev',
'entity',
'textMatches',
'textScore',
'semanticScore',
'matchSource'
])
// The flattened fields are projections of the entity, as always.
expect(row.entity).toBeDefined()
expect(row.type).toBe(row.entity.type)
expect(row.metadata).toBe(row.entity.metadata)
expect(row.data).toBe(row.entity.data)
expect(row._rev).toBe(row.entity._rev)
// The match visibility survives the deferral — every leg's fields, on the
// rows that leg contributed, exactly as the eager pipeline set them.
expect(['text', 'semantic', 'both']).toContain(row.matchSource)
for (const r of rows) {
if (r.matchSource === 'semantic') {
expect(r.textMatches).toEqual([])
expect(r.textScore).toBeUndefined()
} else {
expect(r.textMatches).toEqual(['orbital', 'telemetry'])
expect(typeof r.textScore).toBe('number')
}
if (r.matchSource === 'text') {
expect(r.semanticScore).toBeUndefined()
} else {
expect(typeof r.semanticScore).toBe('number')
}
}
})
it('reads canonical for the page only — one batch, `limit` rows', async () => {
// Warm any first-read verification before the counters are read.
await brain.find({ query: QUERY, where: { lane: 'alpha' }, limit: 1 } as any)
const hydrate = vi.spyOn(brain as any, 'batchGet')
try {
const results = await brain.find({ query: QUERY, where: { lane: 'alpha' }, limit: 10 } as any)
expect(results).toHaveLength(10)
expect(hydrate).toHaveBeenCalledTimes(1)
expect((hydrate.mock.calls[0][0] as string[]).length).toBe(10)
} finally {
hydrate.mockRestore()
}
})
it('asks the text index about the universe only, never the whole store', async () => {
const index = (brain as any).metadataIndex
const wholeStore = vi.spyOn(index, 'getIdsForTextQuery')
const within = vi.spyOn(index, 'getIdsForTextQueryWithin')
try {
await brain.find({ query: QUERY, where: { lane: 'alpha' }, limit: 10 } as any)
expect(wholeStore).not.toHaveBeenCalled()
expect(within).toHaveBeenCalledTimes(1)
const askedIds = within.mock.calls[0][1] as string[]
const universe = await (brain as any).filterIdsBelted({ lane: 'alpha' })
expect(askedIds).toHaveLength(universe.length)
// What the text leg marshals is bounded by the universe, not the store.
const marshalled = (await within.mock.results[0].value) as unknown[]
expect(marshalled.length).toBeLessThanOrEqual(universe.length)
expect(marshalled).toHaveLength(MATCHES)
} finally {
wholeStore.mockRestore()
within.mockRestore()
}
})
it('the two text doors agree: within is the whole-store answer restricted', async () => {
const index = (brain as any).metadataIndex
const universe: string[] = await (brain as any).filterIdsBelted({
lane: 'alpha',
retracted: { missing: true }
})
const inUniverse = new Set(universe)
const whole = await index.getIdsForTextQuery(QUERY)
const within = await index.getIdsForTextQueryWithin(QUERY, universe)
expect(within).toEqual(whole.filter((m: any) => inUniverse.has(m.id)))
expect(await index.getIdsForTextQueryWithin(QUERY, [])).toEqual([])
})
})
/**
* FIXTURE B the query's words are common OUTSIDE the universe, so the old
* order's text leg was entirely consumed by rows the filter then discarded.
* This is the corrected behaviour, held by name.
*/
describe('hybrid find: the text leg ranks inside the filter, not around it', () => {
let brain: Brainy<any>
const QUERY = 'orbital telemetry drift'
const NOISE = 150
const KEEP = 15
const keepIds: string[] = []
beforeAll(async () => {
brain = new Brainy({ requireSubtype: false, storage: { type: 'memory' } })
await brain.init()
let seed = 5000
// Added FIRST and matching one more query word, so they lead the
// store-wide text ranking outright — and none of them pass the filter.
for (let i = 0; i < NOISE; i++) {
await brain.add({
id: `noise-${i}`,
data: `orbital telemetry drift report ${i}`,
type: NounType.Document,
metadata: { lane: 'beta' },
vector: seededVector(seed++)
})
}
for (let i = 0; i < KEEP; i++) {
const id = `keep-${i}`
await brain.add({
id,
data: `orbital telemetry summary ${i}`,
type: NounType.Document,
metadata: { lane: 'alpha' },
vector: seededVector(seed++)
})
keepIds.push(resolveEntityId(id))
}
})
it('the old order let the filter consume the whole text leg', async () => {
const index = (brain as any).metadataIndex
const universe: string[] = await (brain as any).filterIdsBelted({ lane: 'alpha' })
expect(universe).toHaveLength(KEEP)
const inUniverse = new Set(universe)
// The store-wide prefix the old text leg took (limit 10 → limit * 4).
const prefix = (await index.getIdsForTextQuery(QUERY)).slice(0, 40)
expect(prefix).toHaveLength(40)
expect(prefix.filter((m: any) => inUniverse.has(m.id))).toHaveLength(0)
// Every row the old text leg ranked was then discarded by the filter, so
// the old answer carried NO text contribution at all — fifteen rows that
// match the query's words exactly, and not one of them reached the page
// through the text leg. What the old order returned was whatever the
// semantic leg alone happened to reach.
const legacy = await legacyHybridFind(brain as any, {
query: QUERY,
where: { lane: 'alpha' },
limit: 10
})
for (const r of legacy) {
expect(r.matchSource).toBe('semantic')
expect(r.textScore).toBeUndefined()
expect(r.textMatches).toEqual([])
}
})
it('the new order ranks the text leg inside the universe', async () => {
const results = await brain.find({
query: QUERY,
where: { lane: 'alpha' },
limit: 10
} as any)
expect(results).toHaveLength(10)
const keeps = new Set(keepIds)
for (const r of results) {
expect(keeps.has(r.id)).toBe(true)
expect(r.metadata.lane).toBe('alpha')
// The text leg is the contributor the old order threw away.
expect(['text', 'both']).toContain(r.matchSource)
expect(r.textScore).toBe(1)
expect(r.textMatches).toEqual(['orbital', 'telemetry'])
}
})
it('paging reaches every matching row the old order could not see', async () => {
const seen = new Set<string>()
for (let offset = 0; offset < KEEP; offset += 5) {
const page = await brain.find({
query: QUERY,
where: { lane: 'alpha' },
limit: 5,
offset
} as any)
expect(page).toHaveLength(5)
for (const r of page) {
expect(seen.has(r.id)).toBe(false)
seen.add(r.id)
}
}
expect(seen.size).toBe(KEEP)
expect([...seen].sort()).toEqual([...keepIds].sort())
})
it('reads canonical for the page only, on the truncating shape too', async () => {
await brain.find({ query: QUERY, where: { lane: 'alpha' }, limit: 1 } as any)
const hydrate = vi.spyOn(brain as any, 'batchGet')
try {
const results = await brain.find({ query: QUERY, where: { lane: 'alpha' }, limit: 10 } as any)
expect(results).toHaveLength(10)
expect(hydrate).toHaveBeenCalledTimes(1)
expect((hydrate.mock.calls[0][0] as string[]).length).toBe(10)
} finally {
hydrate.mockRestore()
}
})
})