feat: add match visibility and semantic highlighting to hybrid search
- Add textMatches, textScore, semanticScore, matchSource to search results - Add highlight() method for zero-config text + semantic highlighting - Increase word indexing limit to 5000 (handles articles/chapters) - Optimize findMatchingWords() with O(1) fast path for semantic-only results - Add production safety limits (500 chunks for highlight) - Add comprehensive tests for new features (35 tests) - Update docs with match visibility and highlight() API
This commit is contained in:
parent
b0ea2f3810
commit
4adba1b254
7 changed files with 1727 additions and 31 deletions
431
src/brainy.ts
431
src/brainy.ts
|
|
@ -86,6 +86,25 @@ import { NounType, VerbType } from './types/graphTypes.js'
|
|||
import { BrainyInterface } from './types/brainyInterface.js'
|
||||
import type { IntegrationHub } from './integrations/core/IntegrationHub.js'
|
||||
|
||||
/**
|
||||
* Stopwords for semantic highlighting (v7.8.0)
|
||||
* These common words are skipped when highlighting individual words
|
||||
* to focus on meaningful content words.
|
||||
*/
|
||||
const STOPWORDS = new Set([
|
||||
'a', 'an', 'the', 'is', 'are', 'was', 'were', 'be', 'been',
|
||||
'being', 'have', 'has', 'had', 'do', 'does', 'did', 'will', 'would', 'could', 'should',
|
||||
'may', 'might', 'must', 'shall', 'can', 'to', 'of', 'in', 'for', 'on', 'with', 'at',
|
||||
'by', 'from', 'as', 'into', 'through', 'during', 'before', 'after', 'above', 'below',
|
||||
'between', 'under', 'again', 'further', 'then', 'once', 'here', 'there', 'when',
|
||||
'where', 'why', 'how', 'all', 'each', 'few', 'more', 'most', 'other', 'some', 'such',
|
||||
'no', 'nor', 'not', 'only', 'own', 'same', 'so', 'than', 'too', 'very', 'just',
|
||||
'and', 'but', 'or', 'if', 'because', 'until', 'while', 'although', 'though',
|
||||
'this', 'that', 'these', 'those', 'it', 'its', 'i', 'me', 'my', 'you', 'your',
|
||||
'he', 'him', 'his', 'she', 'her', 'we', 'us', 'our', 'they', 'them', 'their',
|
||||
'what', 'which', 'who', 'whom', 'whose', 'am'
|
||||
])
|
||||
|
||||
/**
|
||||
* The main Brainy class - Clean, Beautiful, Powerful
|
||||
* REAL IMPLEMENTATION - No stubs, no mocks
|
||||
|
|
@ -1921,25 +1940,49 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
|||
return results
|
||||
}
|
||||
|
||||
// Execute parallel searches for optimal performance
|
||||
const searchPromises: Promise<Result<T>[]>[] = []
|
||||
|
||||
// Vector search component
|
||||
if (params.query || params.vector) {
|
||||
searchPromises.push(this.executeVectorSearch(params))
|
||||
// v7.7.0: Zero-Config Hybrid Search
|
||||
// Determine search mode: auto (default) combines text + semantic for query searches
|
||||
const searchMode = params.searchMode || 'auto'
|
||||
const limit = params.limit || 10
|
||||
|
||||
// Handle text-only query (user explicitly wants text search)
|
||||
if (searchMode === 'text' && params.query && params.query.trim() !== '') {
|
||||
results = await this.executeTextSearch(params.query, limit * 2)
|
||||
}
|
||||
|
||||
// Proximity search component
|
||||
if (params.near) {
|
||||
searchPromises.push(this.executeProximitySearch(params))
|
||||
// Handle semantic-only query (user explicitly wants vector search)
|
||||
else if ((searchMode === 'semantic' || searchMode === 'vector') && (params.query || params.vector)) {
|
||||
results = await this.executeVectorSearch(params)
|
||||
}
|
||||
|
||||
// Execute searches in parallel
|
||||
if (searchPromises.length > 0) {
|
||||
const searchResults = await Promise.all(searchPromises)
|
||||
for (const batch of searchResults) {
|
||||
results.push(...batch)
|
||||
}
|
||||
// Handle explicit hybrid or auto mode with query
|
||||
else if ((searchMode === 'auto' || searchMode === 'hybrid') && params.query && params.query.trim() !== '' && !params.vector) {
|
||||
// Zero-config hybrid: combine text + semantic search with RRF fusion
|
||||
const [textResults, semanticResults] = await Promise.all([
|
||||
this.executeTextSearch(params.query, limit * 2),
|
||||
this.executeVectorSearch(params)
|
||||
])
|
||||
|
||||
// Use user-specified alpha or auto-detect based on query length
|
||||
const alpha = params.hybridAlpha ?? this.autoAlpha(params.query)
|
||||
|
||||
// v7.8.0: Tokenize query for match visibility
|
||||
const queryWords = this.metadataIndex.tokenize(params.query)
|
||||
|
||||
// RRF fusion combines both result sets with match visibility
|
||||
results = await this.rrfFusion(textResults, semanticResults, alpha, queryWords)
|
||||
}
|
||||
// Handle direct vector search (no query text) - no hybrid needed
|
||||
else if (params.vector && !params.query) {
|
||||
results = await this.executeVectorSearch(params)
|
||||
}
|
||||
// Handle proximity search
|
||||
else if (params.near) {
|
||||
results = await this.executeProximitySearch(params)
|
||||
}
|
||||
|
||||
// Execute parallel searches for additional criteria (proximity search in addition to query)
|
||||
if (params.near && params.query) {
|
||||
const proximityResults = await this.executeProximitySearch(params)
|
||||
results.push(...proximityResults)
|
||||
}
|
||||
|
||||
// Remove duplicate results from parallel searches
|
||||
|
|
@ -2109,11 +2152,10 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
|||
results.sort((a, b) => b.score - a.score)
|
||||
}
|
||||
|
||||
const limit = params.limit || 10
|
||||
const offset = params.offset || 0
|
||||
const finalOffset = params.offset || 0
|
||||
|
||||
// Efficient pagination - only slice what we need
|
||||
return results.slice(offset, offset + limit)
|
||||
// Efficient pagination - only slice what we need (limit already defined above)
|
||||
return results.slice(finalOffset, finalOffset + limit)
|
||||
})
|
||||
|
||||
// Record performance for auto-tuning
|
||||
|
|
@ -4662,6 +4704,160 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
|||
return 1 - distance
|
||||
}
|
||||
|
||||
/**
|
||||
* Zero-config hybrid highlighting (v7.8.0)
|
||||
*
|
||||
* Returns both exact text matches AND semantically similar concepts.
|
||||
* Perfect for UI highlighting at different levels:
|
||||
* - matchType: 'text' = exact word match (highlight strongly)
|
||||
* - matchType: 'semantic' = concept match (highlight softly)
|
||||
*
|
||||
* @param params.query - The search query
|
||||
* @param params.text - The text to highlight (e.g., entity.data)
|
||||
* @param params.granularity - 'word' | 'phrase' | 'sentence' (default: 'word')
|
||||
* @param params.threshold - Minimum similarity for semantic matches (default: 0.5)
|
||||
* @returns Array of highlights with text, score, position, and matchType
|
||||
*
|
||||
* @example
|
||||
* ```typescript
|
||||
* const highlights = await brain.highlight({
|
||||
* query: "david the warrior",
|
||||
* text: "David Smith is a brave fighter who battles dragons"
|
||||
* })
|
||||
* // Returns: [
|
||||
* // { text: "David", score: 1.0, position: [0, 5], matchType: 'text' }, // Exact
|
||||
* // { text: "fighter", score: 0.78, position: [25, 32], matchType: 'semantic' }, // Concept
|
||||
* // { text: "battles", score: 0.72, position: [37, 44], matchType: 'semantic' } // Concept
|
||||
* // ]
|
||||
* ```
|
||||
*/
|
||||
async highlight(params: import('./types/brainy.types.js').HighlightParams): Promise<import('./types/brainy.types.js').Highlight[]> {
|
||||
await this.ensureInitialized()
|
||||
|
||||
const { query, text, granularity = 'word', threshold = 0.5 } = params
|
||||
|
||||
if (!query || !text) {
|
||||
return []
|
||||
}
|
||||
|
||||
// Split text into chunks based on granularity
|
||||
const allChunks = this.splitForHighlighting(text, granularity)
|
||||
|
||||
if (allChunks.length === 0) {
|
||||
return []
|
||||
}
|
||||
|
||||
// v7.8.0 Production safety: Limit chunks to prevent memory explosion
|
||||
// At 500 words × 384 dimensions × 4 bytes = 768KB temp memory (acceptable)
|
||||
// At 10,000 words = 15MB temp memory (too much for concurrent requests)
|
||||
const MAX_HIGHLIGHT_CHUNKS = 500
|
||||
const chunks = allChunks.slice(0, MAX_HIGHLIGHT_CHUNKS)
|
||||
|
||||
// Track all highlights (keyed by position to avoid duplicates)
|
||||
const highlightMap = new Map<string, import('./types/brainy.types.js').Highlight>()
|
||||
|
||||
// === PHASE 1: Find exact text matches (score = 1.0, matchType = 'text') ===
|
||||
const queryWords = this.metadataIndex.tokenize(query)
|
||||
const queryWordsLower = new Set(queryWords.map(w => w.toLowerCase()))
|
||||
|
||||
for (const chunk of chunks) {
|
||||
const chunkLower = chunk.text.toLowerCase().replace(/[^\w\s]/g, '')
|
||||
if (queryWordsLower.has(chunkLower)) {
|
||||
const key = `${chunk.position[0]}-${chunk.position[1]}`
|
||||
highlightMap.set(key, {
|
||||
text: chunk.text,
|
||||
score: 1.0,
|
||||
position: chunk.position,
|
||||
matchType: 'text'
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// === PHASE 2: Find semantic matches (score varies, matchType = 'semantic') ===
|
||||
// Get query embedding
|
||||
const queryVector = await this.embed(query)
|
||||
|
||||
// Batch embed all chunks (efficient!)
|
||||
const chunkTexts = chunks.map(c => c.text)
|
||||
const chunkVectors = await this.embedBatch(chunkTexts)
|
||||
|
||||
// Calculate semantic similarities
|
||||
for (let i = 0; i < chunks.length; i++) {
|
||||
const key = `${chunks[i].position[0]}-${chunks[i].position[1]}`
|
||||
|
||||
// Skip if already a text match (text matches take priority)
|
||||
if (highlightMap.has(key)) continue
|
||||
|
||||
const distance = this.distance(queryVector, chunkVectors[i])
|
||||
const similarity = 1 - distance
|
||||
|
||||
if (similarity >= threshold) {
|
||||
highlightMap.set(key, {
|
||||
text: chunks[i].text,
|
||||
score: similarity,
|
||||
position: chunks[i].position,
|
||||
matchType: 'semantic'
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// Sort by score descending (text matches will be first with score=1.0)
|
||||
const highlights = Array.from(highlightMap.values())
|
||||
return highlights.sort((a, b) => b.score - a.score)
|
||||
}
|
||||
|
||||
/**
|
||||
* Split text into chunks for highlighting (v7.8.0)
|
||||
* @internal
|
||||
*/
|
||||
private splitForHighlighting(text: string, granularity: string): Array<{ text: string, position: [number, number] }> {
|
||||
const results: Array<{ text: string, position: [number, number] }> = []
|
||||
|
||||
if (granularity === 'word') {
|
||||
// Split on whitespace, track positions
|
||||
const regex = /\S+/g
|
||||
let match
|
||||
while ((match = regex.exec(text)) !== null) {
|
||||
// Skip stopwords
|
||||
if (!STOPWORDS.has(match[0].toLowerCase())) {
|
||||
results.push({ text: match[0], position: [match.index, match.index + match[0].length] })
|
||||
}
|
||||
}
|
||||
} else if (granularity === 'sentence') {
|
||||
// Split on sentence boundaries
|
||||
const regex = /[^.!?]+[.!?]+/g
|
||||
let match
|
||||
while ((match = regex.exec(text)) !== null) {
|
||||
results.push({ text: match[0].trim(), position: [match.index, match.index + match[0].length] })
|
||||
}
|
||||
// Handle text without sentence-ending punctuation
|
||||
if (results.length === 0 && text.trim()) {
|
||||
results.push({ text: text.trim(), position: [0, text.length] })
|
||||
}
|
||||
} else if (granularity === 'phrase') {
|
||||
// Sliding window of 2-4 words
|
||||
const words: Array<{ text: string, start: number, end: number }> = []
|
||||
const regex = /\S+/g
|
||||
let match
|
||||
while ((match = regex.exec(text)) !== null) {
|
||||
words.push({ text: match[0], start: match.index, end: match.index + match[0].length })
|
||||
}
|
||||
|
||||
// Generate 2-4 word phrases
|
||||
for (let windowSize = 2; windowSize <= 4; windowSize++) {
|
||||
for (let i = 0; i <= words.length - windowSize; i++) {
|
||||
const phraseWords = words.slice(i, i + windowSize)
|
||||
const phraseText = phraseWords.map(w => w.text).join(' ')
|
||||
const start = phraseWords[0].start
|
||||
const end = phraseWords[phraseWords.length - 1].end
|
||||
results.push({ text: phraseText, position: [start, end] })
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return results
|
||||
}
|
||||
|
||||
/**
|
||||
* Get comprehensive index statistics
|
||||
*
|
||||
|
|
@ -5370,6 +5566,199 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
|||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Execute text search using word index (v7.7.0)
|
||||
*
|
||||
* Performs keyword-based search using the __words__ index in MetadataIndexManager.
|
||||
* Returns results ranked by word match count.
|
||||
*
|
||||
* @param query - Text query to search for
|
||||
* @param limit - Maximum results to return
|
||||
* @returns Array of Results with scores based on match count
|
||||
*/
|
||||
private async executeTextSearch(query: string, limit: number): Promise<Result<T>[]> {
|
||||
const textMatches = await this.metadataIndex.getIdsForTextQuery(query)
|
||||
if (textMatches.length === 0) return []
|
||||
|
||||
// Take top matches and load entities
|
||||
const topMatches = textMatches.slice(0, limit * 2) // Get more for filtering
|
||||
const ids = topMatches.map(m => m.id)
|
||||
const entitiesMap = await this.batchGet(ids)
|
||||
|
||||
// Create results with scores based on match count
|
||||
const maxMatches = topMatches[0]?.matchCount || 1
|
||||
const results: Result<T>[] = []
|
||||
|
||||
for (const match of topMatches) {
|
||||
const entity = entitiesMap.get(match.id)
|
||||
if (entity) {
|
||||
// Normalize score to 0-1 range based on match count
|
||||
const score = match.matchCount / maxMatches
|
||||
results.push(this.createResult(match.id, score, entity))
|
||||
}
|
||||
}
|
||||
|
||||
return results
|
||||
}
|
||||
|
||||
/**
|
||||
* Auto-detect optimal alpha for hybrid search (v7.7.0)
|
||||
*
|
||||
* Short queries (1-2 words) favor text search (lower alpha)
|
||||
* Long queries (5+ words) favor semantic search (higher alpha)
|
||||
*
|
||||
* @param query - The search query
|
||||
* @returns Alpha value between 0 (text only) and 1 (semantic only)
|
||||
*/
|
||||
private autoAlpha(query: string): number {
|
||||
const wordCount = query.trim().split(/\s+/).filter(w => w.length > 0).length
|
||||
if (wordCount <= 2) return 0.3 // Favor text for short queries
|
||||
if (wordCount <= 5) return 0.5 // Balanced
|
||||
return 0.7 // Favor semantic for long queries
|
||||
}
|
||||
|
||||
/**
|
||||
* Reciprocal Rank Fusion (RRF) for combining search results (v7.7.0)
|
||||
*
|
||||
* RRF is a proven fusion algorithm that:
|
||||
* - Doesn't require score normalization
|
||||
* - Handles different score distributions
|
||||
* - Gives higher weight to top-ranked items
|
||||
*
|
||||
* Formula: score(d) = sum(1 / (k + rank(d))) for each list
|
||||
*
|
||||
* v7.8.0: Now includes match visibility (textMatches, textScore, semanticScore, matchSource)
|
||||
*
|
||||
* @param textResults - Results from text search
|
||||
* @param semanticResults - Results from semantic search
|
||||
* @param alpha - Weight for semantic (0=text only, 1=semantic only)
|
||||
* @param queryWords - Original query words for match tracking
|
||||
* @param k - RRF constant (default: 60, standard in literature)
|
||||
* @returns Fused results sorted by combined score with match visibility
|
||||
*/
|
||||
private async rrfFusion(
|
||||
textResults: Result<T>[],
|
||||
semanticResults: Result<T>[],
|
||||
alpha: number,
|
||||
queryWords: string[],
|
||||
k: number = 60
|
||||
): Promise<Result<T>[]> {
|
||||
// Track scores and match details per entity
|
||||
interface MatchData {
|
||||
rrf: number
|
||||
textScore?: number
|
||||
semanticScore?: number
|
||||
textMatches: string[]
|
||||
hasText: boolean
|
||||
hasSemantic: boolean
|
||||
}
|
||||
const matchData = new Map<string, MatchData>()
|
||||
const entityMap = new Map<string, Entity<T>>()
|
||||
|
||||
// Text contribution (1 - alpha weight)
|
||||
const textWeight = 1 - alpha
|
||||
textResults.forEach((r, rank) => {
|
||||
const rrfScore = textWeight * (1 / (k + rank + 1))
|
||||
const existing = matchData.get(r.id) || { rrf: 0, textMatches: [], hasText: false, hasSemantic: false }
|
||||
existing.rrf += rrfScore
|
||||
existing.textScore = r.score // Original text search score (0-1)
|
||||
existing.hasText = true
|
||||
matchData.set(r.id, existing)
|
||||
if (r.entity) entityMap.set(r.id, r.entity)
|
||||
})
|
||||
|
||||
// Semantic contribution (alpha weight)
|
||||
semanticResults.forEach((r, rank) => {
|
||||
const rrfScore = alpha * (1 / (k + rank + 1))
|
||||
const existing = matchData.get(r.id) || { rrf: 0, textMatches: [], hasText: false, hasSemantic: false }
|
||||
existing.rrf += rrfScore
|
||||
existing.semanticScore = r.score // Original semantic search score (0-1)
|
||||
existing.hasSemantic = true
|
||||
matchData.set(r.id, existing)
|
||||
if (r.entity) entityMap.set(r.id, r.entity)
|
||||
})
|
||||
|
||||
// Sort by fused score
|
||||
const sortedIds = Array.from(matchData.entries())
|
||||
.sort((a, b) => b[1].rrf - a[1].rrf)
|
||||
.map(([id, data]) => ({ id, data }))
|
||||
|
||||
// Build results - need to load any missing entities
|
||||
const missingIds = sortedIds.filter(s => !entityMap.has(s.id)).map(s => s.id)
|
||||
if (missingIds.length > 0) {
|
||||
const loaded = await this.batchGet(missingIds)
|
||||
for (const [id, entity] of loaded) {
|
||||
entityMap.set(id, entity)
|
||||
}
|
||||
}
|
||||
|
||||
// v7.8.0 Performance: Build set of text result IDs for O(1) lookup
|
||||
// This avoids re-extracting text for entities that weren't in text results
|
||||
const textResultIds = new Set(textResults.map(r => r.id))
|
||||
|
||||
// Create final results with match visibility
|
||||
const results: Result<T>[] = []
|
||||
for (const { id, data } of sortedIds) {
|
||||
const entity = entityMap.get(id)
|
||||
if (entity) {
|
||||
// Find which query words matched - uses fast path if entity wasn't in text results
|
||||
const textMatches = this.findMatchingWords(entity, queryWords, textResultIds)
|
||||
|
||||
// Determine match source
|
||||
let matchSource: 'text' | 'semantic' | 'both'
|
||||
if (data.hasText && data.hasSemantic) {
|
||||
matchSource = 'both'
|
||||
} else if (data.hasText) {
|
||||
matchSource = 'text'
|
||||
} else {
|
||||
matchSource = 'semantic'
|
||||
}
|
||||
|
||||
// Create result with match visibility
|
||||
const result = this.createResult(id, data.rrf, entity)
|
||||
result.textMatches = textMatches
|
||||
result.textScore = data.textScore
|
||||
result.semanticScore = data.semanticScore
|
||||
result.matchSource = matchSource
|
||||
|
||||
results.push(result)
|
||||
}
|
||||
}
|
||||
|
||||
return results
|
||||
}
|
||||
|
||||
/**
|
||||
* Find which query words match in an entity's text content (v7.8.0)
|
||||
*
|
||||
* Performance: O(query_words × text_length) - only called when needed
|
||||
* At scale: Use textResultIds set for O(1) lookup instead of re-extracting
|
||||
*
|
||||
* @param entity - Entity to check
|
||||
* @param queryWords - Words from the search query
|
||||
* @param textResultIds - Optional: Set of IDs from text search (O(1) lookup)
|
||||
* @returns Array of matching query words
|
||||
*/
|
||||
private findMatchingWords(
|
||||
entity: Entity<T>,
|
||||
queryWords: string[],
|
||||
textResultIds?: Set<string>
|
||||
): string[] {
|
||||
// Fast path: if entity wasn't in text results, no words matched
|
||||
if (textResultIds && !textResultIds.has(entity.id)) {
|
||||
return []
|
||||
}
|
||||
|
||||
// Slow path: extract text and check each word
|
||||
// Only happens for entities that DID match text search
|
||||
const textContent = this.metadataIndex.extractTextContent({
|
||||
data: entity.data,
|
||||
metadata: entity.metadata
|
||||
}).toLowerCase()
|
||||
|
||||
return queryWords.filter(word => textContent.includes(word.toLowerCase()))
|
||||
}
|
||||
|
||||
/**
|
||||
* Apply graph constraints using O(1) GraphAdjacencyIndex - TRUE Triple Intelligence!
|
||||
*/
|
||||
|
|
|
|||
|
|
@ -82,6 +82,12 @@ export interface Result<T = any> {
|
|||
|
||||
// Score transparency
|
||||
explanation?: ScoreExplanation
|
||||
|
||||
// Match visibility (v7.8.0) - shows what matched in hybrid search
|
||||
textMatches?: string[] // Query words found in entity (e.g., ["david", "warrior"])
|
||||
textScore?: number // Normalized text match score (0-1)
|
||||
semanticScore?: number // Semantic similarity score (0-1)
|
||||
matchSource?: 'text' | 'semantic' | 'both' // Where this result came from
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -192,6 +198,10 @@ export interface FindParams<T = any> {
|
|||
includeRelations?: boolean // Include entity relationships
|
||||
excludeVFS?: boolean // v4.7.0: Exclude VFS entities from results (default: false - VFS included)
|
||||
service?: string // Multi-tenancy filter
|
||||
|
||||
// Hybrid search options (v7.7.0)
|
||||
searchMode?: 'auto' | 'text' | 'semantic' | 'vector' | 'hybrid' // Search strategy: auto (default), text-only, semantic/vector-only, or explicit hybrid
|
||||
hybridAlpha?: number // Weight between text (0.0) and semantic (1.0) search, default: auto-detected by query length
|
||||
|
||||
// Triple Intelligence Fusion
|
||||
fusion?: {
|
||||
|
|
@ -223,12 +233,14 @@ export interface GraphConstraints {
|
|||
/**
|
||||
* Search modes
|
||||
*/
|
||||
export type SearchMode =
|
||||
| 'auto' // Automatically choose best mode
|
||||
| 'vector' // Pure vector search
|
||||
export type SearchMode =
|
||||
| 'auto' // Automatically choose best mode (default - enables hybrid text+semantic)
|
||||
| 'vector' // Pure vector search (semantic only)
|
||||
| 'semantic' // Alias for vector search
|
||||
| 'text' // Pure text/keyword search
|
||||
| 'metadata' // Pure metadata filtering
|
||||
| 'graph' // Pure graph traversal
|
||||
| 'hybrid' // Combine all intelligences
|
||||
| 'hybrid' // Combine all intelligences (explicit hybrid mode)
|
||||
|
||||
/**
|
||||
* Parameters for similarity search
|
||||
|
|
@ -777,6 +789,62 @@ export interface NeuralAnomalyParams {
|
|||
returnScores?: boolean // Return anomaly scores
|
||||
}
|
||||
|
||||
// ============= Semantic Highlighting Types (v7.8.0) =============
|
||||
|
||||
/**
|
||||
* Parameters for hybrid highlighting (v7.8.0)
|
||||
*
|
||||
* Zero-config highlighting that returns both text (exact) and semantic (concept) matches.
|
||||
* Perfect for UI highlighting at different levels.
|
||||
*
|
||||
* @example
|
||||
* ```typescript
|
||||
* const highlights = await brain.highlight({
|
||||
* query: "david the warrior",
|
||||
* text: "David Smith is a brave fighter who battles dragons"
|
||||
* })
|
||||
* // Returns: [
|
||||
* // { text: "David", score: 1.0, position: [0, 5], matchType: 'text' }, // Exact match
|
||||
* // { text: "fighter", score: 0.78, position: [25, 32], matchType: 'semantic' }, // Concept match
|
||||
* // { text: "battles", score: 0.72, position: [37, 44], matchType: 'semantic' } // Concept match
|
||||
* // ]
|
||||
* ```
|
||||
*/
|
||||
export interface HighlightParams {
|
||||
/** The search query to match against */
|
||||
query: string
|
||||
|
||||
/** The text to highlight (e.g., entity.data) */
|
||||
text: string
|
||||
|
||||
/** Granularity of highlighting: 'word' (default), 'phrase', or 'sentence' */
|
||||
granularity?: 'word' | 'phrase' | 'sentence'
|
||||
|
||||
/** Minimum semantic similarity score for semantic matches (default: 0.5) */
|
||||
threshold?: number
|
||||
}
|
||||
|
||||
/**
|
||||
* A highlight showing which text matched the query
|
||||
*
|
||||
* matchType tells the UI how to style the highlight:
|
||||
* - 'text': Exact word match (strongest signal, highest confidence)
|
||||
* - 'semantic': Conceptually similar match (may need softer highlight)
|
||||
*/
|
||||
export interface Highlight {
|
||||
/** The text that matched */
|
||||
text: string
|
||||
|
||||
/** Match score (0-1). For text matches, always 1.0. For semantic, varies by similarity. */
|
||||
score: number
|
||||
|
||||
/** Position in original text [start, end] */
|
||||
position: [number, number]
|
||||
|
||||
/** Match type: 'text' (exact word match) or 'semantic' (concept match) */
|
||||
matchType: 'text' | 'semantic'
|
||||
}
|
||||
|
||||
// ============= Export all types =============
|
||||
|
||||
export * from './graphTypes.js' // Re-export NounType, VerbType, etc.
|
||||
|
|
@ -1222,9 +1222,156 @@ export class MetadataIndexManager {
|
|||
extract(data)
|
||||
}
|
||||
|
||||
// v7.7.0: Extract words for hybrid text search
|
||||
// v7.8.0: Production-scale word limit (5000 words)
|
||||
// - Handles articles, chapters, and large documents
|
||||
// - Roaring Bitmaps + Chunked Sparse Index + LRU caching
|
||||
// - Int32 hashes store words as 4-byte values, not strings
|
||||
//
|
||||
// Memory managed by existing optimizations:
|
||||
// - Roaring Bitmaps: 90%+ compression for sparse data
|
||||
// - Chunked Sparse Index: ~50 values per chunk, lazy-loaded
|
||||
// - UnifiedCache LRU: Only hot chunks in memory
|
||||
//
|
||||
// Future: Bloom filter hybrid for unlimited words (see .strategy/BILLION-SCALE-PLAN.md)
|
||||
const textContent = this.extractTextContent(data)
|
||||
if (textContent) {
|
||||
const MAX_WORDS_PER_ENTITY = 5000 // Handles articles/chapters, memory-safe at scale
|
||||
const allWords = this.tokenize(textContent)
|
||||
const words = allWords.slice(0, MAX_WORDS_PER_ENTITY)
|
||||
|
||||
if (allWords.length > MAX_WORDS_PER_ENTITY) {
|
||||
// Log once per entity, not per word - avoids log spam
|
||||
prodLog.debug(
|
||||
`Entity text has ${allWords.length} words, indexing first ${MAX_WORDS_PER_ENTITY} for hybrid search`
|
||||
)
|
||||
}
|
||||
|
||||
for (const word of words) {
|
||||
// Hash word to int32 for memory efficiency (saves ~10GB at 1B scale)
|
||||
const wordHash = this.hashWord(word)
|
||||
fields.push({ field: '__words__', value: wordHash })
|
||||
}
|
||||
}
|
||||
|
||||
return fields
|
||||
}
|
||||
|
||||
/**
|
||||
* Extract text content from entity data for word indexing (v7.7.0)
|
||||
*
|
||||
* Recursively extracts string values from data, excluding:
|
||||
* - vector, embedding, connections, level, id (internal fields)
|
||||
* - Arrays with more than 10 elements (likely vectors/bulk data)
|
||||
* - Numeric-only keys (array indices)
|
||||
*
|
||||
* @param data - Entity data or metadata
|
||||
* @returns Concatenated text content
|
||||
*/
|
||||
extractTextContent(data: any): string {
|
||||
if (data === null || data === undefined) return ''
|
||||
if (typeof data === 'string') return data
|
||||
if (typeof data === 'number' || typeof data === 'boolean') return String(data)
|
||||
if (Array.isArray(data)) {
|
||||
// Skip large arrays (likely vectors)
|
||||
if (data.length > 10) return ''
|
||||
return data.map(d => this.extractTextContent(d)).filter(Boolean).join(' ')
|
||||
}
|
||||
if (typeof data === 'object') {
|
||||
const skipKeys = new Set(['vector', 'embedding', 'embeddings', 'connections', 'level', 'id'])
|
||||
const texts: string[] = []
|
||||
for (const [key, value] of Object.entries(data)) {
|
||||
// Skip internal fields and numeric keys (array indices)
|
||||
if (skipKeys.has(key) || /^\d+$/.test(key)) continue
|
||||
const text = this.extractTextContent(value)
|
||||
if (text) texts.push(text)
|
||||
}
|
||||
return texts.join(' ')
|
||||
}
|
||||
return ''
|
||||
}
|
||||
|
||||
/**
|
||||
* Tokenize text into words for indexing (v7.7.0)
|
||||
*
|
||||
* - Converts to lowercase
|
||||
* - Removes punctuation
|
||||
* - Splits on whitespace
|
||||
* - Filters by length (2-50 chars)
|
||||
* - Deduplicates per entity
|
||||
*
|
||||
* @param text - Text content to tokenize
|
||||
* @returns Array of unique words
|
||||
*/
|
||||
tokenize(text: string): string[] {
|
||||
if (!text) return []
|
||||
return text
|
||||
.toLowerCase()
|
||||
.replace(/[^\w\s]/g, ' ') // Remove punctuation
|
||||
.split(/\s+/) // Split on whitespace
|
||||
.filter(w => w.length >= 2 && w.length <= 50) // Length filter
|
||||
.filter((w, i, arr) => arr.indexOf(w) === i) // Dedupe per entity
|
||||
}
|
||||
|
||||
/**
|
||||
* Hash word to int32 using FNV-1a (v7.7.0)
|
||||
*
|
||||
* FNV-1a is fast with low collision rate, suitable for word hashing.
|
||||
* Saves ~10GB at billion scale by avoiding string storage.
|
||||
*
|
||||
* @param word - Word to hash
|
||||
* @returns Int32 hash value
|
||||
*/
|
||||
hashWord(word: string): number {
|
||||
let hash = 2166136261 // FNV offset basis
|
||||
for (let i = 0; i < word.length; i++) {
|
||||
hash ^= word.charCodeAt(i)
|
||||
hash = Math.imul(hash, 16777619) // FNV prime
|
||||
}
|
||||
return hash | 0 // Convert to signed int32
|
||||
}
|
||||
|
||||
/**
|
||||
* Get entity IDs matching a text query (v7.7.0)
|
||||
*
|
||||
* Performs word-based text search using the __words__ index.
|
||||
* Returns IDs ranked by match count (entities with more matching words first).
|
||||
*
|
||||
* @param query - Text query to search for
|
||||
* @returns Array of { id, matchCount } sorted by matchCount descending
|
||||
*/
|
||||
async getIdsForTextQuery(query: string): Promise<Array<{ id: string; matchCount: number }>> {
|
||||
const queryWords = this.tokenize(query)
|
||||
if (queryWords.length === 0) return []
|
||||
|
||||
// Get IDs for each word hash
|
||||
const wordIdSets: Map<string, number>[] = []
|
||||
for (const word of queryWords) {
|
||||
const wordHash = this.hashWord(word)
|
||||
const ids = await this.getIds('__words__', wordHash)
|
||||
const idSet = new Map<string, number>()
|
||||
for (const id of ids) {
|
||||
idSet.set(id, 1)
|
||||
}
|
||||
wordIdSets.push(idSet)
|
||||
}
|
||||
|
||||
if (wordIdSets.length === 0) return []
|
||||
|
||||
// Count matches per entity
|
||||
const matchCounts = new Map<string, number>()
|
||||
for (const idSet of wordIdSets) {
|
||||
for (const [id] of idSet) {
|
||||
matchCounts.set(id, (matchCounts.get(id) || 0) + 1)
|
||||
}
|
||||
}
|
||||
|
||||
// Sort by match count descending
|
||||
return Array.from(matchCounts.entries())
|
||||
.map(([id, matchCount]) => ({ id, matchCount }))
|
||||
.sort((a, b) => b.matchCount - a.matchCount)
|
||||
}
|
||||
|
||||
/**
|
||||
* Add item to metadata indexes
|
||||
*
|
||||
|
|
@ -1240,13 +1387,24 @@ export class MetadataIndexManager {
|
|||
const fields = this.extractIndexableFields(entityOrMetadata)
|
||||
|
||||
// v6.7.0: Sanity check for excessive indexed fields (indicates possible data issue)
|
||||
if (fields.length > 100) {
|
||||
// v7.8.0: Separate threshold for metadata fields vs word fields
|
||||
// - Metadata fields: warn if > 100 (indicates deeply nested metadata)
|
||||
// - Word fields: expected to be many for large documents, warn only for extreme cases
|
||||
const metadataFields = fields.filter(f => f.field !== '__words__')
|
||||
const wordFields = fields.filter(f => f.field === '__words__')
|
||||
|
||||
if (metadataFields.length > 100) {
|
||||
prodLog.warn(
|
||||
`Entity ${id} has ${fields.length} indexed fields (expected ~30). ` +
|
||||
`Possible deeply nested metadata or data issue. First 10 fields: ${fields.slice(0, 10).map(f => f.field).join(', ')}`
|
||||
`Entity ${id} has ${metadataFields.length} metadata fields (expected ~30). ` +
|
||||
`Possible deeply nested metadata. First 10 fields: ${metadataFields.slice(0, 10).map(f => f.field).join(', ')}`
|
||||
)
|
||||
}
|
||||
|
||||
// Words are expected to be many for large documents - only log for extreme cases
|
||||
if (wordFields.length > 5000) {
|
||||
prodLog.debug(`Entity ${id} has ${wordFields.length} indexed words (large document)`)
|
||||
}
|
||||
|
||||
// Sort fields to process 'noun' field first for type-field affinity tracking
|
||||
fields.sort((a, b) => {
|
||||
if (a.field === 'noun') return -1
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue