diff --git a/docs/FIND_SYSTEM.md b/docs/FIND_SYSTEM.md index f43df101..56ecbb00 100644 --- a/docs/FIND_SYSTEM.md +++ b/docs/FIND_SYSTEM.md @@ -4,7 +4,7 @@ Brainy's `find()` method is the most advanced query system in any vector database, combining **Triple Intelligence** (vector + metadata + graph) with **Type-Aware NLP** for natural language understanding. -## Architecture: Three Intelligence Systems +## Architecture: Four Intelligence Systems ### 1. Vector Intelligence (HNSW Index) - **Purpose**: Semantic similarity search using embeddings @@ -13,14 +13,22 @@ Brainy's `find()` method is the most advanced query system in any vector databas - **Data Structure**: Multi-layer graph with 16 connections per node - **Use Cases**: "Find similar documents", "Content like this" -### 2. Metadata Intelligence (Incremental Indices) +### 2. Text Intelligence (Word Index) - v7.7.0 +- **Purpose**: Keyword/exact text matching +- **Algorithm**: Inverted word index with FNV-1a hashing +- **Performance**: O(log C) where C = chunks (~50 values each) +- **Data Structure**: `__words__ → hash → Roaring Bitmap of entity IDs` +- **Use Cases**: "Find exact name", "Keyword search" +- **Integration**: Automatically combined with Vector via RRF fusion + +### 3. Metadata Intelligence (Incremental Indices) - **Purpose**: Fast filtering on structured data - **Algorithm**: HashMap for exact matches, Sorted arrays for ranges - **Performance**: O(1) exact, O(log n) ranges, <1ms typical - **Data Structure**: `Map>` + sorted value arrays - **Use Cases**: "Documents from 2023", "Status equals active" -### 3. Graph Intelligence (Adjacency Maps) +### 4. Graph Intelligence (Adjacency Maps) - **Purpose**: Relationship traversal and connection analysis - **Algorithm**: Pure O(1) neighbor lookups via Map operations - **Performance**: O(1) per hop, ~0.1ms typical @@ -84,6 +92,84 @@ await brain.find({ }) ``` +### 4. Hybrid Search (v7.7.0+) +```typescript +// Zero-config hybrid: automatically combines text + semantic search +await brain.find({ query: "David Smith" }) +// Uses Reciprocal Rank Fusion (RRF) to combine results + +// Force text-only search +await brain.find({ query: "exact keyword", searchMode: 'text' }) + +// Force semantic-only search +await brain.find({ query: "AI concepts", searchMode: 'semantic' }) + +// Custom hybrid weighting (0 = text only, 1 = semantic only) +await brain.find({ query: "search term", hybridAlpha: 0.3 }) +``` + +**How Auto-Alpha Works:** +- Short queries (1-2 words): alpha = 0.3 (favor text matching) +- Medium queries (3-4 words): alpha = 0.5 (balanced) +- Long queries (5+ words): alpha = 0.7 (favor semantic matching) + +### 5. Match Visibility (v7.8.0) + +Search results include match details showing what matched: + +```typescript +const results = await brain.find({ query: 'david the warrior' }) + +// Each result has: +results[0].textMatches // ["david", "warrior"] - exact words found +results[0].textScore // 0.25 - text match quality (0-1) +results[0].semanticScore // 0.87 - semantic similarity (0-1) +results[0].matchSource // 'both' | 'text' | 'semantic' +``` + +Use this for: +- **Highlighting** exact matches in UI (textMatches) +- **Explaining** why a result was found (matchSource) +- **Debugging** search behavior (separate scores) + +### 6. Semantic Highlighting (v7.8.0) + +Highlight which concepts/words in text matched your query: + +```typescript +// Find semantically similar words + exact matches +const highlights = await brain.highlight({ + query: "david the warrior", + text: "David Smith is a brave fighter who battles dragons" +}) + +// Returns: +// [ +// { text: "David", score: 1.0, position: [0, 5], matchType: 'text' }, +// { text: "fighter", score: 0.78, position: [25, 32], matchType: 'semantic' }, +// { text: "battles", score: 0.72, position: [37, 44], matchType: 'semantic' } +// ] +``` + +**Features:** +- `matchType: 'text'` - Exact word match (score = 1.0) +- `matchType: 'semantic'` - Concept match (score varies) +- `position` - [start, end] for precise highlighting +- `granularity` - 'word' (default), 'phrase', or 'sentence' +- `threshold` - Minimum semantic score (default: 0.5) + +**UI Usage Pattern:** +```typescript +// Highlight search results with different styles +function highlightResult(text: string, highlights: Highlight[]) { + return highlights.map(h => ({ + text: h.text, + position: h.position, + style: h.matchType === 'text' ? 'strong' : 'emphasis' // Different UI styles + })) +} +``` + ## Index Usage in Detail ### Metadata Index Operations diff --git a/docs/api/README.md b/docs/api/README.md index 3873474a..e62f4f49 100644 --- a/docs/api/README.md +++ b/docs/api/README.md @@ -202,11 +202,115 @@ const results = await brain.find({ - `depth?`: `number` - Traversal depth - `limit?`: `number` - Max results (default: 10) - `offset?`: `number` - Skip results +- `searchMode?`: `'auto' | 'text' | 'semantic' | 'hybrid'` - Search strategy (v7.7.0): + - `'auto'` (default): Zero-config hybrid combining text + semantic search + - `'text'`: Pure keyword/text matching + - `'semantic'`: Pure vector similarity + - `'hybrid'`: Explicit hybrid mode +- `hybridAlpha?`: `number` - Balance between text (0.0) and semantic (1.0) search. Auto-detected by query length if not specified. **Returns:** `Promise` - Matching entities with scores --- +### Hybrid Search (v7.7.0) + +Brainy automatically combines text (keyword) and semantic (vector) search for optimal results. No configuration needed. + +```typescript +// Zero-config hybrid search (just works) +const results = await brain.find({ + query: 'David Smith' // Finds both exact text matches AND semantically similar +}) + +// Force text-only search (exact keyword matching) +const textResults = await brain.find({ + query: 'exact keyword', + searchMode: 'text' +}) + +// Force semantic-only search (vector similarity) +const semanticResults = await brain.find({ + query: 'artificial intelligence concepts', + searchMode: 'semantic' +}) + +// Custom hybrid weighting (0 = text only, 1 = semantic only) +const customResults = await brain.find({ + query: 'David Smith', + hybridAlpha: 0.3 // Favor text matching +}) +``` + +**How it works:** +- Short queries (1-2 words) automatically favor text matching +- Long queries (5+ words) automatically favor semantic search +- Results are combined using Reciprocal Rank Fusion (RRF) + +--- + +### Match Visibility (v7.8.0) + +Search results include detailed match information: + +```typescript +const results = await brain.find({ query: 'david the warrior' }) + +// Each result now includes: +results[0].textMatches // ["david", "warrior"] - exact query words found +results[0].textScore // 0.25 - text match quality (0-1) +results[0].semanticScore // 0.87 - semantic similarity (0-1) +results[0].matchSource // 'both' | 'text' | 'semantic' +``` + +**Use cases:** +- Highlight exact matches in UI (textMatches) +- Explain why a result ranked high (matchSource) +- Debug search behavior (separate scores) + +--- + +### `highlight(params)` → `Promise` ✨ *New v7.8.0* + +Zero-config highlighting for both exact matches AND semantic concepts. + +```typescript +const highlights = await brain.highlight({ + query: "david the warrior", + text: "David Smith is a brave fighter who battles dragons" +}) + +// Returns both text matches AND semantic matches: +// [ +// { text: "David", score: 1.0, position: [0, 5], matchType: 'text' }, +// { text: "fighter", score: 0.78, position: [25, 32], matchType: 'semantic' }, +// { text: "battles", score: 0.72, position: [37, 44], matchType: 'semantic' } +// ] +``` + +**Parameters:** +- `query`: `string` - The search query +- `text`: `string` - Text to highlight (e.g., entity.data) +- `granularity?`: `'word' | 'phrase' | 'sentence'` - Highlight unit (default: 'word') +- `threshold?`: `number` - Min similarity for semantic matches (default: 0.5) + +**Returns:** `Promise` +- `text` - The matched text +- `score` - Match score (1.0 for text matches, varies for semantic) +- `position` - [start, end] indices in original text +- `matchType` - `'text'` (exact) or `'semantic'` (concept) + +**UI Pattern:** +```typescript +// Style differently based on match type +highlights.forEach(h => { + const style = h.matchType === 'text' ? 'font-weight: bold' : 'background: yellow' + // Apply style from h.position[0] to h.position[1] +}) +``` + +--- + ### Query Operators Brainy uses clean, readable operators: diff --git a/src/brainy.ts b/src/brainy.ts index 01ecfb37..d7bfec7a 100644 --- a/src/brainy.ts +++ b/src/brainy.ts @@ -86,6 +86,25 @@ import { NounType, VerbType } from './types/graphTypes.js' import { BrainyInterface } from './types/brainyInterface.js' import type { IntegrationHub } from './integrations/core/IntegrationHub.js' +/** + * Stopwords for semantic highlighting (v7.8.0) + * These common words are skipped when highlighting individual words + * to focus on meaningful content words. + */ +const STOPWORDS = new Set([ + 'a', 'an', 'the', 'is', 'are', 'was', 'were', 'be', 'been', + 'being', 'have', 'has', 'had', 'do', 'does', 'did', 'will', 'would', 'could', 'should', + 'may', 'might', 'must', 'shall', 'can', 'to', 'of', 'in', 'for', 'on', 'with', 'at', + 'by', 'from', 'as', 'into', 'through', 'during', 'before', 'after', 'above', 'below', + 'between', 'under', 'again', 'further', 'then', 'once', 'here', 'there', 'when', + 'where', 'why', 'how', 'all', 'each', 'few', 'more', 'most', 'other', 'some', 'such', + 'no', 'nor', 'not', 'only', 'own', 'same', 'so', 'than', 'too', 'very', 'just', + 'and', 'but', 'or', 'if', 'because', 'until', 'while', 'although', 'though', + 'this', 'that', 'these', 'those', 'it', 'its', 'i', 'me', 'my', 'you', 'your', + 'he', 'him', 'his', 'she', 'her', 'we', 'us', 'our', 'they', 'them', 'their', + 'what', 'which', 'who', 'whom', 'whose', 'am' +]) + /** * The main Brainy class - Clean, Beautiful, Powerful * REAL IMPLEMENTATION - No stubs, no mocks @@ -1921,25 +1940,49 @@ export class Brainy implements BrainyInterface { return results } - // Execute parallel searches for optimal performance - const searchPromises: Promise[]>[] = [] - - // Vector search component - if (params.query || params.vector) { - searchPromises.push(this.executeVectorSearch(params)) + // v7.7.0: Zero-Config Hybrid Search + // Determine search mode: auto (default) combines text + semantic for query searches + const searchMode = params.searchMode || 'auto' + const limit = params.limit || 10 + + // Handle text-only query (user explicitly wants text search) + if (searchMode === 'text' && params.query && params.query.trim() !== '') { + results = await this.executeTextSearch(params.query, limit * 2) } - - // Proximity search component - if (params.near) { - searchPromises.push(this.executeProximitySearch(params)) + // Handle semantic-only query (user explicitly wants vector search) + else if ((searchMode === 'semantic' || searchMode === 'vector') && (params.query || params.vector)) { + results = await this.executeVectorSearch(params) } - - // Execute searches in parallel - if (searchPromises.length > 0) { - const searchResults = await Promise.all(searchPromises) - for (const batch of searchResults) { - results.push(...batch) - } + // Handle explicit hybrid or auto mode with query + else if ((searchMode === 'auto' || searchMode === 'hybrid') && params.query && params.query.trim() !== '' && !params.vector) { + // Zero-config hybrid: combine text + semantic search with RRF fusion + const [textResults, semanticResults] = await Promise.all([ + this.executeTextSearch(params.query, limit * 2), + this.executeVectorSearch(params) + ]) + + // Use user-specified alpha or auto-detect based on query length + const alpha = params.hybridAlpha ?? this.autoAlpha(params.query) + + // v7.8.0: Tokenize query for match visibility + const queryWords = this.metadataIndex.tokenize(params.query) + + // RRF fusion combines both result sets with match visibility + results = await this.rrfFusion(textResults, semanticResults, alpha, queryWords) + } + // Handle direct vector search (no query text) - no hybrid needed + else if (params.vector && !params.query) { + results = await this.executeVectorSearch(params) + } + // Handle proximity search + else if (params.near) { + results = await this.executeProximitySearch(params) + } + + // Execute parallel searches for additional criteria (proximity search in addition to query) + if (params.near && params.query) { + const proximityResults = await this.executeProximitySearch(params) + results.push(...proximityResults) } // Remove duplicate results from parallel searches @@ -2109,11 +2152,10 @@ export class Brainy implements BrainyInterface { results.sort((a, b) => b.score - a.score) } - const limit = params.limit || 10 - const offset = params.offset || 0 + const finalOffset = params.offset || 0 - // Efficient pagination - only slice what we need - return results.slice(offset, offset + limit) + // Efficient pagination - only slice what we need (limit already defined above) + return results.slice(finalOffset, finalOffset + limit) }) // Record performance for auto-tuning @@ -4662,6 +4704,160 @@ export class Brainy implements BrainyInterface { return 1 - distance } + /** + * Zero-config hybrid highlighting (v7.8.0) + * + * Returns both exact text matches AND semantically similar concepts. + * Perfect for UI highlighting at different levels: + * - matchType: 'text' = exact word match (highlight strongly) + * - matchType: 'semantic' = concept match (highlight softly) + * + * @param params.query - The search query + * @param params.text - The text to highlight (e.g., entity.data) + * @param params.granularity - 'word' | 'phrase' | 'sentence' (default: 'word') + * @param params.threshold - Minimum similarity for semantic matches (default: 0.5) + * @returns Array of highlights with text, score, position, and matchType + * + * @example + * ```typescript + * const highlights = await brain.highlight({ + * query: "david the warrior", + * text: "David Smith is a brave fighter who battles dragons" + * }) + * // Returns: [ + * // { text: "David", score: 1.0, position: [0, 5], matchType: 'text' }, // Exact + * // { text: "fighter", score: 0.78, position: [25, 32], matchType: 'semantic' }, // Concept + * // { text: "battles", score: 0.72, position: [37, 44], matchType: 'semantic' } // Concept + * // ] + * ``` + */ + async highlight(params: import('./types/brainy.types.js').HighlightParams): Promise { + await this.ensureInitialized() + + const { query, text, granularity = 'word', threshold = 0.5 } = params + + if (!query || !text) { + return [] + } + + // Split text into chunks based on granularity + const allChunks = this.splitForHighlighting(text, granularity) + + if (allChunks.length === 0) { + return [] + } + + // v7.8.0 Production safety: Limit chunks to prevent memory explosion + // At 500 words × 384 dimensions × 4 bytes = 768KB temp memory (acceptable) + // At 10,000 words = 15MB temp memory (too much for concurrent requests) + const MAX_HIGHLIGHT_CHUNKS = 500 + const chunks = allChunks.slice(0, MAX_HIGHLIGHT_CHUNKS) + + // Track all highlights (keyed by position to avoid duplicates) + const highlightMap = new Map() + + // === PHASE 1: Find exact text matches (score = 1.0, matchType = 'text') === + const queryWords = this.metadataIndex.tokenize(query) + const queryWordsLower = new Set(queryWords.map(w => w.toLowerCase())) + + for (const chunk of chunks) { + const chunkLower = chunk.text.toLowerCase().replace(/[^\w\s]/g, '') + if (queryWordsLower.has(chunkLower)) { + const key = `${chunk.position[0]}-${chunk.position[1]}` + highlightMap.set(key, { + text: chunk.text, + score: 1.0, + position: chunk.position, + matchType: 'text' + }) + } + } + + // === PHASE 2: Find semantic matches (score varies, matchType = 'semantic') === + // Get query embedding + const queryVector = await this.embed(query) + + // Batch embed all chunks (efficient!) + const chunkTexts = chunks.map(c => c.text) + const chunkVectors = await this.embedBatch(chunkTexts) + + // Calculate semantic similarities + for (let i = 0; i < chunks.length; i++) { + const key = `${chunks[i].position[0]}-${chunks[i].position[1]}` + + // Skip if already a text match (text matches take priority) + if (highlightMap.has(key)) continue + + const distance = this.distance(queryVector, chunkVectors[i]) + const similarity = 1 - distance + + if (similarity >= threshold) { + highlightMap.set(key, { + text: chunks[i].text, + score: similarity, + position: chunks[i].position, + matchType: 'semantic' + }) + } + } + + // Sort by score descending (text matches will be first with score=1.0) + const highlights = Array.from(highlightMap.values()) + return highlights.sort((a, b) => b.score - a.score) + } + + /** + * Split text into chunks for highlighting (v7.8.0) + * @internal + */ + private splitForHighlighting(text: string, granularity: string): Array<{ text: string, position: [number, number] }> { + const results: Array<{ text: string, position: [number, number] }> = [] + + if (granularity === 'word') { + // Split on whitespace, track positions + const regex = /\S+/g + let match + while ((match = regex.exec(text)) !== null) { + // Skip stopwords + if (!STOPWORDS.has(match[0].toLowerCase())) { + results.push({ text: match[0], position: [match.index, match.index + match[0].length] }) + } + } + } else if (granularity === 'sentence') { + // Split on sentence boundaries + const regex = /[^.!?]+[.!?]+/g + let match + while ((match = regex.exec(text)) !== null) { + results.push({ text: match[0].trim(), position: [match.index, match.index + match[0].length] }) + } + // Handle text without sentence-ending punctuation + if (results.length === 0 && text.trim()) { + results.push({ text: text.trim(), position: [0, text.length] }) + } + } else if (granularity === 'phrase') { + // Sliding window of 2-4 words + const words: Array<{ text: string, start: number, end: number }> = [] + const regex = /\S+/g + let match + while ((match = regex.exec(text)) !== null) { + words.push({ text: match[0], start: match.index, end: match.index + match[0].length }) + } + + // Generate 2-4 word phrases + for (let windowSize = 2; windowSize <= 4; windowSize++) { + for (let i = 0; i <= words.length - windowSize; i++) { + const phraseWords = words.slice(i, i + windowSize) + const phraseText = phraseWords.map(w => w.text).join(' ') + const start = phraseWords[0].start + const end = phraseWords[phraseWords.length - 1].end + results.push({ text: phraseText, position: [start, end] }) + } + } + } + + return results + } + /** * Get comprehensive index statistics * @@ -5370,6 +5566,199 @@ export class Brainy implements BrainyInterface { } } + /** + * Execute text search using word index (v7.7.0) + * + * Performs keyword-based search using the __words__ index in MetadataIndexManager. + * Returns results ranked by word match count. + * + * @param query - Text query to search for + * @param limit - Maximum results to return + * @returns Array of Results with scores based on match count + */ + private async executeTextSearch(query: string, limit: number): Promise[]> { + const textMatches = await this.metadataIndex.getIdsForTextQuery(query) + if (textMatches.length === 0) return [] + + // Take top matches and load entities + const topMatches = textMatches.slice(0, limit * 2) // Get more for filtering + const ids = topMatches.map(m => m.id) + const entitiesMap = await this.batchGet(ids) + + // Create results with scores based on match count + const maxMatches = topMatches[0]?.matchCount || 1 + const results: Result[] = [] + + for (const match of topMatches) { + const entity = entitiesMap.get(match.id) + if (entity) { + // Normalize score to 0-1 range based on match count + const score = match.matchCount / maxMatches + results.push(this.createResult(match.id, score, entity)) + } + } + + return results + } + + /** + * Auto-detect optimal alpha for hybrid search (v7.7.0) + * + * Short queries (1-2 words) favor text search (lower alpha) + * Long queries (5+ words) favor semantic search (higher alpha) + * + * @param query - The search query + * @returns Alpha value between 0 (text only) and 1 (semantic only) + */ + private autoAlpha(query: string): number { + const wordCount = query.trim().split(/\s+/).filter(w => w.length > 0).length + if (wordCount <= 2) return 0.3 // Favor text for short queries + if (wordCount <= 5) return 0.5 // Balanced + return 0.7 // Favor semantic for long queries + } + + /** + * Reciprocal Rank Fusion (RRF) for combining search results (v7.7.0) + * + * RRF is a proven fusion algorithm that: + * - Doesn't require score normalization + * - Handles different score distributions + * - Gives higher weight to top-ranked items + * + * Formula: score(d) = sum(1 / (k + rank(d))) for each list + * + * v7.8.0: Now includes match visibility (textMatches, textScore, semanticScore, matchSource) + * + * @param textResults - Results from text search + * @param semanticResults - Results from semantic search + * @param alpha - Weight for semantic (0=text only, 1=semantic only) + * @param queryWords - Original query words for match tracking + * @param k - RRF constant (default: 60, standard in literature) + * @returns Fused results sorted by combined score with match visibility + */ + private async rrfFusion( + textResults: Result[], + semanticResults: Result[], + alpha: number, + queryWords: string[], + k: number = 60 + ): Promise[]> { + // Track scores and match details per entity + interface MatchData { + rrf: number + textScore?: number + semanticScore?: number + textMatches: string[] + hasText: boolean + hasSemantic: boolean + } + const matchData = new Map() + const entityMap = new Map>() + + // Text contribution (1 - alpha weight) + const textWeight = 1 - alpha + textResults.forEach((r, rank) => { + const rrfScore = textWeight * (1 / (k + rank + 1)) + const existing = matchData.get(r.id) || { rrf: 0, textMatches: [], hasText: false, hasSemantic: false } + existing.rrf += rrfScore + existing.textScore = r.score // Original text search score (0-1) + existing.hasText = true + matchData.set(r.id, existing) + if (r.entity) entityMap.set(r.id, r.entity) + }) + + // Semantic contribution (alpha weight) + semanticResults.forEach((r, rank) => { + const rrfScore = alpha * (1 / (k + rank + 1)) + const existing = matchData.get(r.id) || { rrf: 0, textMatches: [], hasText: false, hasSemantic: false } + existing.rrf += rrfScore + existing.semanticScore = r.score // Original semantic search score (0-1) + existing.hasSemantic = true + matchData.set(r.id, existing) + if (r.entity) entityMap.set(r.id, r.entity) + }) + + // Sort by fused score + const sortedIds = Array.from(matchData.entries()) + .sort((a, b) => b[1].rrf - a[1].rrf) + .map(([id, data]) => ({ id, data })) + + // Build results - need to load any missing entities + const missingIds = sortedIds.filter(s => !entityMap.has(s.id)).map(s => s.id) + if (missingIds.length > 0) { + const loaded = await this.batchGet(missingIds) + for (const [id, entity] of loaded) { + entityMap.set(id, entity) + } + } + + // v7.8.0 Performance: Build set of text result IDs for O(1) lookup + // This avoids re-extracting text for entities that weren't in text results + const textResultIds = new Set(textResults.map(r => r.id)) + + // Create final results with match visibility + const results: Result[] = [] + for (const { id, data } of sortedIds) { + const entity = entityMap.get(id) + if (entity) { + // Find which query words matched - uses fast path if entity wasn't in text results + const textMatches = this.findMatchingWords(entity, queryWords, textResultIds) + + // Determine match source + let matchSource: 'text' | 'semantic' | 'both' + if (data.hasText && data.hasSemantic) { + matchSource = 'both' + } else if (data.hasText) { + matchSource = 'text' + } else { + matchSource = 'semantic' + } + + // Create result with match visibility + const result = this.createResult(id, data.rrf, entity) + result.textMatches = textMatches + result.textScore = data.textScore + result.semanticScore = data.semanticScore + result.matchSource = matchSource + + results.push(result) + } + } + + return results + } + + /** + * Find which query words match in an entity's text content (v7.8.0) + * + * Performance: O(query_words × text_length) - only called when needed + * At scale: Use textResultIds set for O(1) lookup instead of re-extracting + * + * @param entity - Entity to check + * @param queryWords - Words from the search query + * @param textResultIds - Optional: Set of IDs from text search (O(1) lookup) + * @returns Array of matching query words + */ + private findMatchingWords( + entity: Entity, + queryWords: string[], + textResultIds?: Set + ): string[] { + // Fast path: if entity wasn't in text results, no words matched + if (textResultIds && !textResultIds.has(entity.id)) { + return [] + } + + // Slow path: extract text and check each word + // Only happens for entities that DID match text search + const textContent = this.metadataIndex.extractTextContent({ + data: entity.data, + metadata: entity.metadata + }).toLowerCase() + + return queryWords.filter(word => textContent.includes(word.toLowerCase())) + } + /** * Apply graph constraints using O(1) GraphAdjacencyIndex - TRUE Triple Intelligence! */ diff --git a/src/types/brainy.types.ts b/src/types/brainy.types.ts index e4a65bc1..53ac4fd4 100644 --- a/src/types/brainy.types.ts +++ b/src/types/brainy.types.ts @@ -82,6 +82,12 @@ export interface Result { // Score transparency explanation?: ScoreExplanation + + // Match visibility (v7.8.0) - shows what matched in hybrid search + textMatches?: string[] // Query words found in entity (e.g., ["david", "warrior"]) + textScore?: number // Normalized text match score (0-1) + semanticScore?: number // Semantic similarity score (0-1) + matchSource?: 'text' | 'semantic' | 'both' // Where this result came from } /** @@ -192,6 +198,10 @@ export interface FindParams { includeRelations?: boolean // Include entity relationships excludeVFS?: boolean // v4.7.0: Exclude VFS entities from results (default: false - VFS included) service?: string // Multi-tenancy filter + + // Hybrid search options (v7.7.0) + searchMode?: 'auto' | 'text' | 'semantic' | 'vector' | 'hybrid' // Search strategy: auto (default), text-only, semantic/vector-only, or explicit hybrid + hybridAlpha?: number // Weight between text (0.0) and semantic (1.0) search, default: auto-detected by query length // Triple Intelligence Fusion fusion?: { @@ -223,12 +233,14 @@ export interface GraphConstraints { /** * Search modes */ -export type SearchMode = - | 'auto' // Automatically choose best mode - | 'vector' // Pure vector search +export type SearchMode = + | 'auto' // Automatically choose best mode (default - enables hybrid text+semantic) + | 'vector' // Pure vector search (semantic only) + | 'semantic' // Alias for vector search + | 'text' // Pure text/keyword search | 'metadata' // Pure metadata filtering | 'graph' // Pure graph traversal - | 'hybrid' // Combine all intelligences + | 'hybrid' // Combine all intelligences (explicit hybrid mode) /** * Parameters for similarity search @@ -777,6 +789,62 @@ export interface NeuralAnomalyParams { returnScores?: boolean // Return anomaly scores } +// ============= Semantic Highlighting Types (v7.8.0) ============= + +/** + * Parameters for hybrid highlighting (v7.8.0) + * + * Zero-config highlighting that returns both text (exact) and semantic (concept) matches. + * Perfect for UI highlighting at different levels. + * + * @example + * ```typescript + * const highlights = await brain.highlight({ + * query: "david the warrior", + * text: "David Smith is a brave fighter who battles dragons" + * }) + * // Returns: [ + * // { text: "David", score: 1.0, position: [0, 5], matchType: 'text' }, // Exact match + * // { text: "fighter", score: 0.78, position: [25, 32], matchType: 'semantic' }, // Concept match + * // { text: "battles", score: 0.72, position: [37, 44], matchType: 'semantic' } // Concept match + * // ] + * ``` + */ +export interface HighlightParams { + /** The search query to match against */ + query: string + + /** The text to highlight (e.g., entity.data) */ + text: string + + /** Granularity of highlighting: 'word' (default), 'phrase', or 'sentence' */ + granularity?: 'word' | 'phrase' | 'sentence' + + /** Minimum semantic similarity score for semantic matches (default: 0.5) */ + threshold?: number +} + +/** + * A highlight showing which text matched the query + * + * matchType tells the UI how to style the highlight: + * - 'text': Exact word match (strongest signal, highest confidence) + * - 'semantic': Conceptually similar match (may need softer highlight) + */ +export interface Highlight { + /** The text that matched */ + text: string + + /** Match score (0-1). For text matches, always 1.0. For semantic, varies by similarity. */ + score: number + + /** Position in original text [start, end] */ + position: [number, number] + + /** Match type: 'text' (exact word match) or 'semantic' (concept match) */ + matchType: 'text' | 'semantic' +} + // ============= Export all types ============= export * from './graphTypes.js' // Re-export NounType, VerbType, etc. \ No newline at end of file diff --git a/src/utils/metadataIndex.ts b/src/utils/metadataIndex.ts index a49c78f3..2942f8dc 100644 --- a/src/utils/metadataIndex.ts +++ b/src/utils/metadataIndex.ts @@ -1222,9 +1222,156 @@ export class MetadataIndexManager { extract(data) } + // v7.7.0: Extract words for hybrid text search + // v7.8.0: Production-scale word limit (5000 words) + // - Handles articles, chapters, and large documents + // - Roaring Bitmaps + Chunked Sparse Index + LRU caching + // - Int32 hashes store words as 4-byte values, not strings + // + // Memory managed by existing optimizations: + // - Roaring Bitmaps: 90%+ compression for sparse data + // - Chunked Sparse Index: ~50 values per chunk, lazy-loaded + // - UnifiedCache LRU: Only hot chunks in memory + // + // Future: Bloom filter hybrid for unlimited words (see .strategy/BILLION-SCALE-PLAN.md) + const textContent = this.extractTextContent(data) + if (textContent) { + const MAX_WORDS_PER_ENTITY = 5000 // Handles articles/chapters, memory-safe at scale + const allWords = this.tokenize(textContent) + const words = allWords.slice(0, MAX_WORDS_PER_ENTITY) + + if (allWords.length > MAX_WORDS_PER_ENTITY) { + // Log once per entity, not per word - avoids log spam + prodLog.debug( + `Entity text has ${allWords.length} words, indexing first ${MAX_WORDS_PER_ENTITY} for hybrid search` + ) + } + + for (const word of words) { + // Hash word to int32 for memory efficiency (saves ~10GB at 1B scale) + const wordHash = this.hashWord(word) + fields.push({ field: '__words__', value: wordHash }) + } + } + return fields } + /** + * Extract text content from entity data for word indexing (v7.7.0) + * + * Recursively extracts string values from data, excluding: + * - vector, embedding, connections, level, id (internal fields) + * - Arrays with more than 10 elements (likely vectors/bulk data) + * - Numeric-only keys (array indices) + * + * @param data - Entity data or metadata + * @returns Concatenated text content + */ + extractTextContent(data: any): string { + if (data === null || data === undefined) return '' + if (typeof data === 'string') return data + if (typeof data === 'number' || typeof data === 'boolean') return String(data) + if (Array.isArray(data)) { + // Skip large arrays (likely vectors) + if (data.length > 10) return '' + return data.map(d => this.extractTextContent(d)).filter(Boolean).join(' ') + } + if (typeof data === 'object') { + const skipKeys = new Set(['vector', 'embedding', 'embeddings', 'connections', 'level', 'id']) + const texts: string[] = [] + for (const [key, value] of Object.entries(data)) { + // Skip internal fields and numeric keys (array indices) + if (skipKeys.has(key) || /^\d+$/.test(key)) continue + const text = this.extractTextContent(value) + if (text) texts.push(text) + } + return texts.join(' ') + } + return '' + } + + /** + * Tokenize text into words for indexing (v7.7.0) + * + * - Converts to lowercase + * - Removes punctuation + * - Splits on whitespace + * - Filters by length (2-50 chars) + * - Deduplicates per entity + * + * @param text - Text content to tokenize + * @returns Array of unique words + */ + tokenize(text: string): string[] { + if (!text) return [] + return text + .toLowerCase() + .replace(/[^\w\s]/g, ' ') // Remove punctuation + .split(/\s+/) // Split on whitespace + .filter(w => w.length >= 2 && w.length <= 50) // Length filter + .filter((w, i, arr) => arr.indexOf(w) === i) // Dedupe per entity + } + + /** + * Hash word to int32 using FNV-1a (v7.7.0) + * + * FNV-1a is fast with low collision rate, suitable for word hashing. + * Saves ~10GB at billion scale by avoiding string storage. + * + * @param word - Word to hash + * @returns Int32 hash value + */ + hashWord(word: string): number { + let hash = 2166136261 // FNV offset basis + for (let i = 0; i < word.length; i++) { + hash ^= word.charCodeAt(i) + hash = Math.imul(hash, 16777619) // FNV prime + } + return hash | 0 // Convert to signed int32 + } + + /** + * Get entity IDs matching a text query (v7.7.0) + * + * Performs word-based text search using the __words__ index. + * Returns IDs ranked by match count (entities with more matching words first). + * + * @param query - Text query to search for + * @returns Array of { id, matchCount } sorted by matchCount descending + */ + async getIdsForTextQuery(query: string): Promise> { + const queryWords = this.tokenize(query) + if (queryWords.length === 0) return [] + + // Get IDs for each word hash + const wordIdSets: Map[] = [] + for (const word of queryWords) { + const wordHash = this.hashWord(word) + const ids = await this.getIds('__words__', wordHash) + const idSet = new Map() + for (const id of ids) { + idSet.set(id, 1) + } + wordIdSets.push(idSet) + } + + if (wordIdSets.length === 0) return [] + + // Count matches per entity + const matchCounts = new Map() + for (const idSet of wordIdSets) { + for (const [id] of idSet) { + matchCounts.set(id, (matchCounts.get(id) || 0) + 1) + } + } + + // Sort by match count descending + return Array.from(matchCounts.entries()) + .map(([id, matchCount]) => ({ id, matchCount })) + .sort((a, b) => b.matchCount - a.matchCount) + } + /** * Add item to metadata indexes * @@ -1240,13 +1387,24 @@ export class MetadataIndexManager { const fields = this.extractIndexableFields(entityOrMetadata) // v6.7.0: Sanity check for excessive indexed fields (indicates possible data issue) - if (fields.length > 100) { + // v7.8.0: Separate threshold for metadata fields vs word fields + // - Metadata fields: warn if > 100 (indicates deeply nested metadata) + // - Word fields: expected to be many for large documents, warn only for extreme cases + const metadataFields = fields.filter(f => f.field !== '__words__') + const wordFields = fields.filter(f => f.field === '__words__') + + if (metadataFields.length > 100) { prodLog.warn( - `Entity ${id} has ${fields.length} indexed fields (expected ~30). ` + - `Possible deeply nested metadata or data issue. First 10 fields: ${fields.slice(0, 10).map(f => f.field).join(', ')}` + `Entity ${id} has ${metadataFields.length} metadata fields (expected ~30). ` + + `Possible deeply nested metadata. First 10 fields: ${metadataFields.slice(0, 10).map(f => f.field).join(', ')}` ) } + // Words are expected to be many for large documents - only log for extreme cases + if (wordFields.length > 5000) { + prodLog.debug(`Entity ${id} has ${wordFields.length} indexed words (large document)`) + } + // Sort fields to process 'noun' field first for type-field affinity tracking fields.sort((a, b) => { if (a.field === 'noun') return -1 diff --git a/tests/integration/hybrid-search-cow.test.ts b/tests/integration/hybrid-search-cow.test.ts new file mode 100644 index 00000000..c0118e61 --- /dev/null +++ b/tests/integration/hybrid-search-cow.test.ts @@ -0,0 +1,150 @@ +/** + * Hybrid Search COW Integration Tests (v7.7.0) + * + * Verifies that hybrid search works correctly with: + * - fork() - COW branching + * - asOf() - Historical queries + * - VFS entities + */ + +import { describe, it, expect, beforeEach, afterEach } from 'vitest' +import { Brainy } from '../../src/brainy' +import { NounType } from '../../src/types/graphTypes' +import * as fs from 'fs' +import * as path from 'path' +import * as os from 'os' + +describe('Hybrid Search with COW (v7.7.0)', () => { + let brain: Brainy + let testDir: string + + beforeEach(async () => { + // Use filesystem storage for COW support + testDir = path.join(os.tmpdir(), `brainy-cow-test-${Date.now()}`) + fs.mkdirSync(testDir, { recursive: true }) + + brain = new Brainy({ + storage: { + type: 'filesystem', + options: { basePath: testDir } + } + }) + await brain.init() + }) + + afterEach(async () => { + await brain.close() + // Cleanup test directory + try { + fs.rmSync(testDir, { recursive: true, force: true }) + } catch (e) { + // Ignore cleanup errors + } + }) + + describe('fork() compatibility', () => { + it('should perform hybrid search in forked brain', async () => { + // Add entity to main branch + const mainId = await brain.add({ + data: 'Python programming language tutorial', + type: NounType.Document, + metadata: { branch: 'main' } + }) + + // Commit main branch + await brain.commit({ message: 'Add Python doc' }) + + // Fork + const fork = await brain.fork('feature-branch') + + // Add entity to fork + const forkId = await fork.add({ + data: 'JavaScript programming guide', + type: NounType.Document, + metadata: { branch: 'feature' } + }) + + // Hybrid search in fork should find both + const forkResults = await fork.find({ + query: 'programming', + limit: 10 + }) + + expect(forkResults.length).toBeGreaterThanOrEqual(2) + expect(forkResults.some(r => r.id === mainId)).toBe(true) + expect(forkResults.some(r => r.id === forkId)).toBe(true) + + // Hybrid search in main should only find main entity + const mainResults = await brain.find({ + query: 'programming', + limit: 10 + }) + + expect(mainResults.some(r => r.id === mainId)).toBe(true) + // Fork entity should NOT be visible in main + expect(mainResults.some(r => r.id === forkId)).toBe(false) + + await fork.close() + }) + + it('should support text-only search in forked brain', async () => { + await brain.add({ + data: 'exact keyword match test', + type: NounType.Document, + metadata: { test: true } + }) + await brain.commit({ message: 'Add test doc' }) + + const fork = await brain.fork('text-search-test') + + const results = await fork.find({ + query: 'exact keyword', + searchMode: 'text', + limit: 5 + }) + + expect(results.length).toBeGreaterThan(0) + await fork.close() + }) + }) + + describe('VFS compatibility', () => { + it('should include VFS file content in hybrid search', async () => { + // Write a file via VFS + const vfs = brain.vfs + await vfs.writeFile('/docs/readme.txt', 'This is a readme file with important information') + + // Search should find VFS content + const results = await brain.find({ + query: 'readme important', + limit: 10 + }) + + // VFS entities should appear in results + expect(results.length).toBeGreaterThan(0) + }) + + it('should exclude VFS with excludeVFS flag in hybrid search', async () => { + // Add regular entity + const entityId = await brain.add({ + data: 'regular document about readme files', + type: NounType.Document, + metadata: { source: 'api' } + }) + + // Write VFS file + await brain.vfs.writeFile('/readme.md', 'VFS readme content') + + // Search with excludeVFS should only find regular entity + const results = await brain.find({ + query: 'readme', + excludeVFS: true, + limit: 10 + }) + + expect(results.some(r => r.id === entityId)).toBe(true) + // VFS entities should be excluded + expect(results.every(r => !r.metadata?.vfsType)).toBe(true) + }) + }) +}) diff --git a/tests/unit/brainy/hybrid-search.test.ts b/tests/unit/brainy/hybrid-search.test.ts new file mode 100644 index 00000000..96fdbd2c --- /dev/null +++ b/tests/unit/brainy/hybrid-search.test.ts @@ -0,0 +1,741 @@ +/** + * Hybrid Search Tests (v7.7.0) + * + * Tests for zero-config hybrid search combining semantic (vector) + text (keyword) matching + * using Reciprocal Rank Fusion (RRF). + */ + +import { describe, it, expect, beforeEach, afterEach } from 'vitest' +import { Brainy } from '../../../src/brainy' +import { createAddParams } from '../../helpers/test-factory' +import { NounType } from '../../../src/types/graphTypes' + +describe('Hybrid Search (v7.7.0)', () => { + let brain: Brainy + + beforeEach(async () => { + brain = new Brainy({ + storage: { type: 'memory' } + }) + await brain.init() + }) + + afterEach(async () => { + await brain.close() + }) + + describe('Zero-Config Hybrid (default mode)', () => { + it('should return results combining text and semantic matches', async () => { + // Add entities with specific text content + const davidId = await brain.add(createAddParams({ + data: 'David Smith is a software engineer at Google', + type: NounType.Person, + metadata: { name: 'David Smith', role: 'engineer' } + })) + + const johnId = await brain.add(createAddParams({ + data: 'John Doe works as a data scientist', + type: NounType.Person, + metadata: { name: 'John Doe', role: 'scientist' } + })) + + const janeId = await brain.add(createAddParams({ + data: 'Jane Miller is a product manager', + type: NounType.Person, + metadata: { name: 'Jane Miller', role: 'manager' } + })) + + // Search for "David Smith" - should find exact text match + const results = await brain.find({ + query: 'David Smith' + }) + + expect(results.length).toBeGreaterThan(0) + expect(results[0].id).toBe(davidId) + }) + + it('should favor text matches for short queries', async () => { + // Add entities with similar semantic meaning but different exact text + const exactMatch = await brain.add(createAddParams({ + data: 'Python programming language tutorial', + type: NounType.Document, + metadata: { title: 'Python Tutorial' } + })) + + await brain.add(createAddParams({ + data: 'JavaScript coding guide for beginners', + type: NounType.Document, + metadata: { title: 'JS Guide' } + })) + + // Short query "Python" should favor text match + const results = await brain.find({ + query: 'Python', + limit: 5 + }) + + expect(results.length).toBeGreaterThan(0) + // The exact text match for "Python" should rank high + const pythonResult = results.find(r => r.id === exactMatch) + expect(pythonResult).toBeDefined() + }) + + it('should favor semantic matches for long queries', async () => { + // Add entities + await brain.add(createAddParams({ + data: 'Machine learning algorithms and neural networks', + type: NounType.Document, + metadata: { category: 'AI' } + })) + + await brain.add(createAddParams({ + data: 'Deep learning frameworks and artificial intelligence research', + type: NounType.Document, + metadata: { category: 'AI' } + })) + + // Long query should use semantic understanding + const results = await brain.find({ + query: 'advanced artificial intelligence and machine learning techniques for data analysis', + limit: 5 + }) + + expect(results.length).toBeGreaterThan(0) + // Both AI-related documents should appear in results + expect(results.some(r => r.metadata?.category === 'AI')).toBe(true) + }) + }) + + describe('Search Mode Override', () => { + it('should use text-only search when searchMode is "text"', async () => { + const exactId = await brain.add(createAddParams({ + data: 'JavaScript programming', + metadata: { exact: true } + })) + + await brain.add(createAddParams({ + data: 'coding and software development', + metadata: { exact: false } + })) + + // Force text-only search + const results = await brain.find({ + query: 'JavaScript', + searchMode: 'text', + limit: 5 + }) + + expect(results.length).toBeGreaterThan(0) + // Should find exact text match + expect(results[0].id).toBe(exactId) + }) + + it('should use semantic-only search when searchMode is "semantic"', async () => { + const aiId = await brain.add(createAddParams({ + data: 'machine learning algorithms neural networks deep learning', + metadata: { category: 'AI' } + })) + + await brain.add(createAddParams({ + data: 'cooking recipes and food preparation kitchen', + metadata: { category: 'food' } + })) + + // Force semantic-only search + const results = await brain.find({ + query: 'machine learning neural networks', + searchMode: 'semantic', + limit: 5 + }) + + expect(results.length).toBeGreaterThan(0) + // Should return results using semantic search path + // (Note: exact ranking depends on embedding model) + expect(results.some(r => r.id === aiId)).toBe(true) + }) + + it('should accept "vector" as alias for semantic', async () => { + await brain.add(createAddParams({ + data: 'machine learning algorithms', + metadata: { category: 'AI' } + })) + + const results = await brain.find({ + query: 'artificial intelligence', + searchMode: 'vector' as any, // Testing alias + limit: 5 + }) + + expect(results.length).toBeGreaterThan(0) + }) + }) + + describe('Hybrid Alpha Configuration', () => { + it('should use custom hybridAlpha when provided', async () => { + await brain.add(createAddParams({ + data: 'exact text match test', + metadata: { type: 'text' } + })) + + await brain.add(createAddParams({ + data: 'similar semantic content', + metadata: { type: 'semantic' } + })) + + // Force more weight on text (alpha = 0.1) + const textWeightedResults = await brain.find({ + query: 'exact text match test', + hybridAlpha: 0.1, // 90% text, 10% semantic + limit: 5 + }) + + expect(textWeightedResults.length).toBeGreaterThan(0) + }) + + it('should auto-detect alpha based on query length', async () => { + await brain.add(createAddParams({ + data: 'test content for verification', + metadata: { test: true } + })) + + // Short query (1-2 words) - should auto-detect alpha ~0.3 + const shortResults = await brain.find({ + query: 'test', + limit: 5 + }) + + // Long query (5+ words) - should auto-detect alpha ~0.7 + const longResults = await brain.find({ + query: 'test content for verification and analysis purposes', + limit: 5 + }) + + // Both should return results + expect(shortResults.length).toBeGreaterThan(0) + expect(longResults.length).toBeGreaterThan(0) + }) + }) + + describe('Word Tokenization', () => { + it('should find matches regardless of case', async () => { + const id = await brain.add(createAddParams({ + data: 'UPPERCASE TEXT and lowercase text', + metadata: { mixed: true } + })) + + const results = await brain.find({ + query: 'uppercase', + searchMode: 'text', + limit: 5 + }) + + expect(results.length).toBeGreaterThan(0) + expect(results[0].id).toBe(id) + }) + + it('should ignore punctuation in text search', async () => { + const id = await brain.add(createAddParams({ + data: 'Hello, World! How are you?', + metadata: { greeting: true } + })) + + const results = await brain.find({ + query: 'hello world', + searchMode: 'text', + limit: 5 + }) + + expect(results.length).toBeGreaterThan(0) + expect(results[0].id).toBe(id) + }) + + it('should handle multi-word searches', async () => { + const id = await brain.add(createAddParams({ + data: 'software engineering best practices for web development', + metadata: { topic: 'engineering' } + })) + + await brain.add(createAddParams({ + data: 'cooking recipes for healthy meals', + metadata: { topic: 'food' } + })) + + // Multi-word search should match entities with more matching words + const results = await brain.find({ + query: 'software engineering web development', + searchMode: 'text', + limit: 5 + }) + + expect(results.length).toBeGreaterThan(0) + expect(results[0].id).toBe(id) + }) + }) + + describe('Integration with Filters', () => { + it('should combine hybrid search with metadata filters', async () => { + const pythonId = await brain.add(createAddParams({ + data: 'Python programming tutorial', + type: NounType.Document, + metadata: { language: 'python' } + })) + + await brain.add(createAddParams({ + data: 'JavaScript programming tutorial', + type: NounType.Document, + metadata: { language: 'javascript' } + })) + + await brain.add(createAddParams({ + data: 'Cooking recipes and kitchen tips', + type: NounType.Document, + metadata: { language: 'english' } + })) + + // Hybrid search + metadata filter - only find python documents + const results = await brain.find({ + query: 'programming tutorial', + where: { language: 'python' }, + limit: 5 + }) + + // Should find only the Python document + expect(results.length).toBeGreaterThanOrEqual(1) + expect(results.some(r => r.id === pythonId)).toBe(true) + expect(results.every(r => r.metadata?.language === 'python')).toBe(true) + }) + + it('should combine hybrid search with type filters', async () => { + await brain.add(createAddParams({ + data: 'John Smith person profile', + type: NounType.Person, + metadata: { name: 'John Smith' } + })) + + const docId = await brain.add(createAddParams({ + data: 'John Smith document reference', + type: NounType.Document, + metadata: { author: 'John Smith' } + })) + + // Search for "John Smith" but only documents + const results = await brain.find({ + query: 'John Smith', + type: NounType.Document, + limit: 5 + }) + + expect(results.length).toBe(1) + expect(results[0].id).toBe(docId) + }) + }) + + describe('RRF Fusion Algorithm', () => { + it('should rank entities appearing in both text and semantic results higher', async () => { + // Entity that should match both text and semantic + const bothMatchId = await brain.add(createAddParams({ + data: 'machine learning artificial intelligence neural networks', + metadata: { relevance: 'high' } + })) + + // Entity that matches semantically but not exact text + await brain.add(createAddParams({ + data: 'deep learning and AI algorithms', + metadata: { relevance: 'medium' } + })) + + // Entity that matches text but less semantically + await brain.add(createAddParams({ + data: 'machine learning is a buzzword', + metadata: { relevance: 'low' } + })) + + const results = await brain.find({ + query: 'machine learning', + limit: 10 + }) + + expect(results.length).toBeGreaterThan(0) + // The entity that matches both should rank high + const topResult = results.find(r => r.id === bothMatchId) + expect(topResult).toBeDefined() + }) + }) + + describe('Edge Cases', () => { + it('should handle empty query gracefully', async () => { + await brain.add(createAddParams({ + data: 'test entity', + metadata: {} + })) + + const results = await brain.find({ + query: '', + limit: 5 + }) + + // Empty query should return results (metadata-only search) + expect(results.length).toBeGreaterThanOrEqual(0) + }) + + it('should handle query with no text matches', async () => { + await brain.add(createAddParams({ + data: 'completely unrelated content', + metadata: {} + })) + + // Query with words not in any entity + const results = await brain.find({ + query: 'xyzabc123', + limit: 5 + }) + + // Should still return semantic results even if no text matches + expect(results).toBeDefined() + }) + + it('should handle special characters in query', async () => { + await brain.add(createAddParams({ + data: 'C++ programming guide', + metadata: {} + })) + + const results = await brain.find({ + query: 'C++ programming', + limit: 5 + }) + + expect(results).toBeDefined() + }) + + it('should handle very short single-character queries', async () => { + await brain.add(createAddParams({ + data: 'a b c d e f', + metadata: {} + })) + + const results = await brain.find({ + query: 'a', + limit: 5 + }) + + // Single char queries may not match (min word length is 2) + expect(results).toBeDefined() + }) + }) + + describe('Performance', () => { + it('should complete hybrid search in reasonable time', async () => { + // Add multiple entities + const promises = Array.from({ length: 20 }, (_, i) => + brain.add(createAddParams({ + data: `Test document number ${i} with various content about software engineering`, + metadata: { index: i } + })) + ) + await Promise.all(promises) + + const startTime = Date.now() + const results = await brain.find({ + query: 'software engineering document', + limit: 10 + }) + const duration = Date.now() - startTime + + expect(results.length).toBeGreaterThan(0) + // Should complete within reasonable time (allowing for embedding generation) + expect(duration).toBeLessThan(5000) + }) + }) + + describe('Large Document Support (v7.8.0)', () => { + it('should index documents with 1000+ words without arbitrary limits', async () => { + // Create a document with 1000+ unique words + const words: string[] = [] + for (let i = 0; i < 1100; i++) { + words.push(`word${i}`) + } + const largeText = words.join(' ') + + const id = await brain.add(createAddParams({ + data: largeText, + metadata: { wordCount: words.length } + })) + + // Search for a word in the middle of the document (beyond old 50-word limit) + const results = await brain.find({ + query: 'word500 word999', + searchMode: 'text', + limit: 5 + }) + + expect(results.length).toBeGreaterThan(0) + expect(results[0].id).toBe(id) + }) + + it('should not trigger sanity warnings for word-heavy entities', async () => { + // Create an entity with many words but simple metadata + const manyWords = Array.from({ length: 200 }, (_, i) => `word${i}`).join(' ') + + // This should not trigger any warnings + const id = await brain.add(createAddParams({ + data: manyWords, + metadata: { simple: 'metadata' } + })) + + expect(id).toBeTruthy() + + // Verify we can find it + const results = await brain.find({ + query: 'word150', + searchMode: 'text', + limit: 5 + }) + + expect(results.length).toBeGreaterThan(0) + }) + }) + + describe('Match Visibility (v7.8.0)', () => { + it('should return textMatches showing which query words matched', async () => { + await brain.add(createAddParams({ + data: 'David Smith is a brave warrior who battles dragons', + type: NounType.Person, + metadata: { name: 'David Smith' } + })) + + const results = await brain.find({ + query: 'david warrior', + limit: 5 + }) + + expect(results.length).toBeGreaterThan(0) + // Should show which words from the query matched + expect(results[0].textMatches).toBeDefined() + expect(results[0].textMatches).toContain('david') + expect(results[0].textMatches).toContain('warrior') + }) + + it('should return textScore and semanticScore separately', async () => { + await brain.add(createAddParams({ + data: 'machine learning artificial intelligence neural networks', + metadata: { topic: 'AI' } + })) + + const results = await brain.find({ + query: 'machine learning', + limit: 5 + }) + + expect(results.length).toBeGreaterThan(0) + // Should have separate scores + expect(results[0].textScore).toBeDefined() + expect(typeof results[0].textScore).toBe('number') + expect(results[0].semanticScore).toBeDefined() + expect(typeof results[0].semanticScore).toBe('number') + }) + + it('should return matchSource indicating where result came from', async () => { + // Entity that matches both text and semantic + const bothId = await brain.add(createAddParams({ + data: 'Python programming language tutorial', + metadata: {} + })) + + const results = await brain.find({ + query: 'Python programming', + limit: 10 + }) + + expect(results.length).toBeGreaterThan(0) + const matchingResult = results.find(r => r.id === bothId) + expect(matchingResult).toBeDefined() + expect(matchingResult?.matchSource).toBeDefined() + expect(['text', 'semantic', 'both']).toContain(matchingResult?.matchSource) + }) + + it('should show matchSource as "both" for entities matching text AND semantic', async () => { + const id = await brain.add(createAddParams({ + data: 'JavaScript Node.js programming backend development', + metadata: {} + })) + + const results = await brain.find({ + query: 'JavaScript programming', // Exact words + semantic similarity + limit: 5 + }) + + const result = results.find(r => r.id === id) + expect(result).toBeDefined() + // This should match both text (exact words) and semantic + expect(result?.matchSource).toBe('both') + }) + }) + + describe('Hybrid Highlighting (v7.8.0)', () => { + it('should return both text and semantic matches with matchType', async () => { + const highlights = await brain.highlight({ + query: 'warrior fighter', // "warrior" and "fighter" are query words + text: 'A brave warrior and a skilled soldier fight battles', + granularity: 'word', + threshold: 0.3 + }) + + expect(highlights.length).toBeGreaterThan(0) + + // Should have text matches (exact query words) + const textMatches = highlights.filter(h => h.matchType === 'text') + const semanticMatches = highlights.filter(h => h.matchType === 'semantic') + + // "warrior" and "fighter" should be text matches + const textWords = textMatches.map(h => h.text.toLowerCase()) + expect(textWords).toContain('warrior') + + // Text matches should have score = 1.0 + for (const h of textMatches) { + expect(h.score).toBe(1.0) + } + + // Semantic matches should have score < 1.0 but >= threshold + for (const h of semanticMatches) { + expect(h.score).toBeGreaterThanOrEqual(0.3) + expect(h.score).toBeLessThan(1.0) + } + }) + + it('should highlight semantically similar words', async () => { + const highlights = await brain.highlight({ + query: 'warrior', + text: 'David is a brave soldier who battles enemies', + granularity: 'word', + threshold: 0.3 + }) + + expect(highlights.length).toBeGreaterThan(0) + // Should find semantically similar words like "soldier", "battles" + const highlightedWords = highlights.map(h => h.text.toLowerCase()) + expect(highlightedWords.some(w => ['soldier', 'battles', 'brave', 'enemies', 'david'].includes(w))).toBe(true) + }) + + it('should return scores for each highlight', async () => { + const highlights = await brain.highlight({ + query: 'programming', + text: 'Software engineering code development Python JavaScript', + granularity: 'word', + threshold: 0.3 + }) + + expect(highlights.length).toBeGreaterThan(0) + for (const h of highlights) { + expect(typeof h.score).toBe('number') + expect(h.score).toBeGreaterThanOrEqual(0.3) + expect(h.score).toBeLessThanOrEqual(1) + expect(['text', 'semantic']).toContain(h.matchType) + } + }) + + it('should return positions for each highlight', async () => { + const text = 'Apple banana cherry date' + const highlights = await brain.highlight({ + query: 'fruit', + text, + granularity: 'word', + threshold: 0.2 + }) + + for (const h of highlights) { + expect(h.position).toBeDefined() + expect(h.position.length).toBe(2) + expect(h.position[0]).toBeLessThan(h.position[1]) + // Verify the position matches the text + expect(text.substring(h.position[0], h.position[1])).toBe(h.text) + } + }) + + it('should filter semantic matches by threshold', async () => { + const highlightsLow = await brain.highlight({ + query: 'food', + text: 'Apple banana cherry date elephant forest guitar', + granularity: 'word', + threshold: 0.2 + }) + + const highlightsHigh = await brain.highlight({ + query: 'food', + text: 'Apple banana cherry date elephant forest guitar', + granularity: 'word', + threshold: 0.6 + }) + + // Higher threshold should return fewer or equal results + expect(highlightsHigh.length).toBeLessThanOrEqual(highlightsLow.length) + }) + + it('should support sentence granularity', async () => { + const text = 'The cat sat on the mat. Dogs love to play. Birds fly south in winter.' + const highlights = await brain.highlight({ + query: 'animals pets', + text, + granularity: 'sentence', + threshold: 0.3 + }) + + for (const h of highlights) { + // Each highlight should be a full sentence + expect(h.text).toMatch(/[.!?]$/) + } + }) + + it('should skip stopwords when highlighting words', async () => { + const highlights = await brain.highlight({ + query: 'test', + text: 'the a an is are test important', + granularity: 'word', + threshold: 0.1 + }) + + const highlightedWords = highlights.map(h => h.text.toLowerCase()) + // Should not include common stopwords + expect(highlightedWords).not.toContain('the') + expect(highlightedWords).not.toContain('a') + expect(highlightedWords).not.toContain('an') + expect(highlightedWords).not.toContain('is') + expect(highlightedWords).not.toContain('are') + }) + + it('should handle empty text gracefully', async () => { + const highlights = await brain.highlight({ + query: 'test', + text: '', + granularity: 'word' + }) + + expect(highlights).toEqual([]) + }) + + it('should handle empty query gracefully', async () => { + const highlights = await brain.highlight({ + query: '', + text: 'some text content', + granularity: 'word' + }) + + expect(highlights).toEqual([]) + }) + + it('should prioritize text matches over semantic for same word', async () => { + const highlights = await brain.highlight({ + query: 'programming', + text: 'programming is fun and coding is great', // "programming" is in text + granularity: 'word', + threshold: 0.3 + }) + + // The exact word "programming" should be a text match with score 1.0 + const programmingMatch = highlights.find(h => h.text.toLowerCase() === 'programming') + expect(programmingMatch).toBeDefined() + expect(programmingMatch?.matchType).toBe('text') + expect(programmingMatch?.score).toBe(1.0) + }) + }) +})