feat: add match visibility and semantic highlighting to hybrid search
- Add textMatches, textScore, semanticScore, matchSource to search results - Add highlight() method for zero-config text + semantic highlighting - Increase word indexing limit to 5000 (handles articles/chapters) - Optimize findMatchingWords() with O(1) fast path for semantic-only results - Add production safety limits (500 chunks for highlight) - Add comprehensive tests for new features (35 tests) - Update docs with match visibility and highlight() API
This commit is contained in:
parent
b0ea2f3810
commit
4adba1b254
7 changed files with 1727 additions and 31 deletions
150
tests/integration/hybrid-search-cow.test.ts
Normal file
150
tests/integration/hybrid-search-cow.test.ts
Normal file
|
|
@ -0,0 +1,150 @@
|
|||
/**
|
||||
* Hybrid Search COW Integration Tests (v7.7.0)
|
||||
*
|
||||
* Verifies that hybrid search works correctly with:
|
||||
* - fork() - COW branching
|
||||
* - asOf() - Historical queries
|
||||
* - VFS entities
|
||||
*/
|
||||
|
||||
import { describe, it, expect, beforeEach, afterEach } from 'vitest'
|
||||
import { Brainy } from '../../src/brainy'
|
||||
import { NounType } from '../../src/types/graphTypes'
|
||||
import * as fs from 'fs'
|
||||
import * as path from 'path'
|
||||
import * as os from 'os'
|
||||
|
||||
describe('Hybrid Search with COW (v7.7.0)', () => {
|
||||
let brain: Brainy<any>
|
||||
let testDir: string
|
||||
|
||||
beforeEach(async () => {
|
||||
// Use filesystem storage for COW support
|
||||
testDir = path.join(os.tmpdir(), `brainy-cow-test-${Date.now()}`)
|
||||
fs.mkdirSync(testDir, { recursive: true })
|
||||
|
||||
brain = new Brainy({
|
||||
storage: {
|
||||
type: 'filesystem',
|
||||
options: { basePath: testDir }
|
||||
}
|
||||
})
|
||||
await brain.init()
|
||||
})
|
||||
|
||||
afterEach(async () => {
|
||||
await brain.close()
|
||||
// Cleanup test directory
|
||||
try {
|
||||
fs.rmSync(testDir, { recursive: true, force: true })
|
||||
} catch (e) {
|
||||
// Ignore cleanup errors
|
||||
}
|
||||
})
|
||||
|
||||
describe('fork() compatibility', () => {
|
||||
it('should perform hybrid search in forked brain', async () => {
|
||||
// Add entity to main branch
|
||||
const mainId = await brain.add({
|
||||
data: 'Python programming language tutorial',
|
||||
type: NounType.Document,
|
||||
metadata: { branch: 'main' }
|
||||
})
|
||||
|
||||
// Commit main branch
|
||||
await brain.commit({ message: 'Add Python doc' })
|
||||
|
||||
// Fork
|
||||
const fork = await brain.fork('feature-branch')
|
||||
|
||||
// Add entity to fork
|
||||
const forkId = await fork.add({
|
||||
data: 'JavaScript programming guide',
|
||||
type: NounType.Document,
|
||||
metadata: { branch: 'feature' }
|
||||
})
|
||||
|
||||
// Hybrid search in fork should find both
|
||||
const forkResults = await fork.find({
|
||||
query: 'programming',
|
||||
limit: 10
|
||||
})
|
||||
|
||||
expect(forkResults.length).toBeGreaterThanOrEqual(2)
|
||||
expect(forkResults.some(r => r.id === mainId)).toBe(true)
|
||||
expect(forkResults.some(r => r.id === forkId)).toBe(true)
|
||||
|
||||
// Hybrid search in main should only find main entity
|
||||
const mainResults = await brain.find({
|
||||
query: 'programming',
|
||||
limit: 10
|
||||
})
|
||||
|
||||
expect(mainResults.some(r => r.id === mainId)).toBe(true)
|
||||
// Fork entity should NOT be visible in main
|
||||
expect(mainResults.some(r => r.id === forkId)).toBe(false)
|
||||
|
||||
await fork.close()
|
||||
})
|
||||
|
||||
it('should support text-only search in forked brain', async () => {
|
||||
await brain.add({
|
||||
data: 'exact keyword match test',
|
||||
type: NounType.Document,
|
||||
metadata: { test: true }
|
||||
})
|
||||
await brain.commit({ message: 'Add test doc' })
|
||||
|
||||
const fork = await brain.fork('text-search-test')
|
||||
|
||||
const results = await fork.find({
|
||||
query: 'exact keyword',
|
||||
searchMode: 'text',
|
||||
limit: 5
|
||||
})
|
||||
|
||||
expect(results.length).toBeGreaterThan(0)
|
||||
await fork.close()
|
||||
})
|
||||
})
|
||||
|
||||
describe('VFS compatibility', () => {
|
||||
it('should include VFS file content in hybrid search', async () => {
|
||||
// Write a file via VFS
|
||||
const vfs = brain.vfs
|
||||
await vfs.writeFile('/docs/readme.txt', 'This is a readme file with important information')
|
||||
|
||||
// Search should find VFS content
|
||||
const results = await brain.find({
|
||||
query: 'readme important',
|
||||
limit: 10
|
||||
})
|
||||
|
||||
// VFS entities should appear in results
|
||||
expect(results.length).toBeGreaterThan(0)
|
||||
})
|
||||
|
||||
it('should exclude VFS with excludeVFS flag in hybrid search', async () => {
|
||||
// Add regular entity
|
||||
const entityId = await brain.add({
|
||||
data: 'regular document about readme files',
|
||||
type: NounType.Document,
|
||||
metadata: { source: 'api' }
|
||||
})
|
||||
|
||||
// Write VFS file
|
||||
await brain.vfs.writeFile('/readme.md', 'VFS readme content')
|
||||
|
||||
// Search with excludeVFS should only find regular entity
|
||||
const results = await brain.find({
|
||||
query: 'readme',
|
||||
excludeVFS: true,
|
||||
limit: 10
|
||||
})
|
||||
|
||||
expect(results.some(r => r.id === entityId)).toBe(true)
|
||||
// VFS entities should be excluded
|
||||
expect(results.every(r => !r.metadata?.vfsType)).toBe(true)
|
||||
})
|
||||
})
|
||||
})
|
||||
741
tests/unit/brainy/hybrid-search.test.ts
Normal file
741
tests/unit/brainy/hybrid-search.test.ts
Normal file
|
|
@ -0,0 +1,741 @@
|
|||
/**
|
||||
* Hybrid Search Tests (v7.7.0)
|
||||
*
|
||||
* Tests for zero-config hybrid search combining semantic (vector) + text (keyword) matching
|
||||
* using Reciprocal Rank Fusion (RRF).
|
||||
*/
|
||||
|
||||
import { describe, it, expect, beforeEach, afterEach } from 'vitest'
|
||||
import { Brainy } from '../../../src/brainy'
|
||||
import { createAddParams } from '../../helpers/test-factory'
|
||||
import { NounType } from '../../../src/types/graphTypes'
|
||||
|
||||
describe('Hybrid Search (v7.7.0)', () => {
|
||||
let brain: Brainy<any>
|
||||
|
||||
beforeEach(async () => {
|
||||
brain = new Brainy({
|
||||
storage: { type: 'memory' }
|
||||
})
|
||||
await brain.init()
|
||||
})
|
||||
|
||||
afterEach(async () => {
|
||||
await brain.close()
|
||||
})
|
||||
|
||||
describe('Zero-Config Hybrid (default mode)', () => {
|
||||
it('should return results combining text and semantic matches', async () => {
|
||||
// Add entities with specific text content
|
||||
const davidId = await brain.add(createAddParams({
|
||||
data: 'David Smith is a software engineer at Google',
|
||||
type: NounType.Person,
|
||||
metadata: { name: 'David Smith', role: 'engineer' }
|
||||
}))
|
||||
|
||||
const johnId = await brain.add(createAddParams({
|
||||
data: 'John Doe works as a data scientist',
|
||||
type: NounType.Person,
|
||||
metadata: { name: 'John Doe', role: 'scientist' }
|
||||
}))
|
||||
|
||||
const janeId = await brain.add(createAddParams({
|
||||
data: 'Jane Miller is a product manager',
|
||||
type: NounType.Person,
|
||||
metadata: { name: 'Jane Miller', role: 'manager' }
|
||||
}))
|
||||
|
||||
// Search for "David Smith" - should find exact text match
|
||||
const results = await brain.find({
|
||||
query: 'David Smith'
|
||||
})
|
||||
|
||||
expect(results.length).toBeGreaterThan(0)
|
||||
expect(results[0].id).toBe(davidId)
|
||||
})
|
||||
|
||||
it('should favor text matches for short queries', async () => {
|
||||
// Add entities with similar semantic meaning but different exact text
|
||||
const exactMatch = await brain.add(createAddParams({
|
||||
data: 'Python programming language tutorial',
|
||||
type: NounType.Document,
|
||||
metadata: { title: 'Python Tutorial' }
|
||||
}))
|
||||
|
||||
await brain.add(createAddParams({
|
||||
data: 'JavaScript coding guide for beginners',
|
||||
type: NounType.Document,
|
||||
metadata: { title: 'JS Guide' }
|
||||
}))
|
||||
|
||||
// Short query "Python" should favor text match
|
||||
const results = await brain.find({
|
||||
query: 'Python',
|
||||
limit: 5
|
||||
})
|
||||
|
||||
expect(results.length).toBeGreaterThan(0)
|
||||
// The exact text match for "Python" should rank high
|
||||
const pythonResult = results.find(r => r.id === exactMatch)
|
||||
expect(pythonResult).toBeDefined()
|
||||
})
|
||||
|
||||
it('should favor semantic matches for long queries', async () => {
|
||||
// Add entities
|
||||
await brain.add(createAddParams({
|
||||
data: 'Machine learning algorithms and neural networks',
|
||||
type: NounType.Document,
|
||||
metadata: { category: 'AI' }
|
||||
}))
|
||||
|
||||
await brain.add(createAddParams({
|
||||
data: 'Deep learning frameworks and artificial intelligence research',
|
||||
type: NounType.Document,
|
||||
metadata: { category: 'AI' }
|
||||
}))
|
||||
|
||||
// Long query should use semantic understanding
|
||||
const results = await brain.find({
|
||||
query: 'advanced artificial intelligence and machine learning techniques for data analysis',
|
||||
limit: 5
|
||||
})
|
||||
|
||||
expect(results.length).toBeGreaterThan(0)
|
||||
// Both AI-related documents should appear in results
|
||||
expect(results.some(r => r.metadata?.category === 'AI')).toBe(true)
|
||||
})
|
||||
})
|
||||
|
||||
describe('Search Mode Override', () => {
|
||||
it('should use text-only search when searchMode is "text"', async () => {
|
||||
const exactId = await brain.add(createAddParams({
|
||||
data: 'JavaScript programming',
|
||||
metadata: { exact: true }
|
||||
}))
|
||||
|
||||
await brain.add(createAddParams({
|
||||
data: 'coding and software development',
|
||||
metadata: { exact: false }
|
||||
}))
|
||||
|
||||
// Force text-only search
|
||||
const results = await brain.find({
|
||||
query: 'JavaScript',
|
||||
searchMode: 'text',
|
||||
limit: 5
|
||||
})
|
||||
|
||||
expect(results.length).toBeGreaterThan(0)
|
||||
// Should find exact text match
|
||||
expect(results[0].id).toBe(exactId)
|
||||
})
|
||||
|
||||
it('should use semantic-only search when searchMode is "semantic"', async () => {
|
||||
const aiId = await brain.add(createAddParams({
|
||||
data: 'machine learning algorithms neural networks deep learning',
|
||||
metadata: { category: 'AI' }
|
||||
}))
|
||||
|
||||
await brain.add(createAddParams({
|
||||
data: 'cooking recipes and food preparation kitchen',
|
||||
metadata: { category: 'food' }
|
||||
}))
|
||||
|
||||
// Force semantic-only search
|
||||
const results = await brain.find({
|
||||
query: 'machine learning neural networks',
|
||||
searchMode: 'semantic',
|
||||
limit: 5
|
||||
})
|
||||
|
||||
expect(results.length).toBeGreaterThan(0)
|
||||
// Should return results using semantic search path
|
||||
// (Note: exact ranking depends on embedding model)
|
||||
expect(results.some(r => r.id === aiId)).toBe(true)
|
||||
})
|
||||
|
||||
it('should accept "vector" as alias for semantic', async () => {
|
||||
await brain.add(createAddParams({
|
||||
data: 'machine learning algorithms',
|
||||
metadata: { category: 'AI' }
|
||||
}))
|
||||
|
||||
const results = await brain.find({
|
||||
query: 'artificial intelligence',
|
||||
searchMode: 'vector' as any, // Testing alias
|
||||
limit: 5
|
||||
})
|
||||
|
||||
expect(results.length).toBeGreaterThan(0)
|
||||
})
|
||||
})
|
||||
|
||||
describe('Hybrid Alpha Configuration', () => {
|
||||
it('should use custom hybridAlpha when provided', async () => {
|
||||
await brain.add(createAddParams({
|
||||
data: 'exact text match test',
|
||||
metadata: { type: 'text' }
|
||||
}))
|
||||
|
||||
await brain.add(createAddParams({
|
||||
data: 'similar semantic content',
|
||||
metadata: { type: 'semantic' }
|
||||
}))
|
||||
|
||||
// Force more weight on text (alpha = 0.1)
|
||||
const textWeightedResults = await brain.find({
|
||||
query: 'exact text match test',
|
||||
hybridAlpha: 0.1, // 90% text, 10% semantic
|
||||
limit: 5
|
||||
})
|
||||
|
||||
expect(textWeightedResults.length).toBeGreaterThan(0)
|
||||
})
|
||||
|
||||
it('should auto-detect alpha based on query length', async () => {
|
||||
await brain.add(createAddParams({
|
||||
data: 'test content for verification',
|
||||
metadata: { test: true }
|
||||
}))
|
||||
|
||||
// Short query (1-2 words) - should auto-detect alpha ~0.3
|
||||
const shortResults = await brain.find({
|
||||
query: 'test',
|
||||
limit: 5
|
||||
})
|
||||
|
||||
// Long query (5+ words) - should auto-detect alpha ~0.7
|
||||
const longResults = await brain.find({
|
||||
query: 'test content for verification and analysis purposes',
|
||||
limit: 5
|
||||
})
|
||||
|
||||
// Both should return results
|
||||
expect(shortResults.length).toBeGreaterThan(0)
|
||||
expect(longResults.length).toBeGreaterThan(0)
|
||||
})
|
||||
})
|
||||
|
||||
describe('Word Tokenization', () => {
|
||||
it('should find matches regardless of case', async () => {
|
||||
const id = await brain.add(createAddParams({
|
||||
data: 'UPPERCASE TEXT and lowercase text',
|
||||
metadata: { mixed: true }
|
||||
}))
|
||||
|
||||
const results = await brain.find({
|
||||
query: 'uppercase',
|
||||
searchMode: 'text',
|
||||
limit: 5
|
||||
})
|
||||
|
||||
expect(results.length).toBeGreaterThan(0)
|
||||
expect(results[0].id).toBe(id)
|
||||
})
|
||||
|
||||
it('should ignore punctuation in text search', async () => {
|
||||
const id = await brain.add(createAddParams({
|
||||
data: 'Hello, World! How are you?',
|
||||
metadata: { greeting: true }
|
||||
}))
|
||||
|
||||
const results = await brain.find({
|
||||
query: 'hello world',
|
||||
searchMode: 'text',
|
||||
limit: 5
|
||||
})
|
||||
|
||||
expect(results.length).toBeGreaterThan(0)
|
||||
expect(results[0].id).toBe(id)
|
||||
})
|
||||
|
||||
it('should handle multi-word searches', async () => {
|
||||
const id = await brain.add(createAddParams({
|
||||
data: 'software engineering best practices for web development',
|
||||
metadata: { topic: 'engineering' }
|
||||
}))
|
||||
|
||||
await brain.add(createAddParams({
|
||||
data: 'cooking recipes for healthy meals',
|
||||
metadata: { topic: 'food' }
|
||||
}))
|
||||
|
||||
// Multi-word search should match entities with more matching words
|
||||
const results = await brain.find({
|
||||
query: 'software engineering web development',
|
||||
searchMode: 'text',
|
||||
limit: 5
|
||||
})
|
||||
|
||||
expect(results.length).toBeGreaterThan(0)
|
||||
expect(results[0].id).toBe(id)
|
||||
})
|
||||
})
|
||||
|
||||
describe('Integration with Filters', () => {
|
||||
it('should combine hybrid search with metadata filters', async () => {
|
||||
const pythonId = await brain.add(createAddParams({
|
||||
data: 'Python programming tutorial',
|
||||
type: NounType.Document,
|
||||
metadata: { language: 'python' }
|
||||
}))
|
||||
|
||||
await brain.add(createAddParams({
|
||||
data: 'JavaScript programming tutorial',
|
||||
type: NounType.Document,
|
||||
metadata: { language: 'javascript' }
|
||||
}))
|
||||
|
||||
await brain.add(createAddParams({
|
||||
data: 'Cooking recipes and kitchen tips',
|
||||
type: NounType.Document,
|
||||
metadata: { language: 'english' }
|
||||
}))
|
||||
|
||||
// Hybrid search + metadata filter - only find python documents
|
||||
const results = await brain.find({
|
||||
query: 'programming tutorial',
|
||||
where: { language: 'python' },
|
||||
limit: 5
|
||||
})
|
||||
|
||||
// Should find only the Python document
|
||||
expect(results.length).toBeGreaterThanOrEqual(1)
|
||||
expect(results.some(r => r.id === pythonId)).toBe(true)
|
||||
expect(results.every(r => r.metadata?.language === 'python')).toBe(true)
|
||||
})
|
||||
|
||||
it('should combine hybrid search with type filters', async () => {
|
||||
await brain.add(createAddParams({
|
||||
data: 'John Smith person profile',
|
||||
type: NounType.Person,
|
||||
metadata: { name: 'John Smith' }
|
||||
}))
|
||||
|
||||
const docId = await brain.add(createAddParams({
|
||||
data: 'John Smith document reference',
|
||||
type: NounType.Document,
|
||||
metadata: { author: 'John Smith' }
|
||||
}))
|
||||
|
||||
// Search for "John Smith" but only documents
|
||||
const results = await brain.find({
|
||||
query: 'John Smith',
|
||||
type: NounType.Document,
|
||||
limit: 5
|
||||
})
|
||||
|
||||
expect(results.length).toBe(1)
|
||||
expect(results[0].id).toBe(docId)
|
||||
})
|
||||
})
|
||||
|
||||
describe('RRF Fusion Algorithm', () => {
|
||||
it('should rank entities appearing in both text and semantic results higher', async () => {
|
||||
// Entity that should match both text and semantic
|
||||
const bothMatchId = await brain.add(createAddParams({
|
||||
data: 'machine learning artificial intelligence neural networks',
|
||||
metadata: { relevance: 'high' }
|
||||
}))
|
||||
|
||||
// Entity that matches semantically but not exact text
|
||||
await brain.add(createAddParams({
|
||||
data: 'deep learning and AI algorithms',
|
||||
metadata: { relevance: 'medium' }
|
||||
}))
|
||||
|
||||
// Entity that matches text but less semantically
|
||||
await brain.add(createAddParams({
|
||||
data: 'machine learning is a buzzword',
|
||||
metadata: { relevance: 'low' }
|
||||
}))
|
||||
|
||||
const results = await brain.find({
|
||||
query: 'machine learning',
|
||||
limit: 10
|
||||
})
|
||||
|
||||
expect(results.length).toBeGreaterThan(0)
|
||||
// The entity that matches both should rank high
|
||||
const topResult = results.find(r => r.id === bothMatchId)
|
||||
expect(topResult).toBeDefined()
|
||||
})
|
||||
})
|
||||
|
||||
describe('Edge Cases', () => {
|
||||
it('should handle empty query gracefully', async () => {
|
||||
await brain.add(createAddParams({
|
||||
data: 'test entity',
|
||||
metadata: {}
|
||||
}))
|
||||
|
||||
const results = await brain.find({
|
||||
query: '',
|
||||
limit: 5
|
||||
})
|
||||
|
||||
// Empty query should return results (metadata-only search)
|
||||
expect(results.length).toBeGreaterThanOrEqual(0)
|
||||
})
|
||||
|
||||
it('should handle query with no text matches', async () => {
|
||||
await brain.add(createAddParams({
|
||||
data: 'completely unrelated content',
|
||||
metadata: {}
|
||||
}))
|
||||
|
||||
// Query with words not in any entity
|
||||
const results = await brain.find({
|
||||
query: 'xyzabc123',
|
||||
limit: 5
|
||||
})
|
||||
|
||||
// Should still return semantic results even if no text matches
|
||||
expect(results).toBeDefined()
|
||||
})
|
||||
|
||||
it('should handle special characters in query', async () => {
|
||||
await brain.add(createAddParams({
|
||||
data: 'C++ programming guide',
|
||||
metadata: {}
|
||||
}))
|
||||
|
||||
const results = await brain.find({
|
||||
query: 'C++ programming',
|
||||
limit: 5
|
||||
})
|
||||
|
||||
expect(results).toBeDefined()
|
||||
})
|
||||
|
||||
it('should handle very short single-character queries', async () => {
|
||||
await brain.add(createAddParams({
|
||||
data: 'a b c d e f',
|
||||
metadata: {}
|
||||
}))
|
||||
|
||||
const results = await brain.find({
|
||||
query: 'a',
|
||||
limit: 5
|
||||
})
|
||||
|
||||
// Single char queries may not match (min word length is 2)
|
||||
expect(results).toBeDefined()
|
||||
})
|
||||
})
|
||||
|
||||
describe('Performance', () => {
|
||||
it('should complete hybrid search in reasonable time', async () => {
|
||||
// Add multiple entities
|
||||
const promises = Array.from({ length: 20 }, (_, i) =>
|
||||
brain.add(createAddParams({
|
||||
data: `Test document number ${i} with various content about software engineering`,
|
||||
metadata: { index: i }
|
||||
}))
|
||||
)
|
||||
await Promise.all(promises)
|
||||
|
||||
const startTime = Date.now()
|
||||
const results = await brain.find({
|
||||
query: 'software engineering document',
|
||||
limit: 10
|
||||
})
|
||||
const duration = Date.now() - startTime
|
||||
|
||||
expect(results.length).toBeGreaterThan(0)
|
||||
// Should complete within reasonable time (allowing for embedding generation)
|
||||
expect(duration).toBeLessThan(5000)
|
||||
})
|
||||
})
|
||||
|
||||
describe('Large Document Support (v7.8.0)', () => {
|
||||
it('should index documents with 1000+ words without arbitrary limits', async () => {
|
||||
// Create a document with 1000+ unique words
|
||||
const words: string[] = []
|
||||
for (let i = 0; i < 1100; i++) {
|
||||
words.push(`word${i}`)
|
||||
}
|
||||
const largeText = words.join(' ')
|
||||
|
||||
const id = await brain.add(createAddParams({
|
||||
data: largeText,
|
||||
metadata: { wordCount: words.length }
|
||||
}))
|
||||
|
||||
// Search for a word in the middle of the document (beyond old 50-word limit)
|
||||
const results = await brain.find({
|
||||
query: 'word500 word999',
|
||||
searchMode: 'text',
|
||||
limit: 5
|
||||
})
|
||||
|
||||
expect(results.length).toBeGreaterThan(0)
|
||||
expect(results[0].id).toBe(id)
|
||||
})
|
||||
|
||||
it('should not trigger sanity warnings for word-heavy entities', async () => {
|
||||
// Create an entity with many words but simple metadata
|
||||
const manyWords = Array.from({ length: 200 }, (_, i) => `word${i}`).join(' ')
|
||||
|
||||
// This should not trigger any warnings
|
||||
const id = await brain.add(createAddParams({
|
||||
data: manyWords,
|
||||
metadata: { simple: 'metadata' }
|
||||
}))
|
||||
|
||||
expect(id).toBeTruthy()
|
||||
|
||||
// Verify we can find it
|
||||
const results = await brain.find({
|
||||
query: 'word150',
|
||||
searchMode: 'text',
|
||||
limit: 5
|
||||
})
|
||||
|
||||
expect(results.length).toBeGreaterThan(0)
|
||||
})
|
||||
})
|
||||
|
||||
describe('Match Visibility (v7.8.0)', () => {
|
||||
it('should return textMatches showing which query words matched', async () => {
|
||||
await brain.add(createAddParams({
|
||||
data: 'David Smith is a brave warrior who battles dragons',
|
||||
type: NounType.Person,
|
||||
metadata: { name: 'David Smith' }
|
||||
}))
|
||||
|
||||
const results = await brain.find({
|
||||
query: 'david warrior',
|
||||
limit: 5
|
||||
})
|
||||
|
||||
expect(results.length).toBeGreaterThan(0)
|
||||
// Should show which words from the query matched
|
||||
expect(results[0].textMatches).toBeDefined()
|
||||
expect(results[0].textMatches).toContain('david')
|
||||
expect(results[0].textMatches).toContain('warrior')
|
||||
})
|
||||
|
||||
it('should return textScore and semanticScore separately', async () => {
|
||||
await brain.add(createAddParams({
|
||||
data: 'machine learning artificial intelligence neural networks',
|
||||
metadata: { topic: 'AI' }
|
||||
}))
|
||||
|
||||
const results = await brain.find({
|
||||
query: 'machine learning',
|
||||
limit: 5
|
||||
})
|
||||
|
||||
expect(results.length).toBeGreaterThan(0)
|
||||
// Should have separate scores
|
||||
expect(results[0].textScore).toBeDefined()
|
||||
expect(typeof results[0].textScore).toBe('number')
|
||||
expect(results[0].semanticScore).toBeDefined()
|
||||
expect(typeof results[0].semanticScore).toBe('number')
|
||||
})
|
||||
|
||||
it('should return matchSource indicating where result came from', async () => {
|
||||
// Entity that matches both text and semantic
|
||||
const bothId = await brain.add(createAddParams({
|
||||
data: 'Python programming language tutorial',
|
||||
metadata: {}
|
||||
}))
|
||||
|
||||
const results = await brain.find({
|
||||
query: 'Python programming',
|
||||
limit: 10
|
||||
})
|
||||
|
||||
expect(results.length).toBeGreaterThan(0)
|
||||
const matchingResult = results.find(r => r.id === bothId)
|
||||
expect(matchingResult).toBeDefined()
|
||||
expect(matchingResult?.matchSource).toBeDefined()
|
||||
expect(['text', 'semantic', 'both']).toContain(matchingResult?.matchSource)
|
||||
})
|
||||
|
||||
it('should show matchSource as "both" for entities matching text AND semantic', async () => {
|
||||
const id = await brain.add(createAddParams({
|
||||
data: 'JavaScript Node.js programming backend development',
|
||||
metadata: {}
|
||||
}))
|
||||
|
||||
const results = await brain.find({
|
||||
query: 'JavaScript programming', // Exact words + semantic similarity
|
||||
limit: 5
|
||||
})
|
||||
|
||||
const result = results.find(r => r.id === id)
|
||||
expect(result).toBeDefined()
|
||||
// This should match both text (exact words) and semantic
|
||||
expect(result?.matchSource).toBe('both')
|
||||
})
|
||||
})
|
||||
|
||||
describe('Hybrid Highlighting (v7.8.0)', () => {
|
||||
it('should return both text and semantic matches with matchType', async () => {
|
||||
const highlights = await brain.highlight({
|
||||
query: 'warrior fighter', // "warrior" and "fighter" are query words
|
||||
text: 'A brave warrior and a skilled soldier fight battles',
|
||||
granularity: 'word',
|
||||
threshold: 0.3
|
||||
})
|
||||
|
||||
expect(highlights.length).toBeGreaterThan(0)
|
||||
|
||||
// Should have text matches (exact query words)
|
||||
const textMatches = highlights.filter(h => h.matchType === 'text')
|
||||
const semanticMatches = highlights.filter(h => h.matchType === 'semantic')
|
||||
|
||||
// "warrior" and "fighter" should be text matches
|
||||
const textWords = textMatches.map(h => h.text.toLowerCase())
|
||||
expect(textWords).toContain('warrior')
|
||||
|
||||
// Text matches should have score = 1.0
|
||||
for (const h of textMatches) {
|
||||
expect(h.score).toBe(1.0)
|
||||
}
|
||||
|
||||
// Semantic matches should have score < 1.0 but >= threshold
|
||||
for (const h of semanticMatches) {
|
||||
expect(h.score).toBeGreaterThanOrEqual(0.3)
|
||||
expect(h.score).toBeLessThan(1.0)
|
||||
}
|
||||
})
|
||||
|
||||
it('should highlight semantically similar words', async () => {
|
||||
const highlights = await brain.highlight({
|
||||
query: 'warrior',
|
||||
text: 'David is a brave soldier who battles enemies',
|
||||
granularity: 'word',
|
||||
threshold: 0.3
|
||||
})
|
||||
|
||||
expect(highlights.length).toBeGreaterThan(0)
|
||||
// Should find semantically similar words like "soldier", "battles"
|
||||
const highlightedWords = highlights.map(h => h.text.toLowerCase())
|
||||
expect(highlightedWords.some(w => ['soldier', 'battles', 'brave', 'enemies', 'david'].includes(w))).toBe(true)
|
||||
})
|
||||
|
||||
it('should return scores for each highlight', async () => {
|
||||
const highlights = await brain.highlight({
|
||||
query: 'programming',
|
||||
text: 'Software engineering code development Python JavaScript',
|
||||
granularity: 'word',
|
||||
threshold: 0.3
|
||||
})
|
||||
|
||||
expect(highlights.length).toBeGreaterThan(0)
|
||||
for (const h of highlights) {
|
||||
expect(typeof h.score).toBe('number')
|
||||
expect(h.score).toBeGreaterThanOrEqual(0.3)
|
||||
expect(h.score).toBeLessThanOrEqual(1)
|
||||
expect(['text', 'semantic']).toContain(h.matchType)
|
||||
}
|
||||
})
|
||||
|
||||
it('should return positions for each highlight', async () => {
|
||||
const text = 'Apple banana cherry date'
|
||||
const highlights = await brain.highlight({
|
||||
query: 'fruit',
|
||||
text,
|
||||
granularity: 'word',
|
||||
threshold: 0.2
|
||||
})
|
||||
|
||||
for (const h of highlights) {
|
||||
expect(h.position).toBeDefined()
|
||||
expect(h.position.length).toBe(2)
|
||||
expect(h.position[0]).toBeLessThan(h.position[1])
|
||||
// Verify the position matches the text
|
||||
expect(text.substring(h.position[0], h.position[1])).toBe(h.text)
|
||||
}
|
||||
})
|
||||
|
||||
it('should filter semantic matches by threshold', async () => {
|
||||
const highlightsLow = await brain.highlight({
|
||||
query: 'food',
|
||||
text: 'Apple banana cherry date elephant forest guitar',
|
||||
granularity: 'word',
|
||||
threshold: 0.2
|
||||
})
|
||||
|
||||
const highlightsHigh = await brain.highlight({
|
||||
query: 'food',
|
||||
text: 'Apple banana cherry date elephant forest guitar',
|
||||
granularity: 'word',
|
||||
threshold: 0.6
|
||||
})
|
||||
|
||||
// Higher threshold should return fewer or equal results
|
||||
expect(highlightsHigh.length).toBeLessThanOrEqual(highlightsLow.length)
|
||||
})
|
||||
|
||||
it('should support sentence granularity', async () => {
|
||||
const text = 'The cat sat on the mat. Dogs love to play. Birds fly south in winter.'
|
||||
const highlights = await brain.highlight({
|
||||
query: 'animals pets',
|
||||
text,
|
||||
granularity: 'sentence',
|
||||
threshold: 0.3
|
||||
})
|
||||
|
||||
for (const h of highlights) {
|
||||
// Each highlight should be a full sentence
|
||||
expect(h.text).toMatch(/[.!?]$/)
|
||||
}
|
||||
})
|
||||
|
||||
it('should skip stopwords when highlighting words', async () => {
|
||||
const highlights = await brain.highlight({
|
||||
query: 'test',
|
||||
text: 'the a an is are test important',
|
||||
granularity: 'word',
|
||||
threshold: 0.1
|
||||
})
|
||||
|
||||
const highlightedWords = highlights.map(h => h.text.toLowerCase())
|
||||
// Should not include common stopwords
|
||||
expect(highlightedWords).not.toContain('the')
|
||||
expect(highlightedWords).not.toContain('a')
|
||||
expect(highlightedWords).not.toContain('an')
|
||||
expect(highlightedWords).not.toContain('is')
|
||||
expect(highlightedWords).not.toContain('are')
|
||||
})
|
||||
|
||||
it('should handle empty text gracefully', async () => {
|
||||
const highlights = await brain.highlight({
|
||||
query: 'test',
|
||||
text: '',
|
||||
granularity: 'word'
|
||||
})
|
||||
|
||||
expect(highlights).toEqual([])
|
||||
})
|
||||
|
||||
it('should handle empty query gracefully', async () => {
|
||||
const highlights = await brain.highlight({
|
||||
query: '',
|
||||
text: 'some text content',
|
||||
granularity: 'word'
|
||||
})
|
||||
|
||||
expect(highlights).toEqual([])
|
||||
})
|
||||
|
||||
it('should prioritize text matches over semantic for same word', async () => {
|
||||
const highlights = await brain.highlight({
|
||||
query: 'programming',
|
||||
text: 'programming is fun and coding is great', // "programming" is in text
|
||||
granularity: 'word',
|
||||
threshold: 0.3
|
||||
})
|
||||
|
||||
// The exact word "programming" should be a text match with score 1.0
|
||||
const programmingMatch = highlights.find(h => h.text.toLowerCase() === 'programming')
|
||||
expect(programmingMatch).toBeDefined()
|
||||
expect(programmingMatch?.matchType).toBe('text')
|
||||
expect(programmingMatch?.score).toBe(1.0)
|
||||
})
|
||||
})
|
||||
})
|
||||
Loading…
Add table
Add a link
Reference in a new issue