docs: update architecture docs and README for hybrid search

This commit is contained in:
David Snelling 2026-01-26 17:16:49 -08:00
parent 4adba1b254
commit a78f3bb0d2
3 changed files with 267 additions and 0 deletions

View file

@ -399,6 +399,23 @@ await brain.find({
**→ [See all query methods in API Reference](docs/api/README.md#search--query)** **→ [See all query methods in API Reference](docs/api/README.md#search--query)**
### 🔍 **Zero-Config Hybrid Search** (v7.7.0)
Automatically combines text (keyword) and semantic (vector) search for optimal results:
```javascript
// Just works - no configuration needed
const results = await brain.find({ query: 'David Smith' })
// Finds exact text matches AND semantically similar content
// Override when needed
await brain.find({ query: 'exact match', searchMode: 'text' }) // Text only
await brain.find({ query: 'AI concepts', searchMode: 'semantic' }) // Semantic only
await brain.find({ query: 'hybrid', hybridAlpha: 0.3 }) // Custom weighting
```
**→ [Hybrid Search Documentation](docs/api/README.md#hybrid-search-v770)**
### 🌐 **Virtual Filesystem** — Intelligent File Management ### 🌐 **Virtual Filesystem** — Intelligent File Management
Build file explorers and IDEs that never crash: Build file explorers and IDEs that never crash:

View file

@ -25,6 +25,7 @@ Brainy has **3 main indexes** at the top level, each with multiple sub-indexes m
- **FieldTypeInference** - DuckDB-inspired value-based field type detection - **FieldTypeInference** - DuckDB-inspired value-based field type detection
- **Field Sparse Indexes** - Per-field sparse indexes with roaring bitmaps (dynamic count) - **Field Sparse Indexes** - Per-field sparse indexes with roaring bitmaps (dynamic count)
- **Sorted Indexes** - Support orderBy queries (automatically maintained) - **Sorted Indexes** - Support orderBy queries (automatically maintained)
- **Word Index (`__words__`)** - Text search via FNV-1a word hashes (v7.7.0)
**GraphAdjacencyIndex contains:** **GraphAdjacencyIndex contains:**
- **lsmTreeSource** - Source → Targets (outgoing edges) - **lsmTreeSource** - Source → Targets (outgoing edges)
@ -228,6 +229,37 @@ interface ZoneMap {
**Use case**: Enables NLP to understand "find characters named John" → knows 'name' is a character field **Use case**: Enables NLP to understand "find characters named John" → knows 'name' is a character field
#### Word Index (`__words__`) - v7.7.0
```typescript
// Special field for text/keyword search
// Entity text content is tokenized and indexed as word hashes
// Tokenization:
// "David Smith is a software engineer" → ["david", "smith", "is", "software", "engineer"]
// Word Hashing (FNV-1a):
// "david" → hashWord("david") → 1234567 (int32)
// "smith" → hashWord("smith") → 9876543 (int32)
// Index structure (same as other fields):
// __words__ → 1234567 → RoaringBitmap{entity1, entity5, ...}
// __words__ → 9876543 → RoaringBitmap{entity1, entity3, ...}
```
**Design Decisions**:
- **Max 50 words per entity**: Prevents index bloat for large documents
- **FNV-1a hashing**: Fast, low collision rate, int32 output
- **Min word length 2 chars**: Filters out noise words
- **Lowercase normalization**: Case-insensitive matching
- **Automatic integration**: Words extracted via `extractIndexableFields()`
**Hybrid Search** (v7.7.0): Text results combined with vector results using Reciprocal Rank Fusion (RRF):
```typescript
// RRF formula: score(d) = sum(1 / (k + rank(d)))
// where k = 60 (standard constant)
// alpha = weight for semantic (0 = text only, 1 = semantic only)
```
### Query Algorithm (v3.42.0) ### Query Algorithm (v3.42.0)
**Exact Match Query**: **Exact Match Query**:

View file

@ -0,0 +1,218 @@
/**
* MetadataIndexManager Text Indexing Tests (v7.7.0)
*
* Tests for word extraction, tokenization, and hashing used in hybrid search.
*/
import { describe, it, expect, beforeEach, afterEach } from 'vitest'
import { MetadataIndexManager } from '../../../src/utils/metadataIndex'
// Create a test adapter to expose private methods for testing
// This is a common testing pattern to test internal implementation
class TestableMetadataIndexManager extends MetadataIndexManager {
public testTokenize(text: string): string[] {
return this.tokenize(text)
}
public testHashWord(word: string): number {
return this.hashWord(word)
}
}
describe('MetadataIndexManager Text Indexing (v7.7.0)', () => {
describe('tokenize()', () => {
let manager: TestableMetadataIndexManager
beforeEach(() => {
// Create a minimal mock storage for testing
const mockStorage = {
getMetadata: async () => null,
saveMetadata: async () => {},
getNoun: async () => null,
getNouns: async () => ({ items: [], total: 0, hasMore: false }),
getVerbsFromSource: async () => ({ items: [], total: 0, hasMore: false }),
} as any
manager = new TestableMetadataIndexManager(mockStorage, {})
})
it('should convert to lowercase', () => {
const tokens = manager.testTokenize('HELLO World')
expect(tokens).toContain('hello')
expect(tokens).toContain('world')
expect(tokens).not.toContain('HELLO')
expect(tokens).not.toContain('World')
})
it('should remove punctuation', () => {
const tokens = manager.testTokenize('Hello, World! How are you?')
expect(tokens).toContain('hello')
expect(tokens).toContain('world')
expect(tokens).toContain('how')
expect(tokens).toContain('are')
expect(tokens).toContain('you')
})
it('should filter short words (< 2 chars)', () => {
const tokens = manager.testTokenize('I a am is an the to')
expect(tokens).not.toContain('i')
expect(tokens).not.toContain('a')
expect(tokens).toContain('am')
expect(tokens).toContain('is')
expect(tokens).toContain('an')
})
it('should deduplicate words', () => {
const tokens = manager.testTokenize('hello hello world world hello')
expect(tokens.length).toBe(2)
expect(tokens).toContain('hello')
expect(tokens).toContain('world')
})
it('should handle empty string', () => {
const tokens = manager.testTokenize('')
expect(tokens).toEqual([])
})
it('should handle whitespace only', () => {
const tokens = manager.testTokenize(' \t\n ')
expect(tokens).toEqual([])
})
it('should handle special characters', () => {
const tokens = manager.testTokenize('C++ is a programming language')
expect(tokens).toContain('is')
expect(tokens).toContain('programming')
expect(tokens).toContain('language')
})
it('should handle unicode text', () => {
const tokens = manager.testTokenize('Hello 世界 Welt')
expect(tokens).toContain('hello')
expect(tokens).toContain('welt')
})
it('should handle numbers', () => {
const tokens = manager.testTokenize('version 123 release 45')
expect(tokens).toContain('version')
expect(tokens).toContain('123')
expect(tokens).toContain('release')
expect(tokens).toContain('45')
})
it('should handle hyphenated words', () => {
const tokens = manager.testTokenize('state-of-the-art machine-learning')
// Hyphenated words become separate tokens due to punctuation removal
expect(tokens).toContain('state')
expect(tokens).toContain('the')
expect(tokens).toContain('art')
expect(tokens).toContain('machine')
expect(tokens).toContain('learning')
})
})
describe('hashWord()', () => {
let manager: TestableMetadataIndexManager
beforeEach(() => {
const mockStorage = {
getMetadata: async () => null,
saveMetadata: async () => {},
getNoun: async () => null,
getNouns: async () => ({ items: [], total: 0, hasMore: false }),
getVerbsFromSource: async () => ({ items: [], total: 0, hasMore: false }),
} as any
manager = new TestableMetadataIndexManager(mockStorage, {})
})
it('should produce consistent hash for same word', () => {
const hash1 = manager.testHashWord('hello')
const hash2 = manager.testHashWord('hello')
expect(hash1).toBe(hash2)
})
it('should produce different hashes for different words', () => {
const hash1 = manager.testHashWord('hello')
const hash2 = manager.testHashWord('world')
expect(hash1).not.toBe(hash2)
})
it('should produce int32 values', () => {
const hash = manager.testHashWord('test')
expect(Number.isInteger(hash)).toBe(true)
expect(hash).toBeGreaterThanOrEqual(-2147483648)
expect(hash).toBeLessThanOrEqual(2147483647)
})
it('should handle empty string', () => {
const hash = manager.testHashWord('')
expect(Number.isInteger(hash)).toBe(true)
})
it('should handle long words', () => {
const longWord = 'supercalifragilisticexpialidocious'
const hash = manager.testHashWord(longWord)
expect(Number.isInteger(hash)).toBe(true)
})
it('should hash unicode words', () => {
const hash = manager.testHashWord('世界')
expect(Number.isInteger(hash)).toBe(true)
})
it('should be case sensitive (words are lowercased before hashing in tokenize)', () => {
const hash1 = manager.testHashWord('Hello')
const hash2 = manager.testHashWord('hello')
// Hashes are different because case matters in hash function
// But tokenize() lowercases before hashing
expect(hash1).not.toBe(hash2)
})
})
describe('getIdsForTextQuery()', () => {
let manager: MetadataIndexManager
let mockStorage: any
let addedEntities: Map<string, any>
beforeEach(() => {
addedEntities = new Map()
let idCounter = 0
mockStorage = {
getMetadata: async () => null,
saveMetadata: async () => {},
getNoun: async (id: string) => addedEntities.get(id) || null,
getNouns: async () => ({
items: Array.from(addedEntities.values()),
total: addedEntities.size,
hasMore: false
}),
getVerbsFromSource: async () => ({ items: [], total: 0, hasMore: false }),
}
manager = new MetadataIndexManager(mockStorage, {})
})
it('should return empty array for empty query', async () => {
const results = await manager.getIdsForTextQuery('')
expect(results).toEqual([])
})
it('should return empty array for whitespace query', async () => {
const results = await manager.getIdsForTextQuery(' ')
expect(results).toEqual([])
})
it('should return results sorted by match count', async () => {
// This test verifies the interface contract
// Actual matching behavior tested in integration tests
const results = await manager.getIdsForTextQuery('hello world')
expect(Array.isArray(results)).toBe(true)
results.forEach(r => {
expect(r).toHaveProperty('id')
expect(r).toHaveProperty('matchCount')
})
})
})
})