From a78f3bb0d2365426536f58a32c21bc9a23781982 Mon Sep 17 00:00:00 2001 From: David Snelling Date: Mon, 26 Jan 2026 17:16:49 -0800 Subject: [PATCH] docs: update architecture docs and README for hybrid search --- README.md | 17 ++ docs/architecture/index-architecture.md | 32 +++ .../utils/metadataIndex-text-indexing.test.ts | 218 ++++++++++++++++++ 3 files changed, 267 insertions(+) create mode 100644 tests/unit/utils/metadataIndex-text-indexing.test.ts diff --git a/README.md b/README.md index 72f18947..edfc0846 100644 --- a/README.md +++ b/README.md @@ -399,6 +399,23 @@ await brain.find({ **β†’ [See all query methods in API Reference](docs/api/README.md#search--query)** +### πŸ” **Zero-Config Hybrid Search** (v7.7.0) + +Automatically combines text (keyword) and semantic (vector) search for optimal results: + +```javascript +// Just works - no configuration needed +const results = await brain.find({ query: 'David Smith' }) +// Finds exact text matches AND semantically similar content + +// Override when needed +await brain.find({ query: 'exact match', searchMode: 'text' }) // Text only +await brain.find({ query: 'AI concepts', searchMode: 'semantic' }) // Semantic only +await brain.find({ query: 'hybrid', hybridAlpha: 0.3 }) // Custom weighting +``` + +**β†’ [Hybrid Search Documentation](docs/api/README.md#hybrid-search-v770)** + ### 🌐 **Virtual Filesystem** β€” Intelligent File Management Build file explorers and IDEs that never crash: diff --git a/docs/architecture/index-architecture.md b/docs/architecture/index-architecture.md index 3694f48f..9ae8c5cf 100644 --- a/docs/architecture/index-architecture.md +++ b/docs/architecture/index-architecture.md @@ -25,6 +25,7 @@ Brainy has **3 main indexes** at the top level, each with multiple sub-indexes m - **FieldTypeInference** - DuckDB-inspired value-based field type detection - **Field Sparse Indexes** - Per-field sparse indexes with roaring bitmaps (dynamic count) - **Sorted Indexes** - Support orderBy queries (automatically maintained) +- **Word Index (`__words__`)** - Text search via FNV-1a word hashes (v7.7.0) **GraphAdjacencyIndex contains:** - **lsmTreeSource** - Source β†’ Targets (outgoing edges) @@ -228,6 +229,37 @@ interface ZoneMap { **Use case**: Enables NLP to understand "find characters named John" β†’ knows 'name' is a character field +#### Word Index (`__words__`) - v7.7.0 +```typescript +// Special field for text/keyword search +// Entity text content is tokenized and indexed as word hashes + +// Tokenization: +// "David Smith is a software engineer" β†’ ["david", "smith", "is", "software", "engineer"] + +// Word Hashing (FNV-1a): +// "david" β†’ hashWord("david") β†’ 1234567 (int32) +// "smith" β†’ hashWord("smith") β†’ 9876543 (int32) + +// Index structure (same as other fields): +// __words__ β†’ 1234567 β†’ RoaringBitmap{entity1, entity5, ...} +// __words__ β†’ 9876543 β†’ RoaringBitmap{entity1, entity3, ...} +``` + +**Design Decisions**: +- **Max 50 words per entity**: Prevents index bloat for large documents +- **FNV-1a hashing**: Fast, low collision rate, int32 output +- **Min word length 2 chars**: Filters out noise words +- **Lowercase normalization**: Case-insensitive matching +- **Automatic integration**: Words extracted via `extractIndexableFields()` + +**Hybrid Search** (v7.7.0): Text results combined with vector results using Reciprocal Rank Fusion (RRF): +```typescript +// RRF formula: score(d) = sum(1 / (k + rank(d))) +// where k = 60 (standard constant) +// alpha = weight for semantic (0 = text only, 1 = semantic only) +``` + ### Query Algorithm (v3.42.0) **Exact Match Query**: diff --git a/tests/unit/utils/metadataIndex-text-indexing.test.ts b/tests/unit/utils/metadataIndex-text-indexing.test.ts new file mode 100644 index 00000000..d3b34244 --- /dev/null +++ b/tests/unit/utils/metadataIndex-text-indexing.test.ts @@ -0,0 +1,218 @@ +/** + * MetadataIndexManager Text Indexing Tests (v7.7.0) + * + * Tests for word extraction, tokenization, and hashing used in hybrid search. + */ + +import { describe, it, expect, beforeEach, afterEach } from 'vitest' +import { MetadataIndexManager } from '../../../src/utils/metadataIndex' + +// Create a test adapter to expose private methods for testing +// This is a common testing pattern to test internal implementation +class TestableMetadataIndexManager extends MetadataIndexManager { + public testTokenize(text: string): string[] { + return this.tokenize(text) + } + + public testHashWord(word: string): number { + return this.hashWord(word) + } +} + +describe('MetadataIndexManager Text Indexing (v7.7.0)', () => { + describe('tokenize()', () => { + let manager: TestableMetadataIndexManager + + beforeEach(() => { + // Create a minimal mock storage for testing + const mockStorage = { + getMetadata: async () => null, + saveMetadata: async () => {}, + getNoun: async () => null, + getNouns: async () => ({ items: [], total: 0, hasMore: false }), + getVerbsFromSource: async () => ({ items: [], total: 0, hasMore: false }), + } as any + + manager = new TestableMetadataIndexManager(mockStorage, {}) + }) + + it('should convert to lowercase', () => { + const tokens = manager.testTokenize('HELLO World') + expect(tokens).toContain('hello') + expect(tokens).toContain('world') + expect(tokens).not.toContain('HELLO') + expect(tokens).not.toContain('World') + }) + + it('should remove punctuation', () => { + const tokens = manager.testTokenize('Hello, World! How are you?') + expect(tokens).toContain('hello') + expect(tokens).toContain('world') + expect(tokens).toContain('how') + expect(tokens).toContain('are') + expect(tokens).toContain('you') + }) + + it('should filter short words (< 2 chars)', () => { + const tokens = manager.testTokenize('I a am is an the to') + expect(tokens).not.toContain('i') + expect(tokens).not.toContain('a') + expect(tokens).toContain('am') + expect(tokens).toContain('is') + expect(tokens).toContain('an') + }) + + it('should deduplicate words', () => { + const tokens = manager.testTokenize('hello hello world world hello') + expect(tokens.length).toBe(2) + expect(tokens).toContain('hello') + expect(tokens).toContain('world') + }) + + it('should handle empty string', () => { + const tokens = manager.testTokenize('') + expect(tokens).toEqual([]) + }) + + it('should handle whitespace only', () => { + const tokens = manager.testTokenize(' \t\n ') + expect(tokens).toEqual([]) + }) + + it('should handle special characters', () => { + const tokens = manager.testTokenize('C++ is a programming language') + expect(tokens).toContain('is') + expect(tokens).toContain('programming') + expect(tokens).toContain('language') + }) + + it('should handle unicode text', () => { + const tokens = manager.testTokenize('Hello δΈ–η•Œ Welt') + expect(tokens).toContain('hello') + expect(tokens).toContain('welt') + }) + + it('should handle numbers', () => { + const tokens = manager.testTokenize('version 123 release 45') + expect(tokens).toContain('version') + expect(tokens).toContain('123') + expect(tokens).toContain('release') + expect(tokens).toContain('45') + }) + + it('should handle hyphenated words', () => { + const tokens = manager.testTokenize('state-of-the-art machine-learning') + // Hyphenated words become separate tokens due to punctuation removal + expect(tokens).toContain('state') + expect(tokens).toContain('the') + expect(tokens).toContain('art') + expect(tokens).toContain('machine') + expect(tokens).toContain('learning') + }) + }) + + describe('hashWord()', () => { + let manager: TestableMetadataIndexManager + + beforeEach(() => { + const mockStorage = { + getMetadata: async () => null, + saveMetadata: async () => {}, + getNoun: async () => null, + getNouns: async () => ({ items: [], total: 0, hasMore: false }), + getVerbsFromSource: async () => ({ items: [], total: 0, hasMore: false }), + } as any + + manager = new TestableMetadataIndexManager(mockStorage, {}) + }) + + it('should produce consistent hash for same word', () => { + const hash1 = manager.testHashWord('hello') + const hash2 = manager.testHashWord('hello') + expect(hash1).toBe(hash2) + }) + + it('should produce different hashes for different words', () => { + const hash1 = manager.testHashWord('hello') + const hash2 = manager.testHashWord('world') + expect(hash1).not.toBe(hash2) + }) + + it('should produce int32 values', () => { + const hash = manager.testHashWord('test') + expect(Number.isInteger(hash)).toBe(true) + expect(hash).toBeGreaterThanOrEqual(-2147483648) + expect(hash).toBeLessThanOrEqual(2147483647) + }) + + it('should handle empty string', () => { + const hash = manager.testHashWord('') + expect(Number.isInteger(hash)).toBe(true) + }) + + it('should handle long words', () => { + const longWord = 'supercalifragilisticexpialidocious' + const hash = manager.testHashWord(longWord) + expect(Number.isInteger(hash)).toBe(true) + }) + + it('should hash unicode words', () => { + const hash = manager.testHashWord('δΈ–η•Œ') + expect(Number.isInteger(hash)).toBe(true) + }) + + it('should be case sensitive (words are lowercased before hashing in tokenize)', () => { + const hash1 = manager.testHashWord('Hello') + const hash2 = manager.testHashWord('hello') + // Hashes are different because case matters in hash function + // But tokenize() lowercases before hashing + expect(hash1).not.toBe(hash2) + }) + }) + + describe('getIdsForTextQuery()', () => { + let manager: MetadataIndexManager + let mockStorage: any + let addedEntities: Map + + beforeEach(() => { + addedEntities = new Map() + let idCounter = 0 + + mockStorage = { + getMetadata: async () => null, + saveMetadata: async () => {}, + getNoun: async (id: string) => addedEntities.get(id) || null, + getNouns: async () => ({ + items: Array.from(addedEntities.values()), + total: addedEntities.size, + hasMore: false + }), + getVerbsFromSource: async () => ({ items: [], total: 0, hasMore: false }), + } + + manager = new MetadataIndexManager(mockStorage, {}) + }) + + it('should return empty array for empty query', async () => { + const results = await manager.getIdsForTextQuery('') + expect(results).toEqual([]) + }) + + it('should return empty array for whitespace query', async () => { + const results = await manager.getIdsForTextQuery(' ') + expect(results).toEqual([]) + }) + + it('should return results sorted by match count', async () => { + // This test verifies the interface contract + // Actual matching behavior tested in integration tests + const results = await manager.getIdsForTextQuery('hello world') + expect(Array.isArray(results)).toBe(true) + results.forEach(r => { + expect(r).toHaveProperty('id') + expect(r).toHaveProperty('matchCount') + }) + }) + }) +})