feat: v0.49 - Filter discovery API, remove deprecated methods, improve performance

BREAKING CHANGES:
- Removed deprecated getAllNouns() and getAllVerbs() methods
- All internal usage migrated to pagination-based methods

New Features:
- Filter Discovery API:
  - getFilterValues(field): Get all available values for a field
  - getFilterFields(): Get all filterable fields
  - Enables dynamic filter UI generation with O(1) field discovery
- Hybrid metadata indexing with field-level indexes
- Adaptive auto-flush for optimal performance
- LRU caching for metadata indexes

Improvements:
- Fixed ENAMETOOLONG errors from vector-based filenames
- Safe filename generation using hash-based approach
- Scalable chunked value storage for millions of entries
- Performance optimization with adaptive flush thresholds
- Added support for $includes operator in metadata filters

Technical:
- Replaced vector-based filenames with safe hash approach
- Implemented MetadataIndexCache with existing SearchCache pattern
- Field indexes enable O(1) filter discovery
- Adaptive flush based on performance metrics (20-200 entries)
- All tests passing with improved metadata filtering

tests/filter-discovery.test.ts

@@ -0,0 +1,107 @@
+import { describe, it, expect, beforeEach } from 'vitest'
+import { BrainyData } from '../src/index.js'
+
+describe('Filter Discovery API', () => {
+  let brainy: BrainyData
+  
+  beforeEach(async () => {
+    brainy = new BrainyData({
+      storage: { type: 'memory' }
+    })
+    await brainy.init()
+  })
+
+  it('should return available filter values for a field', async () => {
+    // Add test data with metadata
+    await brainy.add('product1', {
+      category: 'electronics',
+      brand: 'Apple',
+      price: 999
+    })
+    
+    await brainy.add('product2', {
+      category: 'electronics',
+      brand: 'Samsung',
+      price: 799
+    })
+    
+    await brainy.add('product3', {
+      category: 'books',
+      brand: 'Penguin',
+      price: 19
+    })
+    
+    // Get filter values for category field
+    const categories = await brainy.getFilterValues('category')
+    expect(categories).toContain('electronics')
+    expect(categories).toContain('books')
+    expect(categories.length).toBe(2)
+    
+    // Get filter values for brand field
+    const brands = await brainy.getFilterValues('brand')
+    expect(brands).toContain('apple') // normalized to lowercase
+    expect(brands).toContain('samsung')
+    expect(brands).toContain('penguin')
+    expect(brands.length).toBe(3)
+  })
+
+  it('should return all available filter fields', async () => {
+    // Add test data with various metadata fields
+    await brainy.add('item1', {
+      category: 'electronics',
+      brand: 'Apple',
+      price: 999,
+      rating: 4.5
+    })
+    
+    await brainy.add('item2', {
+      category: 'books',
+      author: 'Tolkien',
+      pages: 500
+    })
+    
+    // Get all filter fields
+    const fields = await brainy.getFilterFields()
+    
+    // Should include all unique fields from all items (except excluded ones)
+    expect(fields).toContain('category')
+    expect(fields).toContain('brand')
+    expect(fields).toContain('price')
+    expect(fields).toContain('rating')
+    expect(fields).toContain('author')
+    expect(fields).toContain('pages')
+  })
+
+  it('should use cache for repeated filter value requests', async () => {
+    // Add test data
+    await brainy.add('item1', { category: 'electronics' })
+    await brainy.add('item2', { category: 'books' })
+    
+    // First call - loads from storage
+    const start1 = Date.now()
+    const categories1 = await brainy.getFilterValues('category')
+    const time1 = Date.now() - start1
+    
+    // Second call - should use cache and be faster
+    const start2 = Date.now()
+    const categories2 = await brainy.getFilterValues('category')
+    const time2 = Date.now() - start2
+    
+    // Results should be identical
+    expect(categories1).toEqual(categories2)
+    
+    // Cache should be significantly faster (at least 2x)
+    // Note: This might be flaky in CI, so we just check they're equal for now
+    expect(categories2.length).toBe(2)
+  })
+
+  it('should handle empty fields gracefully', async () => {
+    // Try to get values for non-existent field
+    const values = await brainy.getFilterValues('nonexistent')
+    expect(values).toEqual([])
+    
+    // Get fields when no data exists
+    const fields = await brainy.getFilterFields()
+    expect(fields).toEqual([])
+  })
+})

tests/metadata-filter.test.ts

@@ -169,6 +169,14 @@ describe('Metadata Filtering', () => {
     })
 
     it('should filter with MongoDB operators', async () => {
+      // First verify what we have in the index
+      const allResults = await brainy.searchText('developer', 10)
+      console.log('All results before filtering:', allResults.map(r => ({
+        name: r.metadata?.name,
+        skills: r.metadata?.skills,
+        available: r.metadata?.available
+      })))
+      
       const results = await brainy.searchText('developer', 10, {
         metadata: {
           skills: { $includes: 'React' },
@@ -176,7 +184,23 @@ describe('Metadata Filtering', () => {
         }
       })
       
+      console.log('Filtered results:', results.map(r => ({
+        name: r.metadata?.name,
+        skills: r.metadata?.skills,
+        available: r.metadata?.available
+      })))
+      
       expect(results.length).toBeGreaterThan(0)
+      
+      // Check each result individually for debugging
+      for (const r of results) {
+        const hasReact = r.metadata?.skills?.includes('React')
+        const isAvailable = r.metadata?.available === true
+        if (!hasReact || !isAvailable) {
+          console.log('Failed result:', r.metadata)
+        }
+      }
+      
       expect(results.every(r => 
         r.metadata?.skills?.includes('React') && 
         r.metadata?.available === true