fix: correct typo in README major updates section

🤖 Generated with [Claude Code](https://claude.ai/code)

Co-Authored-By: Claude <noreply@anthropic.com>

tests/base-hnsw-test.ts

@@ -0,0 +1,47 @@
+// Test using base HNSW directly
+import { HNSWIndex } from '../dist/hnsw/hnswIndex.js'
+import { euclideanDistance } from '../dist/utils/distance.js'
+
+async function testBaseHNSW() {
+  console.log('🧪 Testing base HNSW directly...')
+  
+  const index = new HNSWIndex(
+    { M: 4, efConstruction: 20, efSearch: 50 },
+    euclideanDistance
+  )
+  
+  // Create test vectors
+  const aliceVector = Array.from({length: 384}, () => Math.random())
+  const bobVector = Array.from({length: 384}, () => Math.random())
+  const queryVector = Array.from({length: 384}, () => Math.random())
+  
+  // Add items to index
+  const aliceId = 'alice-123'
+  const bobId = 'bob-456'
+  
+  await index.addItem({ id: aliceId, vector: aliceVector })
+  await index.addItem({ id: bobId, vector: bobVector })
+  
+  console.log('Added items to index')
+  
+  // Test without filter
+  const allResults = await index.search(queryVector, 10)
+  console.log('All results:', allResults.length)
+  
+  // Test with filter - only allow Alice
+  const aliceOnlyFilter = async (id: string) => {
+    console.log('🔍 Filter called for:', id, id === aliceId ? '✅ ALLOW' : '❌ BLOCK')
+    return id === aliceId
+  }
+  
+  console.log('Testing with filter...')
+  const filteredResults = await index.search(queryVector, 10, aliceOnlyFilter)
+  console.log('Filtered results:', filteredResults.length)
+  
+  const shouldWork = filteredResults.length === 1 && filteredResults[0][0] === aliceId
+  console.log(shouldWork ? '✅ FILTERING WORKS!' : '❌ FILTERING FAILED!')
+  
+  return shouldWork
+}
+
+testBaseHNSW().catch(console.error)

tests/filter-test.ts

@@ -0,0 +1,57 @@
+// Minimal test to debug metadata filtering
+import { BrainyData } from '../dist/brainyData.js'
+
+async function testDirectFiltering() {
+  console.log('🧪 Testing direct filtering...')
+  
+  const brainy = new BrainyData({
+    storage: { forceMemoryStorage: true },
+    hnsw: { M: 4, efConstruction: 20, useOptimizedIndex: false }  // Force regular HNSW
+  })
+  
+  await brainy.init()
+  
+  // Add two items
+  const aliceId = await brainy.add('Senior developer Alice', { level: 'senior', name: 'Alice' })
+  const bobId = await brainy.add('Junior developer Bob', { level: 'junior', name: 'Bob' })
+  
+  console.log('Alice ID:', aliceId)
+  console.log('Bob ID:', bobId)
+  
+  // Test metadata index directly
+  if (brainy.metadataIndex) {
+    const seniorIds = await brainy.metadataIndex.getIds('level', 'senior')
+    const juniorIds = await brainy.metadataIndex.getIds('level', 'junior')
+    console.log('Senior IDs from index:', seniorIds)
+    console.log('Junior IDs from index:', juniorIds)
+  }
+  
+  // Test the HNSW search directly with a simple filter
+  const queryVector = await brainy.embed('developer')
+  console.log('Query vector dimensions:', queryVector.length)
+  
+  // Create a simple filter that only allows Alice
+  const simpleFilter = async (id: string) => {
+    console.log('🔍 Simple filter called for:', id, id === aliceId ? '✅ ALLOW' : '❌ BLOCK')
+    return id === aliceId
+  }
+  
+  console.log('Testing HNSW search with filter...')
+  console.log('Index type:', brainy.index.constructor.name)
+  console.log('Index has search method:', typeof brainy.index.search)
+  console.log('Filter function:', typeof simpleFilter)
+  console.log('About to call search with:', !!simpleFilter)
+  const filteredResults = await brainy.index.search(queryVector, 10, simpleFilter)
+  console.log('Filtered results:', filteredResults.length, 'items')
+  
+  for (const [id, score] of filteredResults) {
+    console.log(`- ${id}: ${score.toFixed(3)} ${id === aliceId ? '(Alice)' : id === bobId ? '(Bob)' : '(Unknown)'}`)
+  }
+  
+  const shouldWork = filteredResults.length === 1 && filteredResults[0][0] === aliceId
+  console.log(shouldWork ? '✅ FILTERING WORKS!' : '❌ FILTERING FAILED!')
+  
+  return shouldWork
+}
+
+testDirectFiltering().catch(console.error)

tests/metadata-filter-debug.test.ts

@@ -0,0 +1,101 @@
+import { describe, it, expect } from 'vitest'
+import { BrainyData } from '../src/brainyData.js'
+
+describe('Metadata Filter Works', () => {
+  it('should filter results by metadata during search', async () => {
+    const brainy = new BrainyData({
+      storage: { forceMemoryStorage: true },
+      hnsw: { M: 4, efConstruction: 20 },
+      logging: { verbose: false }
+    })
+    await brainy.init()
+    
+    // Add test data
+    await brainy.add('Senior developer Alice works with React', { level: 'senior', skill: 'React' })
+    await brainy.add('Junior developer Bob learns Vue', { level: 'junior', skill: 'Vue' })
+    await brainy.add('Senior developer Charlie codes Python', { level: 'senior', skill: 'Python' })
+    
+    // Test 1: Search without filter (should return all 3)
+    const allResults = await brainy.searchText('developer', 10)
+    expect(allResults.length).toBe(3)
+    
+    // Test 2: Search with level filter (should return 2 senior developers)
+    const seniorResults = await brainy.searchText('developer', 10, {
+      metadata: { level: 'senior' }
+    })
+    expect(seniorResults.length).toBe(2)
+    expect(seniorResults.every(r => r.metadata?.level === 'senior')).toBe(true)
+    
+    // Test 3: Search with skill filter (should return 1 React developer)
+    const reactResults = await brainy.searchText('developer', 10, {
+      metadata: { skill: 'React' }
+    })
+    expect(reactResults.length).toBe(1)
+    expect(reactResults[0].metadata?.skill).toBe('React')
+    
+    // Test 4: Search with multiple filters (should return 1 senior React developer)
+    const seniorReactResults = await brainy.searchText('developer', 10, {
+      metadata: { 
+        level: 'senior',
+        skill: 'React'
+      }
+    })
+    expect(seniorReactResults.length).toBe(1)
+    expect(seniorReactResults[0].metadata?.level).toBe('senior')
+    expect(seniorReactResults[0].metadata?.skill).toBe('React')
+    
+    // Test 5: Search with no matches (should return 0)
+    const noResults = await brainy.searchText('developer', 10, {
+      metadata: { level: 'expert' }
+    })
+    expect(noResults.length).toBe(0)
+  })
+  
+  it('should work with searchWithinItems', async () => {
+    const brainy = new BrainyData({
+      storage: { forceMemoryStorage: true },
+      hnsw: { M: 4, efConstruction: 20 },
+      logging: { verbose: false }
+    })
+    await brainy.init()
+    
+    // Add test data
+    const id1 = await brainy.add('Frontend React developer', { type: 'frontend', skill: 'React' })
+    const id2 = await brainy.add('Backend Node developer', { type: 'backend', skill: 'Node' })
+    const id3 = await brainy.add('Frontend Vue developer', { type: 'frontend', skill: 'Vue' })
+    
+    // Search within only frontend developers
+    const frontendResults = await brainy.searchWithinItems('developer', [id1, id3], 10)
+    
+    expect(frontendResults.length).toBe(2)
+    expect(frontendResults.every(r => [id1, id3].includes(r.id))).toBe(true)
+  })
+  
+  it('should handle MongoDB-style operators', async () => {
+    const brainy = new BrainyData({
+      storage: { forceMemoryStorage: true },
+      hnsw: { M: 4, efConstruction: 20 },
+      logging: { verbose: false }
+    })
+    await brainy.init()
+    
+    // Add test data
+    await brainy.add('Developer with 5 years experience', { experience: 5, skills: ['React', 'Node'] })
+    await brainy.add('Developer with 2 years experience', { experience: 2, skills: ['Vue'] })
+    await brainy.add('Developer with 8 years experience', { experience: 8, skills: ['React', 'Python'] })
+    
+    // Test $gt operator
+    const experiencedResults = await brainy.searchText('developer', 10, {
+      metadata: { experience: { $gt: 3 } }
+    })
+    expect(experiencedResults.length).toBe(2)
+    expect(experiencedResults.every(r => (r.metadata?.experience as number) > 3)).toBe(true)
+    
+    // Test $in operator  
+    const skillResults = await brainy.searchText('developer', 10, {
+      metadata: { experience: { $in: [2, 8] } }
+    })
+    expect(skillResults.length).toBe(2)
+    expect(skillResults.every(r => [2, 8].includes(r.metadata?.experience as number))).toBe(true)
+  })
+})

tests/metadata-filter-environments.test.ts

@@ -0,0 +1,272 @@
+import { describe, it, expect, beforeEach } from 'vitest'
+import { BrainyData } from '../src/brainyData.js'
+import { isNode, isBrowser } from '../src/utils/environment.js'
+
+describe('Metadata Filtering - Cross-Environment', () => {
+  const testConfigurations = [
+    {
+      name: 'Memory Storage',
+      config: { storage: { forceMemoryStorage: true } }
+    }
+  ]
+
+  // Add Node.js specific storage adapters
+  if (isNode()) {
+    testConfigurations.push({
+      name: 'FileSystem Storage',
+      config: { storage: { forceFileSystemStorage: true } }
+    })
+  }
+
+  // Add browser specific storage adapters
+  if (isBrowser()) {
+    testConfigurations.push({
+      name: 'OPFS Storage',
+      config: { storage: { requestPersistentStorage: false } }
+    })
+  }
+
+  // Test each storage configuration
+  for (const testConfig of testConfigurations) {
+    describe(`${testConfig.name}`, () => {
+      let brainy: BrainyData
+
+      beforeEach(async () => {
+        brainy = new BrainyData({
+          ...testConfig.config,
+          hnsw: { M: 8, efConstruction: 50 },
+          logging: { verbose: false }
+        })
+        await brainy.init()
+
+        // Add test data
+        const testData = [
+          { 
+            content: 'Senior React developer in San Francisco',
+            metadata: { level: 'senior', skill: 'React', location: 'SF', remote: true }
+          },
+          { 
+            content: 'Junior Vue developer in New York',
+            metadata: { level: 'junior', skill: 'Vue', location: 'NYC', remote: false }
+          },
+          { 
+            content: 'Mid-level TypeScript developer remote',
+            metadata: { level: 'mid', skill: 'TypeScript', location: 'Remote', remote: true }
+          },
+          { 
+            content: 'Senior Python engineer in San Francisco',
+            metadata: { level: 'senior', skill: 'Python', location: 'SF', remote: false }
+          },
+          { 
+            content: 'Senior JavaScript developer in Austin',
+            metadata: { level: 'senior', skill: 'JavaScript', location: 'Austin', remote: true }
+          }
+        ]
+
+        for (const item of testData) {
+          await brainy.add(item.content, item.metadata)
+        }
+      })
+
+      it('should filter by exact metadata match', async () => {
+        const results = await brainy.searchText('developer', 10, {
+          metadata: { level: 'senior' }
+        })
+
+        expect(results.length).toBeGreaterThan(0)
+        expect(results.every(r => r.metadata?.level === 'senior')).toBe(true)
+      })
+
+      it('should filter by multiple fields', async () => {
+        const results = await brainy.searchText('developer', 10, {
+          metadata: { 
+            level: 'senior',
+            location: 'SF'
+          }
+        })
+
+        expect(results.length).toBeGreaterThan(0)
+        expect(results.every(r => 
+          r.metadata?.level === 'senior' && 
+          r.metadata?.location === 'SF'
+        )).toBe(true)
+      })
+
+      it('should handle boolean filters', async () => {
+        const results = await brainy.searchText('developer', 10, {
+          metadata: { remote: true }
+        })
+
+        expect(results.length).toBeGreaterThan(0)
+        expect(results.every(r => r.metadata?.remote === true)).toBe(true)
+      })
+
+      it('should handle $in operator', async () => {
+        const results = await brainy.searchText('developer', 10, {
+          metadata: { 
+            skill: { $in: ['React', 'Vue', 'TypeScript'] }
+          }
+        })
+
+        expect(results.length).toBeGreaterThan(0)
+        expect(results.every(r => 
+          ['React', 'Vue', 'TypeScript'].includes(r.metadata?.skill)
+        )).toBe(true)
+      })
+
+      it('should handle combined filters with $and', async () => {
+        const results = await brainy.searchText('developer', 10, {
+          metadata: {
+            $and: [
+              { level: 'senior' },
+              { remote: true }
+            ]
+          }
+        })
+
+        expect(results.length).toBeGreaterThan(0)
+        expect(results.every(r => 
+          r.metadata?.level === 'senior' && 
+          r.metadata?.remote === true
+        )).toBe(true)
+      })
+
+      it('should handle $or operator', async () => {
+        const results = await brainy.searchText('developer', 10, {
+          metadata: {
+            $or: [
+              { location: 'SF' },
+              { location: 'NYC' }
+            ]
+          }
+        })
+
+        expect(results.length).toBeGreaterThan(0)
+        expect(results.every(r => 
+          r.metadata?.location === 'SF' || 
+          r.metadata?.location === 'NYC'
+        )).toBe(true)
+      })
+
+      it('should return empty results when no items match filter', async () => {
+        const results = await brainy.searchText('developer', 10, {
+          metadata: { 
+            level: 'expert' // Non-existent level
+          }
+        })
+
+        expect(results.length).toBe(0)
+      })
+
+      it('should work with searchWithinItems', async () => {
+        // First get all senior developers
+        const allItems = await brainy.getNouns({
+          filter: {
+            metadata: { level: 'senior' }
+          }
+        })
+        
+        const seniorIds = allItems.items.map(item => item.id)
+        
+        // Search within senior developers only
+        const results = await brainy.searchWithinItems(
+          'JavaScript',
+          seniorIds,
+          5
+        )
+
+        expect(results.length).toBeGreaterThanOrEqual(0)
+        expect(results.every(r => seniorIds.includes(r.id))).toBe(true)
+      })
+
+      it('should handle metadata updates correctly', async () => {
+        // Add an item
+        const id = await brainy.add('Test developer', { level: 'junior' })
+        
+        // Search should find it with junior filter
+        let results = await brainy.searchText('Test developer', 10, {
+          metadata: { level: 'junior' }
+        })
+        expect(results.some(r => r.id === id)).toBe(true)
+
+        // Update metadata
+        await brainy.updateMetadata(id, { level: 'senior' })
+
+        // Should now find it with senior filter
+        results = await brainy.searchText('Test developer', 10, {
+          metadata: { level: 'senior' }
+        })
+        expect(results.some(r => r.id === id)).toBe(true)
+
+        // Should NOT find it with junior filter anymore
+        results = await brainy.searchText('Test developer', 10, {
+          metadata: { level: 'junior' }
+        })
+        expect(results.some(r => r.id === id)).toBe(false)
+      })
+
+      it('should handle null/undefined metadata gracefully', async () => {
+        // Add item without metadata
+        await brainy.add('No metadata item')
+
+        // Search with filter should not crash
+        const results = await brainy.searchText('metadata', 10, {
+          metadata: { level: 'senior' }
+        })
+
+        // Should only return items that match the filter
+        expect(results.every(r => r.metadata?.level === 'senior')).toBe(true)
+      })
+    })
+  }
+
+  describe('Performance considerations', () => {
+    it('should handle large result sets efficiently', async () => {
+      const brainy = new BrainyData({
+        storage: { forceMemoryStorage: true },
+        hnsw: { M: 16, efConstruction: 100 },
+        logging: { verbose: false }
+      })
+      await brainy.init()
+
+      // Add many items
+      const categories = ['A', 'B', 'C', 'D', 'E']
+      const levels = ['junior', 'mid', 'senior']
+      
+      for (let i = 0; i < 100; i++) {
+        await brainy.add(
+          `Item ${i} with various properties`,
+          {
+            category: categories[i % categories.length],
+            level: levels[i % levels.length],
+            index: i
+          }
+        )
+      }
+
+      const startTime = Date.now()
+      
+      // Search with complex filter
+      const results = await brainy.searchText('Item', 20, {
+        metadata: {
+          $and: [
+            { category: { $in: ['A', 'B', 'C'] } },
+            { level: { $ne: 'junior' } }
+          ]
+        }
+      })
+
+      const duration = Date.now() - startTime
+
+      expect(results.length).toBeGreaterThan(0)
+      expect(results.length).toBeLessThanOrEqual(20)
+      expect(duration).toBeLessThan(1000) // Should complete within 1 second
+      
+      // Verify all results match the filter
+      expect(results.every(r => {
+        const m = r.metadata
+        return ['A', 'B', 'C'].includes(m?.category) && m?.level !== 'junior'
+      })).toBe(true)
+    })
+  })
+})

tests/metadata-filter.test.ts

@@ -0,0 +1,245 @@
+import { describe, it, expect, beforeEach } from 'vitest'
+import { BrainyData } from '../src/brainyData.js'
+import { matchesMetadataFilter } from '../src/utils/metadataFilter.js'
+
+describe('Metadata Filtering', () => {
+  let brainy: BrainyData
+
+  beforeEach(async () => {
+    brainy = new BrainyData({
+      storage: { forceMemoryStorage: true },
+      hnsw: { M: 8, efConstruction: 50 },
+      logging: { verbose: false }
+    })
+    await brainy.init()
+    console.log('BrainyData initialized')
+  })
+
+  describe('matchesMetadataFilter', () => {
+    it('should match simple equality filters', () => {
+      const metadata = { level: 'senior', location: 'SF' }
+      
+      expect(matchesMetadataFilter(metadata, { level: 'senior' })).toBe(true)
+      expect(matchesMetadataFilter(metadata, { level: 'junior' })).toBe(false)
+      expect(matchesMetadataFilter(metadata, { level: 'senior', location: 'SF' })).toBe(true)
+      expect(matchesMetadataFilter(metadata, { level: 'senior', location: 'NYC' })).toBe(false)
+    })
+
+    it('should support MongoDB-style operators', () => {
+      const metadata = { age: 30, skills: ['React', 'Vue'], name: 'John' }
+      
+      // $gt, $gte, $lt, $lte
+      expect(matchesMetadataFilter(metadata, { age: { $gt: 25 } })).toBe(true)
+      expect(matchesMetadataFilter(metadata, { age: { $lt: 25 } })).toBe(false)
+      expect(matchesMetadataFilter(metadata, { age: { $gte: 30 } })).toBe(true)
+      expect(matchesMetadataFilter(metadata, { age: { $lte: 30 } })).toBe(true)
+      
+      // $in, $nin
+      expect(matchesMetadataFilter(metadata, { age: { $in: [25, 30, 35] } })).toBe(true)
+      expect(matchesMetadataFilter(metadata, { age: { $nin: [25, 35] } })).toBe(true)
+      expect(matchesMetadataFilter(metadata, { age: { $nin: [30] } })).toBe(false)
+      
+      // $includes for arrays
+      expect(matchesMetadataFilter(metadata, { skills: { $includes: 'React' } })).toBe(true)
+      expect(matchesMetadataFilter(metadata, { skills: { $includes: 'Angular' } })).toBe(false)
+      
+      // $regex
+      expect(matchesMetadataFilter(metadata, { name: { $regex: '^Jo' } })).toBe(true)
+      expect(matchesMetadataFilter(metadata, { name: { $regex: 'hn$' } })).toBe(true)
+      expect(matchesMetadataFilter(metadata, { name: { $regex: 'Jane' } })).toBe(false)
+    })
+
+    it('should support nested fields with dot notation', () => {
+      const metadata = {
+        user: {
+          profile: {
+            level: 'senior',
+            skills: ['React', 'TypeScript']
+          }
+        }
+      }
+      
+      expect(matchesMetadataFilter(metadata, { 'user.profile.level': 'senior' })).toBe(true)
+      expect(matchesMetadataFilter(metadata, { 'user.profile.level': 'junior' })).toBe(false)
+      expect(matchesMetadataFilter(metadata, { 
+        'user.profile.skills': { $includes: 'React' } 
+      })).toBe(true)
+    })
+
+    it('should support logical operators', () => {
+      const metadata = { level: 'senior', location: 'SF', remote: true }
+      
+      // $and
+      expect(matchesMetadataFilter(metadata, {
+        $and: [
+          { level: 'senior' },
+          { location: 'SF' }
+        ]
+      })).toBe(true)
+      
+      expect(matchesMetadataFilter(metadata, {
+        $and: [
+          { level: 'senior' },
+          { location: 'NYC' }
+        ]
+      })).toBe(false)
+      
+      // $or
+      expect(matchesMetadataFilter(metadata, {
+        $or: [
+          { location: 'NYC' },
+          { location: 'SF' }
+        ]
+      })).toBe(true)
+      
+      expect(matchesMetadataFilter(metadata, {
+        $or: [
+          { location: 'NYC' },
+          { location: 'LA' }
+        ]
+      })).toBe(false)
+      
+      // $not
+      expect(matchesMetadataFilter(metadata, {
+        $not: { location: 'NYC' }
+      })).toBe(true)
+      
+      expect(matchesMetadataFilter(metadata, {
+        $not: { location: 'SF' }
+      })).toBe(false)
+    })
+  })
+
+  describe('Search with metadata filtering', () => {
+    beforeEach(async () => {
+      // Add test data
+      const developers = [
+        { name: 'Alice', level: 'senior', skills: ['React', 'TypeScript'], location: 'SF', available: true },
+        { name: 'Bob', level: 'mid', skills: ['Vue', 'JavaScript'], location: 'NYC', available: true },
+        { name: 'Charlie', level: 'senior', skills: ['React', 'Python'], location: 'SF', available: false },
+        { name: 'David', level: 'junior', skills: ['JavaScript'], location: 'LA', available: true },
+        { name: 'Eve', level: 'senior', skills: ['Angular', 'TypeScript'], location: 'NYC', available: true }
+      ]
+      
+      for (const dev of developers) {
+        await brainy.add(
+          `${dev.name} is a ${dev.level} developer with ${dev.skills.join(', ')} skills in ${dev.location}`,
+          dev
+        )
+      }
+    })
+
+    it('should filter by simple metadata fields', async () => {
+      // First check what we have without filter
+      const allResults = await brainy.searchText('developer', 10)
+      console.log('All results:', allResults.map(r => ({ 
+        id: r.id.substring(0, 8), 
+        level: r.metadata?.level,
+        name: r.metadata?.name 
+      })))
+      
+      // Now with filter
+      const results = await brainy.searchText('developer', 10, {
+        metadata: { level: 'senior' }
+      })
+      
+      console.log('Filtered results:', results.map(r => ({ 
+        id: r.id.substring(0, 8), 
+        level: r.metadata?.level,
+        name: r.metadata?.name 
+      })))
+      
+      expect(results.length).toBeGreaterThan(0)
+      expect(results.every(r => r.metadata?.level === 'senior')).toBe(true)
+    })
+
+    it('should filter by multiple metadata fields', async () => {
+      const results = await brainy.searchText('developer', 10, {
+        metadata: { 
+          level: 'senior',
+          location: 'SF'
+        }
+      })
+      
+      expect(results.length).toBeGreaterThan(0)
+      expect(results.every(r => 
+        r.metadata?.level === 'senior' && 
+        r.metadata?.location === 'SF'
+      )).toBe(true)
+    })
+
+    it('should filter with MongoDB operators', async () => {
+      const results = await brainy.searchText('developer', 10, {
+        metadata: {
+          skills: { $includes: 'React' },
+          available: true
+        }
+      })
+      
+      expect(results.length).toBeGreaterThan(0)
+      expect(results.every(r => 
+        r.metadata?.skills?.includes('React') && 
+        r.metadata?.available === true
+      )).toBe(true)
+    })
+
+    it('should filter with complex queries', async () => {
+      const results = await brainy.searchText('developer', 10, {
+        metadata: {
+          $or: [
+            { location: 'SF' },
+            { location: 'NYC' }
+          ],
+          level: { $in: ['senior', 'mid'] }
+        }
+      })
+      
+      expect(results.length).toBeGreaterThan(0)
+      expect(results.every(r => {
+        const m = r.metadata
+        return (m?.location === 'SF' || m?.location === 'NYC') &&
+               (m?.level === 'senior' || m?.level === 'mid')
+      })).toBe(true)
+    })
+  })
+
+  describe('searchWithinItems', () => {
+    let itemIds: string[] = []
+    
+    beforeEach(async () => {
+      // Add test data and collect IDs
+      const items = [
+        { content: 'JavaScript programming', category: 'tech' },
+        { content: 'TypeScript development', category: 'tech' },
+        { content: 'Python data science', category: 'tech' },
+        { content: 'React components', category: 'frontend' },
+        { content: 'Vue templates', category: 'frontend' }
+      ]
+      
+      for (const item of items) {
+        const id = await brainy.add(item.content, item)
+        if (item.category === 'frontend') {
+          itemIds.push(id)
+        }
+      }
+    })
+
+    it('should search only within specified items', async () => {
+      // Search within frontend items only
+      const results = await brainy.searchWithinItems('JavaScript', itemIds, 5)
+      
+      expect(results.length).toBeLessThanOrEqual(itemIds.length)
+      expect(results.every(r => itemIds.includes(r.id))).toBe(true)
+    })
+
+    it('should return empty results if no items match', async () => {
+      const results = await brainy.searchWithinItems('JavaScript', [], 5)
+      expect(results).toEqual([])
+    })
+
+    it('should limit results to k even if more items are provided', async () => {
+      const results = await brainy.searchWithinItems('development', itemIds, 1)
+      expect(results.length).toBe(1)
+    })
+  })
+})

tests/metadata-performance.test.ts

@@ -0,0 +1,568 @@
+/**
+ * Metadata Filtering Performance Analysis
+ * 
+ * This test suite analyzes the performance impact of the metadata filtering system:
+ * 1. Index Build Time - How metadata indexing affects initialization
+ * 2. Index Storage Overhead - Storage space required for inverted indexes
+ * 3. Search Performance - Filtered vs non-filtered search speeds
+ * 4. Memory Usage - Additional memory needed for metadata indexes
+ * 5. Write Performance - Impact on add/update/delete operations
+ */
+
+import { describe, it, expect, beforeEach } from 'vitest'
+import { BrainyData } from '../src/brainyData.js'
+import { MetadataIndexManager } from '../src/utils/metadataIndex.js'
+
+// Helper function to measure execution time
+const measureTime = async (fn: () => Promise<any>): Promise<{ result: any, time: number }> => {
+  const start = performance.now()
+  const result = await fn()
+  const end = performance.now()
+  return { result, time: end - start }
+}
+
+// Helper function to estimate memory usage
+const measureMemory = () => {
+  if (typeof performance.memory !== 'undefined') {
+    return {
+      used: performance.memory.usedJSHeapSize,
+      total: performance.memory.totalJSHeapSize,
+      limit: performance.memory.jsHeapSizeLimit
+    }
+  }
+  return null
+}
+
+// Generate realistic test data with metadata
+const generateTestDataWithMetadata = (count: number) => {
+  const departments = ['Engineering', 'Marketing', 'Sales', 'HR', 'Finance', 'Operations']
+  const levels = ['junior', 'senior', 'staff', 'principal', 'director']
+  const locations = ['SF', 'NYC', 'LA', 'Seattle', 'Austin', 'Boston']
+  const skills = ['JavaScript', 'Python', 'React', 'Node.js', 'TypeScript', 'SQL', 'AWS', 'Docker']
+  const companies = ['TechCorp', 'DataSys', 'CloudInc', 'DevTools', 'AILabs']
+
+  return Array.from({ length: count }, (_, i) => ({
+    text: `Profile ${i}: Professional with extensive experience in software development and team leadership`,
+    metadata: {
+      id: `profile-${i}`,
+      department: departments[i % departments.length],
+      level: levels[i % levels.length],
+      location: locations[i % locations.length],
+      salary: 50000 + (i % 10) * 10000,
+      experience: 1 + (i % 15),
+      skills: skills.slice(0, 2 + (i % 4)),
+      company: companies[i % companies.length],
+      remote: i % 3 === 0,
+      active: i % 5 !== 0,
+      tags: [`tag-${i % 20}`, `category-${i % 10}`],
+      nested: {
+        profile: {
+          rating: 1 + (i % 5),
+          verified: i % 4 === 0
+        },
+        preferences: {
+          timezone: `UTC-${(i % 12) - 6}`,
+          workStyle: i % 2 === 0 ? 'collaborative' : 'independent'
+        }
+      }
+    }
+  }))
+}
+
+describe('Metadata Filtering Performance Analysis', () => {
+  describe('1. Index Build Time Impact', () => {
+    it('should measure initialization time with vs without metadata indexing', async () => {
+      const testData = generateTestDataWithMetadata(500)
+      
+      console.log('\n=== Index Build Time Analysis ===')
+      
+      // Test WITHOUT metadata indexing
+      const withoutIndexing = await measureTime(async () => {
+        const brainy = new BrainyData({
+          storage: { forceMemoryStorage: true },
+          hnsw: { M: 8, efConstruction: 50 },
+          logging: { verbose: false }
+          // No metadataIndex config
+        })
+        await brainy.init()
+        
+        // Add data
+        for (const item of testData) {
+          await brainy.add(item.text, item.metadata)
+        }
+        
+        return brainy
+      })
+      
+      console.log(`WITHOUT indexing: ${withoutIndexing.time.toFixed(2)}ms for 500 items`)
+      console.log(`Per item: ${(withoutIndexing.time / 500).toFixed(2)}ms`)
+      
+      // Test WITH metadata indexing
+      const withIndexing = await measureTime(async () => {
+        const brainy = new BrainyData({
+          storage: { forceMemoryStorage: true },
+          hnsw: { M: 8, efConstruction: 50 },
+          logging: { verbose: false },
+          metadataIndex: {
+            maxIndexSize: 10000,
+            autoOptimize: true,
+            excludeFields: ['id']
+          }
+        })
+        await brainy.init()
+        
+        // Add data
+        for (const item of testData) {
+          await brainy.add(item.text, item.metadata)
+        }
+        
+        return brainy
+      })
+      
+      console.log(`WITH indexing: ${withIndexing.time.toFixed(2)}ms for 500 items`)
+      console.log(`Per item: ${(withIndexing.time / 500).toFixed(2)}ms`)
+      
+      const overhead = ((withIndexing.time - withoutIndexing.time) / withoutIndexing.time) * 100
+      console.log(`Index build overhead: ${overhead.toFixed(1)}%`)
+      
+      // Cleanup
+      await withoutIndexing.result.shutDown()
+      await withIndexing.result.shutDown()
+    })
+    
+    it('should measure batch insert performance with indexing', async () => {
+      const brainy = new BrainyData({
+        storage: { forceMemoryStorage: true },
+        metadataIndex: { autoOptimize: true },
+        logging: { verbose: false }
+      })
+      await brainy.init()
+      
+      const batchSizes = [50, 100, 200, 500]
+      console.log('\n=== Batch Insert Performance ===')
+      
+      for (const size of batchSizes) {
+        const testData = generateTestDataWithMetadata(size)
+        
+        const { time } = await measureTime(async () => {
+          for (const item of testData) {
+            await brainy.add(item.text, item.metadata)
+          }
+        })
+        
+        console.log(`${size} items: ${time.toFixed(2)}ms (${(time / size).toFixed(2)}ms per item)`)
+        
+        // Clear for next batch
+        await brainy.clear()
+      }
+      
+      await brainy.shutDown()
+    })
+  })
+
+  describe('2. Index Storage Overhead', () => {
+    it('should analyze storage requirements for metadata indexes', async () => {
+      const brainy = new BrainyData({
+        storage: { forceMemoryStorage: true },
+        metadataIndex: { autoOptimize: true },
+        logging: { verbose: false }
+      })
+      await brainy.init()
+      
+      const testData = generateTestDataWithMetadata(1000)
+      
+      console.log('\n=== Storage Overhead Analysis ===')
+      
+      // Add data and measure index size
+      for (const item of testData) {
+        await brainy.addNoun(item.text, item.metadata)
+      }
+      
+      // Get index statistics
+      if (brainy.metadataIndex) {
+        const stats = await brainy.metadataIndex.getStats()
+        console.log(`Total index entries: ${stats.totalEntries}`)
+        console.log(`Total indexed IDs: ${stats.totalIds}`)
+        console.log(`Fields indexed: ${stats.fieldsIndexed.length}`)
+        console.log(`Estimated index size: ${stats.indexSize} bytes`)
+        console.log(`Fields: ${stats.fieldsIndexed.join(', ')}`)
+        
+        // Calculate overhead per item
+        const overheadPerItem = stats.indexSize / 1000
+        console.log(`Storage overhead per item: ${overheadPerItem.toFixed(2)} bytes`)
+        
+        // Estimate total storage efficiency
+        const totalDataSize = 1000 * 200 // rough estimate of 200 bytes per item
+        const storageEfficiency = (stats.indexSize / totalDataSize) * 100
+        console.log(`Index storage overhead: ${storageEfficiency.toFixed(1)}% of data size`)
+      }
+      
+      await brainy.shutDown()
+    })
+  })
+
+  describe('3. Search Performance Comparison', () => {
+    it('should compare filtered vs non-filtered search performance', async () => {
+      const brainy = new BrainyData({
+        storage: { forceMemoryStorage: true },
+        metadataIndex: { autoOptimize: true },
+        logging: { verbose: false }
+      })
+      await brainy.init()
+      
+      // Add test data
+      const testData = generateTestDataWithMetadata(1000)
+      for (const item of testData) {
+        await brainy.addNoun(item.text, item.metadata)
+      }
+      
+      console.log('\n=== Search Performance Comparison ===')
+      
+      const searchQuery = 'Professional software development experience'
+      const numSearches = 10
+      
+      // Test 1: No filtering
+      const noFilterTimes: number[] = []
+      for (let i = 0; i < numSearches; i++) {
+        const { time } = await measureTime(async () => {
+          return await brainy.search(searchQuery, 20)
+        })
+        noFilterTimes.push(time)
+      }
+      const avgNoFilter = noFilterTimes.reduce((a, b) => a + b) / numSearches
+      console.log(`No filtering: ${avgNoFilter.toFixed(2)}ms average`)
+      
+      // Test 2: Simple metadata filtering (high selectivity)
+      const simpleFilterTimes: number[] = []
+      for (let i = 0; i < numSearches; i++) {
+        const { time } = await measureTime(async () => {
+          return await brainy.search(searchQuery, 20, {
+            metadata: { department: 'Engineering' }
+          })
+        })
+        simpleFilterTimes.push(time)
+      }
+      const avgSimpleFilter = simpleFilterTimes.reduce((a, b) => a + b) / numSearches
+      console.log(`Simple filter (dept=Engineering): ${avgSimpleFilter.toFixed(2)}ms average`)
+      
+      // Test 3: Complex metadata filtering (low selectivity)
+      const complexFilterTimes: number[] = []
+      for (let i = 0; i < numSearches; i++) {
+        const { time } = await measureTime(async () => {
+          return await brainy.search(searchQuery, 20, {
+            metadata: {
+              department: { $in: ['Engineering', 'Marketing'] },
+              level: { $in: ['senior', 'staff'] },
+              salary: { $gte: 80000 },
+              remote: true
+            }
+          })
+        })
+        complexFilterTimes.push(time)
+      }
+      const avgComplexFilter = complexFilterTimes.reduce((a, b) => a + b) / numSearches
+      console.log(`Complex filter: ${avgComplexFilter.toFixed(2)}ms average`)
+      
+      // Test 4: Nested field filtering
+      const nestedFilterTimes: number[] = []
+      for (let i = 0; i < numSearches; i++) {
+        const { time } = await measureTime(async () => {
+          return await brainy.search(searchQuery, 20, {
+            metadata: {
+              'nested.profile.rating': { $gte: 4 },
+              'nested.profile.verified': true
+            }
+          })
+        })
+        nestedFilterTimes.push(time)
+      }
+      const avgNestedFilter = nestedFilterTimes.reduce((a, b) => a + b) / numSearches
+      console.log(`Nested filter: ${avgNestedFilter.toFixed(2)}ms average`)
+      
+      // Performance analysis
+      console.log('\nPerformance Impact:')
+      console.log(`Simple filter overhead: ${((avgSimpleFilter / avgNoFilter - 1) * 100).toFixed(1)}%`)
+      console.log(`Complex filter overhead: ${((avgComplexFilter / avgNoFilter - 1) * 100).toFixed(1)}%`)
+      console.log(`Nested filter overhead: ${((avgNestedFilter / avgNoFilter - 1) * 100).toFixed(1)}%`)
+      
+      await brainy.shutDown()
+    })
+
+    it('should test search performance with different ef multipliers', async () => {
+      const brainy = new BrainyData({
+        storage: { forceMemoryStorage: true },
+        metadataIndex: { autoOptimize: true },
+        hnsw: { efSearch: 50 }, // Base ef for testing multiplier effect
+        logging: { verbose: false }
+      })
+      await brainy.init()
+      
+      // Add test data
+      const testData = generateTestDataWithMetadata(500)
+      for (const item of testData) {
+        await brainy.addNoun(item.text, item.metadata)
+      }
+      
+      console.log('\n=== EF Multiplier Impact Analysis ===')
+      
+      const searchQuery = 'Professional software development experience'
+      
+      // Test with different selectivity filters
+      const filters = [
+        { name: 'High selectivity', filter: { department: 'Engineering' }, expected: '~17%' },
+        { name: 'Medium selectivity', filter: { level: { $in: ['senior', 'staff'] } }, expected: '~40%' },
+        { name: 'Low selectivity', filter: { active: true }, expected: '~80%' }
+      ]
+      
+      for (const { name, filter, expected } of filters) {
+        const { result, time } = await measureTime(async () => {
+          return await brainy.search(searchQuery, 10, { metadata: filter })
+        })
+        
+        console.log(`${name} (${expected}): ${time.toFixed(2)}ms, ${result.length} results`)
+      }
+      
+      await brainy.shutDown()
+    })
+  })
+
+  describe('4. Memory Usage Analysis', () => {
+    it('should measure memory consumption of metadata indexes', async () => {
+      if (!measureMemory()) {
+        console.log('\nMemory measurement not available in this environment')
+        return
+      }
+      
+      console.log('\n=== Memory Usage Analysis ===')
+      
+      const initialMemory = measureMemory()!
+      console.log(`Initial memory: ${(initialMemory.used / 1024 / 1024).toFixed(2)}MB`)
+      
+      const brainy = new BrainyData({
+        storage: { forceMemoryStorage: true },
+        metadataIndex: { autoOptimize: true },
+        logging: { verbose: false }
+      })
+      await brainy.init()
+      
+      const afterInitMemory = measureMemory()!
+      console.log(`After init: ${(afterInitMemory.used / 1024 / 1024).toFixed(2)}MB`)
+      
+      // Add data in batches and measure memory growth
+      const batchSize = 100
+      const numBatches = 5
+      
+      for (let batch = 1; batch <= numBatches; batch++) {
+        const testData = generateTestDataWithMetadata(batchSize)
+        
+        for (const item of testData) {
+          await brainy.add(item.text, item.metadata)
+        }
+        
+        const currentMemory = measureMemory()!
+        const totalItems = batch * batchSize
+        console.log(`${totalItems} items: ${(currentMemory.used / 1024 / 1024).toFixed(2)}MB`)
+      }
+      
+      // Get final index stats
+      if (brainy.metadataIndex) {
+        const stats = await brainy.metadataIndex.getStats()
+        console.log(`Index entries: ${stats.totalEntries}, Memory per entry: ${((measureMemory()!.used - initialMemory.used) / stats.totalEntries).toFixed(2)} bytes`)
+      }
+      
+      await brainy.shutDown()
+    })
+  })
+
+  describe('5. Write Performance Impact', () => {
+    it('should measure add/update/delete performance with indexing', async () => {
+      const brainy = new BrainyData({
+        storage: { forceMemoryStorage: true },
+        metadataIndex: { autoOptimize: true },
+        logging: { verbose: false }
+      })
+      await brainy.init()
+      
+      console.log('\n=== Write Performance Analysis ===')
+      
+      // Test ADD performance
+      const testData = generateTestDataWithMetadata(200)
+      const { time: addTime } = await measureTime(async () => {
+        for (const item of testData) {
+          await brainy.add(item.text, item.metadata)
+        }
+      })
+      console.log(`ADD: 200 items in ${addTime.toFixed(2)}ms (${(addTime / 200).toFixed(2)}ms per item)`)
+      
+      // Test UPDATE performance
+      const updateData = testData.slice(0, 50).map((item, i) => ({
+        ...item,
+        metadata: {
+          ...item.metadata,
+          level: 'updated-level',
+          salary: item.metadata.salary + 10000,
+          updateCount: i
+        }
+      }))
+      
+      const { time: updateTime } = await measureTime(async () => {
+        for (const item of updateData) {
+          await brainy.updateMetadata(item.metadata.id, item.metadata)
+        }
+      })
+      console.log(`UPDATE: 50 items in ${updateTime.toFixed(2)}ms (${(updateTime / 50).toFixed(2)}ms per item)`)
+      
+      // Test DELETE performance
+      const idsToDelete = testData.slice(100, 150).map(item => item.metadata.id)
+      const { time: deleteTime } = await measureTime(async () => {
+        for (const id of idsToDelete) {
+          await brainy.delete(id)
+        }
+      })
+      console.log(`DELETE: 50 items in ${deleteTime.toFixed(2)}ms (${(deleteTime / 50).toFixed(2)}ms per item)`)
+      
+      // Verify index consistency
+      if (brainy.metadataIndex) {
+        const stats = await brainy.metadataIndex.getStats()
+        console.log(`Final index state: ${stats.totalEntries} entries, ${stats.totalIds} IDs`)
+        
+        // Should have 150 items remaining (200 - 50 deleted)
+        const expectedItems = 200 - 50
+        const actualItems = await brainy.size()
+        console.log(`Data consistency: ${actualItems}/${expectedItems} items remaining`)
+      }
+      
+      await brainy.shutDown()
+    })
+
+    it('should test concurrent write performance', async () => {
+      const brainy = new BrainyData({
+        storage: { forceMemoryStorage: true },
+        metadataIndex: { autoOptimize: true },
+        logging: { verbose: false }
+      })
+      await brainy.init()
+      
+      console.log('\n=== Concurrent Write Performance ===')
+      
+      const testData = generateTestDataWithMetadata(100)
+      
+      // Sequential writes
+      const { time: sequentialTime } = await measureTime(async () => {
+        for (const item of testData) {
+          await brainy.add(item.text, item.metadata)
+        }
+      })
+      
+      await brainy.clear()
+      
+      // Concurrent writes (batched)
+      const batchSize = 20
+      const { time: concurrentTime } = await measureTime(async () => {
+        const promises: Promise<any>[] = []
+        
+        for (let i = 0; i < testData.length; i += batchSize) {
+          const batch = testData.slice(i, i + batchSize)
+          promises.push(
+            Promise.all(batch.map(item => brainy.add(item.text, item.metadata)))
+          )
+        }
+        
+        await Promise.all(promises)
+      })
+      
+      console.log(`Sequential: ${sequentialTime.toFixed(2)}ms`)
+      console.log(`Concurrent (batched): ${concurrentTime.toFixed(2)}ms`)
+      console.log(`Speedup: ${(sequentialTime / concurrentTime).toFixed(2)}x`)
+      
+      await brainy.shutDown()
+    })
+  })
+
+  describe('6. Index Maintenance and Optimization', () => {
+    it('should analyze index rebuild performance', async () => {
+      const brainy = new BrainyData({
+        storage: { forceMemoryStorage: true },
+        metadataIndex: { 
+          autoOptimize: true,
+          rebuildThreshold: 0.1 
+        },
+        logging: { verbose: false }
+      })
+      await brainy.init()
+      
+      console.log('\n=== Index Maintenance Analysis ===')
+      
+      // Add initial data
+      const testData = generateTestDataWithMetadata(300)
+      for (const item of testData) {
+        await brainy.addNoun(item.text, item.metadata)
+      }
+      
+      // Measure manual rebuild
+      if (brainy.metadataIndex) {
+        const { time: rebuildTime } = await measureTime(async () => {
+          await brainy.metadataIndex!.rebuild()
+        })
+        
+        const stats = await brainy.metadataIndex.getStats()
+        console.log(`Rebuild: ${rebuildTime.toFixed(2)}ms for ${stats.totalEntries} entries`)
+        console.log(`Per entry: ${(rebuildTime / stats.totalEntries).toFixed(2)}ms`)
+        
+        // Test flush performance
+        const { time: flushTime } = await measureTime(async () => {
+          await brainy.metadataIndex!.flush()
+        })
+        console.log(`Flush: ${flushTime.toFixed(2)}ms`)
+      }
+      
+      await brainy.shutDown()
+    })
+
+    it('should test index cache performance', async () => {
+      const brainy = new BrainyData({
+        storage: { forceMemoryStorage: true },
+        metadataIndex: { 
+          maxIndexSize: 1000,
+          autoOptimize: true 
+        },
+        logging: { verbose: false }
+      })
+      await brainy.init()
+      
+      console.log('\n=== Index Cache Performance ===')
+      
+      // Add test data
+      const testData = generateTestDataWithMetadata(200)
+      for (const item of testData) {
+        await brainy.addNoun(item.text, item.metadata)
+      }
+      
+      if (!brainy.metadataIndex) return
+      
+      // Test cache hit performance (repeated queries)
+      const filter = { department: 'Engineering' }
+      
+      // First query (cache miss)
+      const { time: cacheMissTime } = await measureTime(async () => {
+        return await brainy.metadataIndex!.getIdsForCriteria(filter)
+      })
+      
+      // Subsequent queries (cache hits)
+      const cacheHitTimes: number[] = []
+      for (let i = 0; i < 10; i++) {
+        const { time } = await measureTime(async () => {
+          return await brainy.metadataIndex!.getIdsForCriteria(filter)
+        })
+        cacheHitTimes.push(time)
+      }
+      
+      const avgCacheHit = cacheHitTimes.reduce((a, b) => a + b) / cacheHitTimes.length
+      console.log(`Cache miss: ${cacheMissTime.toFixed(2)}ms`)
+      console.log(`Cache hit (avg): ${avgCacheHit.toFixed(2)}ms`)
+      console.log(`Cache speedup: ${(cacheMissTime / avgCacheHit).toFixed(2)}x`)
+      
+      await brainy.shutDown()
+    })
+  })
+})

tests/simple-metadata-test.ts

@@ -0,0 +1,51 @@
+// Simple standalone test to check metadata filtering
+import { BrainyData } from '../dist/brainyData.js'
+
+async function testMetadataFiltering() {
+  console.log('Creating BrainyData instance...')
+  const brainy = new BrainyData({
+    storage: { forceMemoryStorage: true },
+    hnsw: { M: 4, efConstruction: 20 },
+    logging: { verbose: true }
+  })
+  
+  console.log('Initializing...')
+  await brainy.init()
+  
+  console.log('Adding test data...')
+  await brainy.add('Senior developer Alice', { level: 'senior', name: 'Alice' })
+  await brainy.add('Junior developer Bob', { level: 'junior', name: 'Bob' })
+  
+  console.log('Searching without filter...')
+  const allResults = await brainy.searchText('developer', 10)
+  console.log('All results:', allResults.map(r => ({
+    metadata: r.metadata,
+    score: r.score.toFixed(3)
+  })))
+  
+  // Check if metadata index is available
+  console.log('Metadata index available?', !!brainy.metadataIndex)
+  
+  console.log('Searching with metadata filter...')
+  const seniorResults = await brainy.searchText('developer', 10, {
+    metadata: { level: 'senior' }
+  })
+  console.log('Senior results:', seniorResults.map(r => ({
+    metadata: r.metadata,
+    score: r.score.toFixed(3)
+  })))
+  
+  if (brainy.metadataIndex) {
+    console.log('Checking metadata index for level:senior...')
+    const levelSeniorIds = await brainy.metadataIndex.getIds('level', 'senior')
+    console.log('IDs with level=senior from index:', levelSeniorIds)
+    
+    const levelJuniorIds = await brainy.metadataIndex.getIds('level', 'junior')  
+    console.log('IDs with level=junior from index:', levelJuniorIds)
+  }
+  
+  console.log(`\nResults: All=${allResults.length}, Senior=${seniorResults.length}`)
+  console.log('Filter working?', seniorResults.length < allResults.length)
+}
+
+testMetadataFiltering().catch(console.error)