brainy/tests/unit/utils/metadataIndex-text-indexing.test.ts

/**
 * MetadataIndexManager Text Indexing Tests (v7.7.0)
 *
 * Tests for word extraction, tokenization, and hashing used in hybrid search.
 */

import { describe, it, expect, beforeEach, afterEach } from 'vitest'
import { MetadataIndexManager } from '../../../src/utils/metadataIndex'

// Create a test adapter to expose private methods for testing
// This is a common testing pattern to test internal implementation
class TestableMetadataIndexManager extends MetadataIndexManager {
  public testTokenize(text: string): string[] {
    return this.tokenize(text)
  }

  public testHashWord(word: string): number {
    return this.hashWord(word)
  }
}

describe('MetadataIndexManager Text Indexing (v7.7.0)', () => {
  describe('tokenize()', () => {
    let manager: TestableMetadataIndexManager

    beforeEach(() => {
      // Create a minimal mock storage for testing
      const mockStorage = {
        getMetadata: async () => null,
        saveMetadata: async () => {},
        getNoun: async () => null,
        getNouns: async () => ({ items: [], total: 0, hasMore: false }),
        getVerbsFromSource: async () => ({ items: [], total: 0, hasMore: false }),
      } as any

      manager = new TestableMetadataIndexManager(mockStorage, {})
    })

    it('should convert to lowercase', () => {
      const tokens = manager.testTokenize('HELLO World')
      expect(tokens).toContain('hello')
      expect(tokens).toContain('world')
      expect(tokens).not.toContain('HELLO')
      expect(tokens).not.toContain('World')
    })

    it('should remove punctuation', () => {
      const tokens = manager.testTokenize('Hello, World! How are you?')
      expect(tokens).toContain('hello')
      expect(tokens).toContain('world')
      expect(tokens).toContain('how')
      expect(tokens).toContain('are')
      expect(tokens).toContain('you')
    })

    it('should filter short words (< 2 chars)', () => {
      const tokens = manager.testTokenize('I a am is an the to')
      expect(tokens).not.toContain('i')
      expect(tokens).not.toContain('a')
      expect(tokens).toContain('am')
      expect(tokens).toContain('is')
      expect(tokens).toContain('an')
    })

    it('should deduplicate words', () => {
      const tokens = manager.testTokenize('hello hello world world hello')
      expect(tokens.length).toBe(2)
      expect(tokens).toContain('hello')
      expect(tokens).toContain('world')
    })

    it('should handle empty string', () => {
      const tokens = manager.testTokenize('')
      expect(tokens).toEqual([])
    })

    it('should handle whitespace only', () => {
      const tokens = manager.testTokenize('   \t\n   ')
      expect(tokens).toEqual([])
    })

    it('should handle special characters', () => {
      const tokens = manager.testTokenize('C++ is a programming language')
      expect(tokens).toContain('is')
      expect(tokens).toContain('programming')
      expect(tokens).toContain('language')
    })

    it('should handle unicode text', () => {
      const tokens = manager.testTokenize('Hello 世界 Welt')
      expect(tokens).toContain('hello')
      expect(tokens).toContain('welt')
    })

    it('should handle numbers', () => {
      const tokens = manager.testTokenize('version 123 release 45')
      expect(tokens).toContain('version')
      expect(tokens).toContain('123')
      expect(tokens).toContain('release')
      expect(tokens).toContain('45')
    })

    it('should handle hyphenated words', () => {
      const tokens = manager.testTokenize('state-of-the-art machine-learning')
      // Hyphenated words become separate tokens due to punctuation removal
      expect(tokens).toContain('state')
      expect(tokens).toContain('the')
      expect(tokens).toContain('art')
      expect(tokens).toContain('machine')
      expect(tokens).toContain('learning')
    })
  })

  describe('hashWord()', () => {
    let manager: TestableMetadataIndexManager

    beforeEach(() => {
      const mockStorage = {
        getMetadata: async () => null,
        saveMetadata: async () => {},
        getNoun: async () => null,
        getNouns: async () => ({ items: [], total: 0, hasMore: false }),
        getVerbsFromSource: async () => ({ items: [], total: 0, hasMore: false }),
      } as any

      manager = new TestableMetadataIndexManager(mockStorage, {})
    })

    it('should produce consistent hash for same word', () => {
      const hash1 = manager.testHashWord('hello')
      const hash2 = manager.testHashWord('hello')
      expect(hash1).toBe(hash2)
    })

    it('should produce different hashes for different words', () => {
      const hash1 = manager.testHashWord('hello')
      const hash2 = manager.testHashWord('world')
      expect(hash1).not.toBe(hash2)
    })

    it('should produce int32 values', () => {
      const hash = manager.testHashWord('test')
      expect(Number.isInteger(hash)).toBe(true)
      expect(hash).toBeGreaterThanOrEqual(-2147483648)
      expect(hash).toBeLessThanOrEqual(2147483647)
    })

    it('should handle empty string', () => {
      const hash = manager.testHashWord('')
      expect(Number.isInteger(hash)).toBe(true)
    })

    it('should handle long words', () => {
      const longWord = 'supercalifragilisticexpialidocious'
      const hash = manager.testHashWord(longWord)
      expect(Number.isInteger(hash)).toBe(true)
    })

    it('should hash unicode words', () => {
      const hash = manager.testHashWord('世界')
      expect(Number.isInteger(hash)).toBe(true)
    })

    it('should be case sensitive (words are lowercased before hashing in tokenize)', () => {
      const hash1 = manager.testHashWord('Hello')
      const hash2 = manager.testHashWord('hello')
      // Hashes are different because case matters in hash function
      // But tokenize() lowercases before hashing
      expect(hash1).not.toBe(hash2)
    })
  })

  describe('getIdsForTextQuery()', () => {
    let manager: MetadataIndexManager
    let mockStorage: any
    let addedEntities: Map<string, any>

    beforeEach(() => {
      addedEntities = new Map()
      let idCounter = 0

      mockStorage = {
        getMetadata: async () => null,
        saveMetadata: async () => {},
        getNoun: async (id: string) => addedEntities.get(id) || null,
        getNouns: async () => ({
          items: Array.from(addedEntities.values()),
          total: addedEntities.size,
          hasMore: false
        }),
        getVerbsFromSource: async () => ({ items: [], total: 0, hasMore: false }),
      }

      manager = new MetadataIndexManager(mockStorage, {})
    })

    it('should return empty array for empty query', async () => {
      const results = await manager.getIdsForTextQuery('')
      expect(results).toEqual([])
    })

    it('should return empty array for whitespace query', async () => {
      const results = await manager.getIdsForTextQuery('   ')
      expect(results).toEqual([])
    })

    it('should return results sorted by match count', async () => {
      // This test verifies the interface contract
      // Actual matching behavior tested in integration tests
      const results = await manager.getIdsForTextQuery('hello world')
      expect(Array.isArray(results)).toBe(true)
      results.forEach(r => {
        expect(r).toHaveProperty('id')
        expect(r).toHaveProperty('matchCount')
      })
    })
  })

  describe('extractTextContent() array handling (v7.9.0 fix)', () => {
    let manager: MetadataIndexManager

    beforeEach(() => {
      const mockStorage = {
        getMetadata: async () => null,
        saveMetadata: async () => {},
        getNoun: async () => null,
        getNouns: async () => ({ items: [], total: 0, hasMore: false }),
        getVerbsFromSource: async () => ({ items: [], total: 0, hasMore: false }),
      } as any

      manager = new MetadataIndexManager(mockStorage, {})
    })

    it('should skip numeric arrays (vectors/embeddings)', () => {
      const result = manager.extractTextContent([0.1, 0.2, 0.3, 0.4, 0.5])
      expect(result).toBe('')
    })

    it('should extract text from string arrays', () => {
      const result = manager.extractTextContent(['hello', 'world', 'test'])
      expect(result).toContain('hello')
      expect(result).toContain('world')
      expect(result).toContain('test')
    })

    it('should extract text from arrays of objects', () => {
      const result = manager.extractTextContent([
        { name: 'Alice', role: 'engineer' },
        { name: 'Bob', role: 'designer' },
        { name: 'Carol', role: 'manager' },
        { name: 'Dave', role: 'analyst' },
        { name: 'Eve', role: 'scientist' },
        { name: 'Frank', role: 'developer' },
        { name: 'Grace', role: 'architect' },
        { name: 'Heidi', role: 'lead' },
        { name: 'Ivan', role: 'intern' },
        { name: 'Judy', role: 'director' },
        { name: 'Karl', role: 'founder' },
      ])
      // Should NOT skip arrays of objects even with >10 elements
      expect(result).toContain('Alice')
      expect(result).toContain('Karl')
      expect(result).toContain('engineer')
      expect(result).toContain('founder')
    })

    it('should handle large numeric arrays (vectors)', () => {
      const vector = Array.from({ length: 384 }, (_, i) => Math.sin(i))
      const result = manager.extractTextContent(vector)
      expect(result).toBe('')
    })

    it('should handle empty arrays', () => {
      const result = manager.extractTextContent([])
      expect(result).toBe('')
    })

    it('should handle mixed arrays starting with string', () => {
      const result = manager.extractTextContent(['text', 42, true])
      expect(result).toContain('text')
      expect(result).toContain('42')
    })
  })
})