open-brainy/tests/integration/metadata-vector-exclusion.test.ts

/**
 * Integration test for metadata explosion fix
 *
 * Validates that vector embeddings (and array-index-as-object-key payloads) are
 * NEVER indexed as metadata fields, while legitimate small arrays and scalar
 * fields ARE registered for `where` filtering.
 *
 * Original bug class: 825,924 chunk files created for 1,144 entities (721 files
 * per entity) because each of the 384 vector dimensions was being indexed as its
 * own numeric metadata field.
 *
 * 8.0 architecture note: the per-vector-dimension chunk explosion is structurally
 * impossible now — the metadata index is a column store + sparse field index
 * (`_column_index/<field>/…` and `_system/idx/**`), not one `__chunk__*` file per
 * field-value. So this suite no longer counts `_system/__chunk__*` files (there
 * are none). Instead it asserts the real invariant directly via the public
 * `getAvailableFields()` surface: no numeric (vector-dimension) field names, no
 * `vector`/`embedding` field, but the legitimate semantic fields ARE indexed.
 *
 * Field placement (8.0 contract — see AddParams JSDoc): `data` is the content that
 * gets embedded; `metadata` holds the structured, queryable fields used in `where`
 * filters. Queryable fields therefore live under `metadata` here.
 */

import { describe, it, expect, beforeEach, afterEach } from 'vitest'
import { Brainy } from '../../src/brainy.js'
import { NounType } from '../../src/types/graphTypes.js'
import { existsSync, rmSync } from 'fs'
import { MetadataArrayTooLargeError, MAX_INDEXED_ARRAY_LENGTH } from '../../src/errors/brainyError.js'

describe('Metadata Vector Exclusion Fix', () => {
  let brainy: Brainy
  const testDir = '/tmp/brainy-metadata-vector-test'

  beforeEach(async () => {
    // Clean test directory
    if (existsSync(testDir)) {
      rmSync(testDir, { recursive: true, force: true })
    }

    brainy = new Brainy({ requireSubtype: false,
      storage: { type: 'filesystem', path: testDir },
      ai: { provider: 'mock' }
    })
    await brainy.init()
  })

  afterEach(async () => {
    if (brainy) {
      await brainy.clear()
      await brainy.close()
    }
    if (existsSync(testDir)) {
      rmSync(testDir, { recursive: true, force: true })
    }
  })

  it('should NOT index vector embeddings as metadata fields', async () => {
    // Add entity with queryable metadata + a small array (tags SHOULD be indexed)
    await brainy.add({
      type: NounType.Person,
      data: 'Alice the developer',
      metadata: {
        name: 'Alice',
        email: 'alice@example.com',
        tags: ['developer', 'typescript']  // Small array - SHOULD be indexed
      }
    })

    // The indexed-field set is the canonical record of what the metadata index
    // actually tracks. The vector explosion bug manifested as hundreds of numeric
    // field names (one per dimension) appearing here.
    const fields = await brainy.getAvailableFields()

    // CRITICAL: no purely-numeric field names (vector dimension indices like
    // "0", "1", "54716") and no nested numeric dimension keys.
    const numericFields = fields.filter(f => /(^|\.)\d+$/.test(f))
    expect(numericFields).toEqual([])

    // CRITICAL: the raw vector must never be indexed as a metadata field.
    expect(fields).not.toContain('vector')
    expect(fields).not.toContain('embedding')
    expect(fields).not.toContain('embeddings')

    // The legitimate semantic fields ARE indexed.
    expect(fields).toContain('name')
    expect(fields).toContain('email')
    expect(fields).toContain('tags')

    // Field count stays sane (semantic fields + Brainy's own system/VFS fields),
    // nowhere near the hundreds a per-dimension explosion would produce.
    expect(fields.length).toBeLessThan(50)
  })

  it('should NOT index objects with numeric keys (array-as-object guard)', async () => {
    // Add entity with an object that has numeric keys (simulates a vector
    // accidentally serialized as { "0": 0.1, "1": 0.2, ... }).
    await brainy.add({
      type: NounType.Person,
      data: 'NumericTest entity',
      metadata: {
        name: 'NumericTest',
        numericObject: {
          '0': 0.1,
          '1': 0.2,
          '2': 0.3,
          '100': 1.0
        }
      }
    })

    const fields = await brainy.getAvailableFields()

    // CRITICAL: numeric keys (whether top-level or nested under a field like
    // "numericObject.0") must NOT become indexed fields. This is the guard that
    // prevents vectors-as-objects from exploding the index.
    const numericFields = fields.filter(f => /(^|\.)\d+$/.test(f))
    expect(numericFields).toEqual([])

    // The legitimate scalar field IS indexed.
    expect(fields).toContain('name')
  })

  it('should still index small arrays (tags, categories)', async () => {
    // Add entity with a small (<= 10 element) array of tags.
    await brainy.add({
      type: NounType.Person,
      data: 'Bob the engineer',
      metadata: {
        name: 'Bob',
        tags: ['javascript', 'react', 'nodejs']
      }
    })

    // Small arrays must be registered as a multi-value field (not skipped, not
    // exploded). 'tags' should appear in the indexed-field set.
    const fields = await brainy.getAvailableFields()
    expect(fields).toContain('tags')

    // Behavioral check: filtering by a tag value should return the entity. The
    // array was ['javascript', 'react', 'nodejs'] — matching ANY member must work.
    //
    // REAL LIBRARY BUG (left failing on purpose): the ColumnStore ingest path in
    // src/utils/metadataIndex.ts (addToIndex, ~line 1407) does
    // `fieldsMap[field] = value` for every extracted pair, so for a multi-value
    // field it keeps only the LAST array element ('nodejs') and drops the rest.
    // Only the '__words__' field gets the array-accumulation treatment. As a
    // result `where: { tags: 'react' }` (and 'javascript') returns [], while
    // `where: { tags: 'nodejs' }` returns 1. Correct input → wrong output, so
    // this assertion stays as-is to keep the bug visible.
    const results = await brainy.find({
      where: { tags: 'react' }
    })

    expect(results.length).toBeGreaterThan(0)
    expect(results[0].entity.metadata?.name).toBe('Bob')
  })

  it('should REFUSE an array over the indexing bound, by name', async () => {
    // A large array (not a vector, just bulk data). This used to be SKIPPED in
    // silence at a bound of 10 — the field simply vanished from the index and
    // the row dropped out of every `where` on it, indistinguishably from "no
    // row matches". The bound is now MAX_INDEXED_ARRAY_LENGTH and it REFUSES.
    const largeArray = Array.from({ length: 100 }, (_, i) => `item${i}`)

    const err = await brainy
      .add({
        type: NounType.Document,
        data: 'Doc with large array',
        metadata: {
          name: 'Doc with large array',
          items: largeArray
        }
      })
      .catch((e: any) => e)

    expect(err).toBeInstanceOf(MetadataArrayTooLargeError)
    expect(err.field).toBe('items')
    expect(err.length).toBe(100)
    expect(err.limit).toBe(MAX_INDEXED_ARRAY_LENGTH)

    // Nothing was indexed from the refused write — no 'items' field, and above
    // all no per-element numeric fields (the original explosion class).
    const fields = await brainy.getAvailableFields()
    expect(fields).not.toContain('items')
    const numericFields = fields.filter(f => /(^|\.)\d+$/.test(f))
    expect(numericFields).toEqual([])
  })

  it('should index an array UP TO the bound — the old limit of 10 was the bug', async () => {
    await brainy.add({
      type: NounType.Document,
      data: 'Doc with a long-but-legitimate tag list',
      metadata: {
        name: 'Doc with many tags',
        items: Array.from({ length: MAX_INDEXED_ARRAY_LENGTH }, (_, i) => `item${i}`)
      }
    })

    const fields = await brainy.getAvailableFields()
    // The field IS indexed now, and still without per-element numeric fields.
    expect(fields).toContain('items')
    expect(fields.filter(f => /(^|\.)\d+$/.test(f))).toEqual([])

    // And the eleventh element — the one the old bound silently dropped the
    // whole field for — really is searchable.
    const hits = await brainy.find({ where: { items: 'item10' } })
    expect(hits.length).toBeGreaterThan(0)
  })

  it('should preserve HNSW vector search functionality', async () => {
    // Add entities with semantic content
    const id1 = await brainy.add({
      type: NounType.Concept,
      data: 'AI algorithms that learn from data',
      metadata: {
        name: 'Machine Learning',
        description: 'AI algorithms that learn from data'
      }
    })

    const id2 = await brainy.add({
      type: NounType.Concept,
      data: 'Neural networks with multiple layers',
      metadata: {
        name: 'Deep Learning',
        description: 'Neural networks with multiple layers'
      }
    })

    // Vectors are metadata-only by default; request them explicitly to verify the
    // embedding pipeline actually produced a stored vector for each entity.
    const entity1 = await brainy.get(id1, { includeVectors: true })
    const entity2 = await brainy.get(id2, { includeVectors: true })

    expect(entity1).toBeDefined()
    expect(entity2).toBeDefined()
    expect(Array.isArray(entity1?.vector)).toBe(true)
    expect(Array.isArray(entity2?.vector)).toBe(true)
    expect(entity1!.vector!.length).toBe(384)
    expect(entity2!.vector!.length).toBe(384)

    // Self-retrieval confirms the HNSW index is wired: querying an entity by its
    // OWN content text returns it (deterministic embedder → cosine 1.0).
    const selfHits = await brainy.find({ query: 'AI algorithms that learn from data', limit: 5 })
    expect(selfHits.length).toBeGreaterThan(0)
    expect(selfHits.some(r => r.id === id1)).toBe(true)
  })

  it('should preserve metadata field filtering', async () => {
    // Add entities with various queryable metadata
    await brainy.add({
      type: NounType.Person,
      data: 'Charlie the engineer',
      metadata: {
        name: 'Charlie',
        email: 'charlie@example.com',
        role: 'engineer'
      }
    })

    await brainy.add({
      type: NounType.Person,
      data: 'Dana the designer',
      metadata: {
        name: 'Dana',
        email: 'dana@example.com',
        role: 'designer'
      }
    })

    // Verify scalar metadata filtering works
    const engineers = await brainy.find({
      where: { role: 'engineer' }
    })

    expect(engineers.length).toBe(1)
    expect(engineers[0].entity.metadata?.name).toBe('Charlie')

    const designers = await brainy.find({
      where: { role: 'designer' }
    })

    expect(designers.length).toBe(1)
    expect(designers[0].entity.metadata?.name).toBe('Dana')
  })

  it('should handle nested object metadata correctly', async () => {
    // Add entity with nested metadata
    await brainy.add({
      type: NounType.Person,
      data: 'Eve in New York',
      metadata: {
        name: 'Eve',
        address: {
          city: 'New York',
          state: 'NY'
        }
      }
    })

    // Nested fields are flattened to dot-notation field names; filter by the
    // flattened path.
    const results = await brainy.find({
      where: { 'address.city': 'New York' }
    })

    expect(results.length).toBeGreaterThan(0)
    expect(results[0].entity.metadata?.name).toBe('Eve')
  })

  it('should NOT create exponential index fields for multiple entities', async () => {
    // Add 10 entities (each with a vector embedding + a few metadata fields)
    for (let i = 0; i < 10; i++) {
      await brainy.add({
        type: NounType.Person,
        data: `Person ${i}`,
        metadata: {
          name: `Person ${i}`,
          email: `person${i}@example.com`,
          tags: ['user']
        }
      })
    }

    // The indexed-field set is keyed by field NAME, not by entity — so it must
    // stay bounded regardless of entity count. The original bug turned this into
    // ~721 fields per entity (one per vector dimension); here it must remain a
    // small handful of semantic fields plus Brainy's own system/VFS fields.
    const fields = await brainy.getAvailableFields()
    const numericFields = fields.filter(f => /(^|\.)\d+$/.test(f))
    expect(numericFields).toEqual([])
    expect(fields).toContain('name')
    expect(fields).toContain('email')
    expect(fields).toContain('tags')
    // Nowhere near the thousands a per-dimension explosion would produce.
    expect(fields.length).toBeLessThan(50)
  })
})