open-brainy/tests/unit/indexes/columnStore/column-store-mixed-kind.test.ts

/**
 * @module column-store-mixed-kind.test
 * @description Typed posting lists: one field, several value KINDS, each
 * answerable on its own.
 *
 * The behaviour these pin replaced a first-writer type freeze. The first value
 * a field ever saw fixed that field's type; every later value of another kind
 * was coerced to it, and when coercion failed — `Number('electronics')` — the
 * value was dropped from the index with no error at all. The row stayed
 * readable by id and by vector and vanished from every equality filter on the
 * field. These tests therefore care about ORDER: strings-then-numbers and
 * numbers-then-strings have to behave identically, because neither writer owns
 * the field.
 *
 * Kinds never coerce into one another at query time either. `5` and `'5'` are
 * different values and match different rows.
 */

import { describe, it, expect, beforeEach, afterEach } from 'vitest'
import { ColumnStore } from '../../../../src/indexes/columnStore/ColumnStore.js'
import { MemoryStorage } from '../../../../src/storage/adapters/memoryStorage.js'
import { EntityIdMapper } from '../../../../src/utils/entityIdMapper.js'

describe('ColumnStore — typed posting lists per (field, kind)', () => {
  let storage: MemoryStorage
  let idMapper: EntityIdMapper
  let store: ColumnStore

  beforeEach(async () => {
    storage = new MemoryStorage()
    await storage.init()
    idMapper = new EntityIdMapper({ storage, storageKey: 'test:idMapper' })
    await idMapper.init()

    store = new ColumnStore({ flushThreshold: 10 })
    await store.init(storage, idMapper)
  })

  afterEach(async () => {
    await store.close()
  })

  /** Resolve a filter to the sorted UUIDs it matched. */
  const uuidsOf = async (field: string, value: unknown): Promise<string[]> => {
    const bitmap = await store.filter(field, value)
    return Array.from(bitmap)
      .map((id) => idMapper.getUuid(Number(id)))
      .filter((u): u is string => u !== undefined)
      .sort()
  }

  describe('equality answers on the query value’s own kind', () => {
    it('serves numbers written AFTER strings on the same field', async () => {
      store.addEntity(BigInt(idMapper.getOrAssign('s1')), { category: 'electronics' })
      store.addEntity(BigInt(idMapper.getOrAssign('s2')), { category: 'furniture' })
      store.addEntity(BigInt(idMapper.getOrAssign('n1')), { category: 5 })
      store.addEntity(BigInt(idMapper.getOrAssign('n2')), { category: 5 })
      store.addEntity(BigInt(idMapper.getOrAssign('n3')), { category: 7 })

      // The numbers are in the index, though a string got there first.
      expect(await uuidsOf('category', 5)).toEqual(['n1', 'n2'])
      expect(await uuidsOf('category', 7)).toEqual(['n3'])
      // And the strings did not move.
      expect(await uuidsOf('category', 'electronics')).toEqual(['s1'])
      expect(await uuidsOf('category', 'furniture')).toEqual(['s2'])
    })

    it('serves strings written AFTER numbers on the same field', async () => {
      store.addEntity(BigInt(idMapper.getOrAssign('n1')), { category: 5 })
      store.addEntity(BigInt(idMapper.getOrAssign('n2')), { category: 5 })
      store.addEntity(BigInt(idMapper.getOrAssign('s1')), { category: 'electronics' })
      store.addEntity(BigInt(idMapper.getOrAssign('s2')), { category: 'electronics' })

      // 'electronics' would have become NaN and been dropped under the freeze.
      expect(await uuidsOf('category', 'electronics')).toEqual(['s1', 's2'])
      expect(await uuidsOf('category', 5)).toEqual(['n1', 'n2'])
    })

    it('does not coerce a number query into the string postings, or back', async () => {
      store.addEntity(BigInt(idMapper.getOrAssign('num')), { code: 5 })
      store.addEntity(BigInt(idMapper.getOrAssign('str')), { code: '5' })

      expect(await uuidsOf('code', 5)).toEqual(['num'])
      expect(await uuidsOf('code', '5')).toEqual(['str'])
    })

    it('serves booleans mixed into a field that already holds strings and numbers', async () => {
      store.addEntity(BigInt(idMapper.getOrAssign('s1')), { flag: 'yes' })
      store.addEntity(BigInt(idMapper.getOrAssign('n1')), { flag: 1 })
      store.addEntity(BigInt(idMapper.getOrAssign('b1')), { flag: true })
      store.addEntity(BigInt(idMapper.getOrAssign('b2')), { flag: false })

      expect(await uuidsOf('flag', true)).toEqual(['b1'])
      expect(await uuidsOf('flag', false)).toEqual(['b2'])
      // `true` stores as 1 internally; that is an encoding, not a value.
      expect(await uuidsOf('flag', 1)).toEqual(['n1'])
      expect(await uuidsOf('flag', 'yes')).toEqual(['s1'])
    })

    it('answers nothing — not something coerced — for a kind the field never held', async () => {
      store.addEntity(BigInt(idMapper.getOrAssign('s1')), { category: 'electronics' })

      expect(await uuidsOf('category', 5)).toEqual([])
      expect(await uuidsOf('category', true)).toEqual([])
    })

    it('holds every kind across a flush, not just the one in the tail buffer', async () => {
      store.addEntity(BigInt(idMapper.getOrAssign('s1')), { category: 'electronics' })
      store.addEntity(BigInt(idMapper.getOrAssign('n1')), { category: 5 })
      await store.flush()
      store.addEntity(BigInt(idMapper.getOrAssign('s2')), { category: 'electronics' })
      store.addEntity(BigInt(idMapper.getOrAssign('n2')), { category: 5 })

      expect(await uuidsOf('category', 'electronics')).toEqual(['s1', 's2'])
      expect(await uuidsOf('category', 5)).toEqual(['n1', 'n2'])
    })
  })

  describe('range filters read the numeric postings', () => {
    it('ranges over the numeric subset of a mixed field, ignoring its strings', async () => {
      store.addEntity(BigInt(idMapper.getOrAssign('cheap')), { price: 100 })
      store.addEntity(BigInt(idMapper.getOrAssign('mid')), { price: 500 })
      store.addEntity(BigInt(idMapper.getOrAssign('dear')), { price: 900 })
      store.addEntity(BigInt(idMapper.getOrAssign('unpriced')), { price: 'on request' })
      await store.flush()

      const inRange = await store.rangeQuery('price', 200, 1000)
      const uuids = Array.from(inRange)
        .map((id) => idMapper.getUuid(Number(id)))
        .sort()
      expect(uuids).toEqual(['dear', 'mid'])
    })

    it('an unbounded range still reports every kind — it is the “has a value” probe', async () => {
      store.addEntity(BigInt(idMapper.getOrAssign('n1')), { mixed: 42 })
      store.addEntity(BigInt(idMapper.getOrAssign('s1')), { mixed: 'text' })
      store.addEntity(BigInt(idMapper.getOrAssign('b1')), { mixed: true })
      await store.flush()

      const anyValue = await store.rangeQuery('mixed')
      const uuids = Array.from(anyValue)
        .map((id) => idMapper.getUuid(Number(id)))
        .sort()
      expect(uuids).toEqual(['b1', 'n1', 's1'])
    })
  })

  describe('the index reports what a field actually holds', () => {
    it('names every kind present, not the one that got there first', async () => {
      store.addEntity(BigInt(idMapper.getOrAssign('s1')), { category: 'electronics' })
      expect(store.getFieldKinds('category')).toEqual(['string'])

      store.addEntity(BigInt(idMapper.getOrAssign('n1')), { category: 5 })
      store.addEntity(BigInt(idMapper.getOrAssign('b1')), { category: true })
      expect(store.getFieldKinds('category')).toEqual(['number', 'string', 'boolean'])

      // And the field is still ONE field by name.
      expect(store.getIndexedFields()).toEqual(['category'])
      expect(store.hasField('category')).toBe(true)
    })

    it('reports an unknown field as holding nothing', () => {
      expect(store.getFieldKinds('never-written')).toEqual([])
    })
  })

  describe('an integer column widens rather than rounding', () => {
    it('keeps a non-integer written after integers as itself', async () => {
      store.addEntity(BigInt(idMapper.getOrAssign('a')), { score: 4 })
      store.addEntity(BigInt(idMapper.getOrAssign('b')), { score: 4.5 })
      store.addEntity(BigInt(idMapper.getOrAssign('c')), { score: 5 })
      await store.flush()

      // 4.5 used to round to 5 and answer `score === 5` alongside c.
      expect(await uuidsOf('score', 4.5)).toEqual(['b'])
      expect(await uuidsOf('score', 5)).toEqual(['c'])
      expect(await uuidsOf('score', 4)).toEqual(['a'])
    })
  })

  describe('close then reopen', () => {
    it('keeps every typed posting, on the same storage', async () => {
      store.addEntity(BigInt(idMapper.getOrAssign('s1')), { category: 'electronics' })
      store.addEntity(BigInt(idMapper.getOrAssign('n1')), { category: 5 })
      store.addEntity(BigInt(idMapper.getOrAssign('b1')), { category: true })
      store.addEntity(BigInt(idMapper.getOrAssign('f1')), { score: 1.5 })
      await store.flush()
      await store.close()

      store = new ColumnStore({ flushThreshold: 10 })
      await store.init(storage, idMapper)

      expect(store.getFieldKinds('category')).toEqual(['number', 'string', 'boolean'])
      expect(await uuidsOf('category', 'electronics')).toEqual(['s1'])
      expect(await uuidsOf('category', 5)).toEqual(['n1'])
      expect(await uuidsOf('category', true)).toEqual(['b1'])
      expect(await uuidsOf('score', 1.5)).toEqual(['f1'])
    })

    it('accepts new values of every kind after the reopen', async () => {
      store.addEntity(BigInt(idMapper.getOrAssign('s1')), { category: 'electronics' })
      store.addEntity(BigInt(idMapper.getOrAssign('n1')), { category: 5 })
      await store.flush()
      await store.close()

      store = new ColumnStore({ flushThreshold: 10 })
      await store.init(storage, idMapper)

      store.addEntity(BigInt(idMapper.getOrAssign('s2')), { category: 'electronics' })
      store.addEntity(BigInt(idMapper.getOrAssign('n2')), { category: 5 })
      store.addEntity(BigInt(idMapper.getOrAssign('b1')), { category: false })
      await store.flush()

      expect(await uuidsOf('category', 'electronics')).toEqual(['s1', 's2'])
      expect(await uuidsOf('category', 5)).toEqual(['n1', 'n2'])
      expect(await uuidsOf('category', false)).toEqual(['b1'])
    })

    it('opens an index written by the pre-typed-postings shape and reads it unchanged', async () => {
      // A single-kind field is byte-identical to what the old writer produced:
      // one manifest at `_column_index/<field>/MANIFEST.json`, no kind
      // subdirectory anywhere. That IS the old on-disk shape, so proving the
      // new reader serves it proves an old index still opens.
      store.addEntity(BigInt(idMapper.getOrAssign('a')), { status: 'active' })
      store.addEntity(BigInt(idMapper.getOrAssign('b')), { status: 'archived' })
      await store.flush()

      const keys = await (storage as unknown as {
        listObjectsUnderPath: (prefix: string) => Promise<string[]>
      }).listObjectsUnderPath('_column_index/')
      expect(keys.some((k) => k.includes('/k/'))).toBe(false)

      await store.close()
      store = new ColumnStore({ flushThreshold: 10 })
      await store.init(storage, idMapper)

      expect(store.getFieldKinds('status')).toEqual(['string'])
      expect(await uuidsOf('status', 'active')).toEqual(['a'])
    })
  })
})