/** * @module column-store-mixed-kind.test * @description Typed posting lists: one field, several value KINDS, each * answerable on its own. * * The behaviour these pin replaced a first-writer type freeze. The first value * a field ever saw fixed that field's type; every later value of another kind * was coerced to it, and when coercion failed — `Number('electronics')` — the * value was dropped from the index with no error at all. The row stayed * readable by id and by vector and vanished from every equality filter on the * field. These tests therefore care about ORDER: strings-then-numbers and * numbers-then-strings have to behave identically, because neither writer owns * the field. * * Kinds never coerce into one another at query time either. `5` and `'5'` are * different values and match different rows. */ import { describe, it, expect, beforeEach, afterEach } from 'vitest' import { ColumnStore } from '../../../../src/indexes/columnStore/ColumnStore.js' import { MemoryStorage } from '../../../../src/storage/adapters/memoryStorage.js' import { EntityIdMapper } from '../../../../src/utils/entityIdMapper.js' describe('ColumnStore — typed posting lists per (field, kind)', () => { let storage: MemoryStorage let idMapper: EntityIdMapper let store: ColumnStore beforeEach(async () => { storage = new MemoryStorage() await storage.init() idMapper = new EntityIdMapper({ storage, storageKey: 'test:idMapper' }) await idMapper.init() store = new ColumnStore({ flushThreshold: 10 }) await store.init(storage, idMapper) }) afterEach(async () => { await store.close() }) /** Resolve a filter to the sorted UUIDs it matched. */ const uuidsOf = async (field: string, value: unknown): Promise => { const bitmap = await store.filter(field, value) return Array.from(bitmap) .map((id) => idMapper.getUuid(Number(id))) .filter((u): u is string => u !== undefined) .sort() } describe('equality answers on the query value’s own kind', () => { it('serves numbers written AFTER strings on the same field', async () => { store.addEntity(BigInt(idMapper.getOrAssign('s1')), { category: 'electronics' }) store.addEntity(BigInt(idMapper.getOrAssign('s2')), { category: 'furniture' }) store.addEntity(BigInt(idMapper.getOrAssign('n1')), { category: 5 }) store.addEntity(BigInt(idMapper.getOrAssign('n2')), { category: 5 }) store.addEntity(BigInt(idMapper.getOrAssign('n3')), { category: 7 }) // The numbers are in the index, though a string got there first. expect(await uuidsOf('category', 5)).toEqual(['n1', 'n2']) expect(await uuidsOf('category', 7)).toEqual(['n3']) // And the strings did not move. expect(await uuidsOf('category', 'electronics')).toEqual(['s1']) expect(await uuidsOf('category', 'furniture')).toEqual(['s2']) }) it('serves strings written AFTER numbers on the same field', async () => { store.addEntity(BigInt(idMapper.getOrAssign('n1')), { category: 5 }) store.addEntity(BigInt(idMapper.getOrAssign('n2')), { category: 5 }) store.addEntity(BigInt(idMapper.getOrAssign('s1')), { category: 'electronics' }) store.addEntity(BigInt(idMapper.getOrAssign('s2')), { category: 'electronics' }) // 'electronics' would have become NaN and been dropped under the freeze. expect(await uuidsOf('category', 'electronics')).toEqual(['s1', 's2']) expect(await uuidsOf('category', 5)).toEqual(['n1', 'n2']) }) it('does not coerce a number query into the string postings, or back', async () => { store.addEntity(BigInt(idMapper.getOrAssign('num')), { code: 5 }) store.addEntity(BigInt(idMapper.getOrAssign('str')), { code: '5' }) expect(await uuidsOf('code', 5)).toEqual(['num']) expect(await uuidsOf('code', '5')).toEqual(['str']) }) it('serves booleans mixed into a field that already holds strings and numbers', async () => { store.addEntity(BigInt(idMapper.getOrAssign('s1')), { flag: 'yes' }) store.addEntity(BigInt(idMapper.getOrAssign('n1')), { flag: 1 }) store.addEntity(BigInt(idMapper.getOrAssign('b1')), { flag: true }) store.addEntity(BigInt(idMapper.getOrAssign('b2')), { flag: false }) expect(await uuidsOf('flag', true)).toEqual(['b1']) expect(await uuidsOf('flag', false)).toEqual(['b2']) // `true` stores as 1 internally; that is an encoding, not a value. expect(await uuidsOf('flag', 1)).toEqual(['n1']) expect(await uuidsOf('flag', 'yes')).toEqual(['s1']) }) it('answers nothing — not something coerced — for a kind the field never held', async () => { store.addEntity(BigInt(idMapper.getOrAssign('s1')), { category: 'electronics' }) expect(await uuidsOf('category', 5)).toEqual([]) expect(await uuidsOf('category', true)).toEqual([]) }) it('holds every kind across a flush, not just the one in the tail buffer', async () => { store.addEntity(BigInt(idMapper.getOrAssign('s1')), { category: 'electronics' }) store.addEntity(BigInt(idMapper.getOrAssign('n1')), { category: 5 }) await store.flush() store.addEntity(BigInt(idMapper.getOrAssign('s2')), { category: 'electronics' }) store.addEntity(BigInt(idMapper.getOrAssign('n2')), { category: 5 }) expect(await uuidsOf('category', 'electronics')).toEqual(['s1', 's2']) expect(await uuidsOf('category', 5)).toEqual(['n1', 'n2']) }) }) describe('range filters read the numeric postings', () => { it('ranges over the numeric subset of a mixed field, ignoring its strings', async () => { store.addEntity(BigInt(idMapper.getOrAssign('cheap')), { price: 100 }) store.addEntity(BigInt(idMapper.getOrAssign('mid')), { price: 500 }) store.addEntity(BigInt(idMapper.getOrAssign('dear')), { price: 900 }) store.addEntity(BigInt(idMapper.getOrAssign('unpriced')), { price: 'on request' }) await store.flush() const inRange = await store.rangeQuery('price', 200, 1000) const uuids = Array.from(inRange) .map((id) => idMapper.getUuid(Number(id))) .sort() expect(uuids).toEqual(['dear', 'mid']) }) it('an unbounded range still reports every kind — it is the “has a value” probe', async () => { store.addEntity(BigInt(idMapper.getOrAssign('n1')), { mixed: 42 }) store.addEntity(BigInt(idMapper.getOrAssign('s1')), { mixed: 'text' }) store.addEntity(BigInt(idMapper.getOrAssign('b1')), { mixed: true }) await store.flush() const anyValue = await store.rangeQuery('mixed') const uuids = Array.from(anyValue) .map((id) => idMapper.getUuid(Number(id))) .sort() expect(uuids).toEqual(['b1', 'n1', 's1']) }) }) describe('the index reports what a field actually holds', () => { it('names every kind present, not the one that got there first', async () => { store.addEntity(BigInt(idMapper.getOrAssign('s1')), { category: 'electronics' }) expect(store.getFieldKinds('category')).toEqual(['string']) store.addEntity(BigInt(idMapper.getOrAssign('n1')), { category: 5 }) store.addEntity(BigInt(idMapper.getOrAssign('b1')), { category: true }) expect(store.getFieldKinds('category')).toEqual(['number', 'string', 'boolean']) // And the field is still ONE field by name. expect(store.getIndexedFields()).toEqual(['category']) expect(store.hasField('category')).toBe(true) }) it('reports an unknown field as holding nothing', () => { expect(store.getFieldKinds('never-written')).toEqual([]) }) }) describe('an integer column widens rather than rounding', () => { it('keeps a non-integer written after integers as itself', async () => { store.addEntity(BigInt(idMapper.getOrAssign('a')), { score: 4 }) store.addEntity(BigInt(idMapper.getOrAssign('b')), { score: 4.5 }) store.addEntity(BigInt(idMapper.getOrAssign('c')), { score: 5 }) await store.flush() // 4.5 used to round to 5 and answer `score === 5` alongside c. expect(await uuidsOf('score', 4.5)).toEqual(['b']) expect(await uuidsOf('score', 5)).toEqual(['c']) expect(await uuidsOf('score', 4)).toEqual(['a']) }) }) describe('close then reopen', () => { it('keeps every typed posting, on the same storage', async () => { store.addEntity(BigInt(idMapper.getOrAssign('s1')), { category: 'electronics' }) store.addEntity(BigInt(idMapper.getOrAssign('n1')), { category: 5 }) store.addEntity(BigInt(idMapper.getOrAssign('b1')), { category: true }) store.addEntity(BigInt(idMapper.getOrAssign('f1')), { score: 1.5 }) await store.flush() await store.close() store = new ColumnStore({ flushThreshold: 10 }) await store.init(storage, idMapper) expect(store.getFieldKinds('category')).toEqual(['number', 'string', 'boolean']) expect(await uuidsOf('category', 'electronics')).toEqual(['s1']) expect(await uuidsOf('category', 5)).toEqual(['n1']) expect(await uuidsOf('category', true)).toEqual(['b1']) expect(await uuidsOf('score', 1.5)).toEqual(['f1']) }) it('accepts new values of every kind after the reopen', async () => { store.addEntity(BigInt(idMapper.getOrAssign('s1')), { category: 'electronics' }) store.addEntity(BigInt(idMapper.getOrAssign('n1')), { category: 5 }) await store.flush() await store.close() store = new ColumnStore({ flushThreshold: 10 }) await store.init(storage, idMapper) store.addEntity(BigInt(idMapper.getOrAssign('s2')), { category: 'electronics' }) store.addEntity(BigInt(idMapper.getOrAssign('n2')), { category: 5 }) store.addEntity(BigInt(idMapper.getOrAssign('b1')), { category: false }) await store.flush() expect(await uuidsOf('category', 'electronics')).toEqual(['s1', 's2']) expect(await uuidsOf('category', 5)).toEqual(['n1', 'n2']) expect(await uuidsOf('category', false)).toEqual(['b1']) }) it('opens an index written by the pre-typed-postings shape and reads it unchanged', async () => { // A single-kind field is byte-identical to what the old writer produced: // one manifest at `_column_index//MANIFEST.json`, no kind // subdirectory anywhere. That IS the old on-disk shape, so proving the // new reader serves it proves an old index still opens. store.addEntity(BigInt(idMapper.getOrAssign('a')), { status: 'active' }) store.addEntity(BigInt(idMapper.getOrAssign('b')), { status: 'archived' }) await store.flush() const keys = await (storage as unknown as { listObjectsUnderPath: (prefix: string) => Promise }).listObjectsUnderPath('_column_index/') expect(keys.some((k) => k.includes('/k/'))).toBe(false) await store.close() store = new ColumnStore({ flushThreshold: 10 }) await store.init(storage, idMapper) expect(store.getFieldKinds('status')).toEqual(['string']) expect(await uuidsOf('status', 'active')).toEqual(['a']) }) }) })