David Snelling
a128f0eda5
The metadata index fixed a field's value type from the first value it saw.
Every later value of another kind was coerced to that type, and when coercion
failed — `Number('electronics')` is NaN — the value was dropped from the index
with no error at all. The row stayed readable by id and by vector search and
vanished only from equality filters on that one field, which is what made it so
quiet: writing `category: 'electronics'` rows and then `category: 5` rows left
`where { category: 5 }` returning nothing, while the same rows in a
numbers-only corpus answered correctly.
The column store now keeps one posting column per (field, kind), where a kind
is a JavaScript typeof class. The first kind a field sees keeps the historical
`_column_index/<field>/` layout, so a single-kind field is byte-identical to
what earlier versions wrote and an index written before this opens unchanged;
each later kind takes its own column at `_column_index/<field>/k/<kind>/`.
Equality reads the column matching the query value's own kind, so `{c: 5}` and
`{c: '5'}` match different rows and neither is coerced into the other. Ranges
route by the kind of their bounds, and an unbounded range — the "has any value"
probe behind `exists` — reads every kind. A mixed field orders by kind first,
then by value, because a number and a string have no order between them. A
value that cannot be encoded for the column its own kind selected now raises
instead of being skipped: that path is unreachable by construction, and if it
is ever reached it is the silent drop this change exists to end.
Two neighbours fell out of the same routing. A boolean query value is now
encoded to the 1/0 the column stores, so boolean equality matches at all. And
an integer column widens to f64 the first time a non-integer arrives, so 4.5 is
stored as itself rather than rounded to 5 and answering the wrong query.
Field type inference reports every kind a field holds beside its dominant
reading, rather than leaving callers to treat one type as the whole answer.
Pins: mixed-kind equality in both write orders, `5` vs `'5'`, booleans mixed in,
a numeric range over a mixed field's numbers, close/reopen keeping every typed
posting, and an index in the pre-existing on-disk shape still reading.
`tests/critical-neural-validation.test.ts` — which writes `category` as strings
in one test and as numbers in another against one shared brain — passes whole
for the first time.
241 lines
11 KiB
TypeScript
241 lines
11 KiB
TypeScript
/**
|
||
* @module column-store-mixed-kind.test
|
||
* @description Typed posting lists: one field, several value KINDS, each
|
||
* answerable on its own.
|
||
*
|
||
* The behaviour these pin replaced a first-writer type freeze. The first value
|
||
* a field ever saw fixed that field's type; every later value of another kind
|
||
* was coerced to it, and when coercion failed — `Number('electronics')` — the
|
||
* value was dropped from the index with no error at all. The row stayed
|
||
* readable by id and by vector and vanished from every equality filter on the
|
||
* field. These tests therefore care about ORDER: strings-then-numbers and
|
||
* numbers-then-strings have to behave identically, because neither writer owns
|
||
* the field.
|
||
*
|
||
* Kinds never coerce into one another at query time either. `5` and `'5'` are
|
||
* different values and match different rows.
|
||
*/
|
||
|
||
import { describe, it, expect, beforeEach, afterEach } from 'vitest'
|
||
import { ColumnStore } from '../../../../src/indexes/columnStore/ColumnStore.js'
|
||
import { MemoryStorage } from '../../../../src/storage/adapters/memoryStorage.js'
|
||
import { EntityIdMapper } from '../../../../src/utils/entityIdMapper.js'
|
||
|
||
describe('ColumnStore — typed posting lists per (field, kind)', () => {
|
||
let storage: MemoryStorage
|
||
let idMapper: EntityIdMapper
|
||
let store: ColumnStore
|
||
|
||
beforeEach(async () => {
|
||
storage = new MemoryStorage()
|
||
await storage.init()
|
||
idMapper = new EntityIdMapper({ storage, storageKey: 'test:idMapper' })
|
||
await idMapper.init()
|
||
|
||
store = new ColumnStore({ flushThreshold: 10 })
|
||
await store.init(storage, idMapper)
|
||
})
|
||
|
||
afterEach(async () => {
|
||
await store.close()
|
||
})
|
||
|
||
/** Resolve a filter to the sorted UUIDs it matched. */
|
||
const uuidsOf = async (field: string, value: unknown): Promise<string[]> => {
|
||
const bitmap = await store.filter(field, value)
|
||
return Array.from(bitmap)
|
||
.map((id) => idMapper.getUuid(Number(id)))
|
||
.filter((u): u is string => u !== undefined)
|
||
.sort()
|
||
}
|
||
|
||
describe('equality answers on the query value’s own kind', () => {
|
||
it('serves numbers written AFTER strings on the same field', async () => {
|
||
store.addEntity(BigInt(idMapper.getOrAssign('s1')), { category: 'electronics' })
|
||
store.addEntity(BigInt(idMapper.getOrAssign('s2')), { category: 'furniture' })
|
||
store.addEntity(BigInt(idMapper.getOrAssign('n1')), { category: 5 })
|
||
store.addEntity(BigInt(idMapper.getOrAssign('n2')), { category: 5 })
|
||
store.addEntity(BigInt(idMapper.getOrAssign('n3')), { category: 7 })
|
||
|
||
// The numbers are in the index, though a string got there first.
|
||
expect(await uuidsOf('category', 5)).toEqual(['n1', 'n2'])
|
||
expect(await uuidsOf('category', 7)).toEqual(['n3'])
|
||
// And the strings did not move.
|
||
expect(await uuidsOf('category', 'electronics')).toEqual(['s1'])
|
||
expect(await uuidsOf('category', 'furniture')).toEqual(['s2'])
|
||
})
|
||
|
||
it('serves strings written AFTER numbers on the same field', async () => {
|
||
store.addEntity(BigInt(idMapper.getOrAssign('n1')), { category: 5 })
|
||
store.addEntity(BigInt(idMapper.getOrAssign('n2')), { category: 5 })
|
||
store.addEntity(BigInt(idMapper.getOrAssign('s1')), { category: 'electronics' })
|
||
store.addEntity(BigInt(idMapper.getOrAssign('s2')), { category: 'electronics' })
|
||
|
||
// 'electronics' would have become NaN and been dropped under the freeze.
|
||
expect(await uuidsOf('category', 'electronics')).toEqual(['s1', 's2'])
|
||
expect(await uuidsOf('category', 5)).toEqual(['n1', 'n2'])
|
||
})
|
||
|
||
it('does not coerce a number query into the string postings, or back', async () => {
|
||
store.addEntity(BigInt(idMapper.getOrAssign('num')), { code: 5 })
|
||
store.addEntity(BigInt(idMapper.getOrAssign('str')), { code: '5' })
|
||
|
||
expect(await uuidsOf('code', 5)).toEqual(['num'])
|
||
expect(await uuidsOf('code', '5')).toEqual(['str'])
|
||
})
|
||
|
||
it('serves booleans mixed into a field that already holds strings and numbers', async () => {
|
||
store.addEntity(BigInt(idMapper.getOrAssign('s1')), { flag: 'yes' })
|
||
store.addEntity(BigInt(idMapper.getOrAssign('n1')), { flag: 1 })
|
||
store.addEntity(BigInt(idMapper.getOrAssign('b1')), { flag: true })
|
||
store.addEntity(BigInt(idMapper.getOrAssign('b2')), { flag: false })
|
||
|
||
expect(await uuidsOf('flag', true)).toEqual(['b1'])
|
||
expect(await uuidsOf('flag', false)).toEqual(['b2'])
|
||
// `true` stores as 1 internally; that is an encoding, not a value.
|
||
expect(await uuidsOf('flag', 1)).toEqual(['n1'])
|
||
expect(await uuidsOf('flag', 'yes')).toEqual(['s1'])
|
||
})
|
||
|
||
it('answers nothing — not something coerced — for a kind the field never held', async () => {
|
||
store.addEntity(BigInt(idMapper.getOrAssign('s1')), { category: 'electronics' })
|
||
|
||
expect(await uuidsOf('category', 5)).toEqual([])
|
||
expect(await uuidsOf('category', true)).toEqual([])
|
||
})
|
||
|
||
it('holds every kind across a flush, not just the one in the tail buffer', async () => {
|
||
store.addEntity(BigInt(idMapper.getOrAssign('s1')), { category: 'electronics' })
|
||
store.addEntity(BigInt(idMapper.getOrAssign('n1')), { category: 5 })
|
||
await store.flush()
|
||
store.addEntity(BigInt(idMapper.getOrAssign('s2')), { category: 'electronics' })
|
||
store.addEntity(BigInt(idMapper.getOrAssign('n2')), { category: 5 })
|
||
|
||
expect(await uuidsOf('category', 'electronics')).toEqual(['s1', 's2'])
|
||
expect(await uuidsOf('category', 5)).toEqual(['n1', 'n2'])
|
||
})
|
||
})
|
||
|
||
describe('range filters read the numeric postings', () => {
|
||
it('ranges over the numeric subset of a mixed field, ignoring its strings', async () => {
|
||
store.addEntity(BigInt(idMapper.getOrAssign('cheap')), { price: 100 })
|
||
store.addEntity(BigInt(idMapper.getOrAssign('mid')), { price: 500 })
|
||
store.addEntity(BigInt(idMapper.getOrAssign('dear')), { price: 900 })
|
||
store.addEntity(BigInt(idMapper.getOrAssign('unpriced')), { price: 'on request' })
|
||
await store.flush()
|
||
|
||
const inRange = await store.rangeQuery('price', 200, 1000)
|
||
const uuids = Array.from(inRange)
|
||
.map((id) => idMapper.getUuid(Number(id)))
|
||
.sort()
|
||
expect(uuids).toEqual(['dear', 'mid'])
|
||
})
|
||
|
||
it('an unbounded range still reports every kind — it is the “has a value” probe', async () => {
|
||
store.addEntity(BigInt(idMapper.getOrAssign('n1')), { mixed: 42 })
|
||
store.addEntity(BigInt(idMapper.getOrAssign('s1')), { mixed: 'text' })
|
||
store.addEntity(BigInt(idMapper.getOrAssign('b1')), { mixed: true })
|
||
await store.flush()
|
||
|
||
const anyValue = await store.rangeQuery('mixed')
|
||
const uuids = Array.from(anyValue)
|
||
.map((id) => idMapper.getUuid(Number(id)))
|
||
.sort()
|
||
expect(uuids).toEqual(['b1', 'n1', 's1'])
|
||
})
|
||
})
|
||
|
||
describe('the index reports what a field actually holds', () => {
|
||
it('names every kind present, not the one that got there first', async () => {
|
||
store.addEntity(BigInt(idMapper.getOrAssign('s1')), { category: 'electronics' })
|
||
expect(store.getFieldKinds('category')).toEqual(['string'])
|
||
|
||
store.addEntity(BigInt(idMapper.getOrAssign('n1')), { category: 5 })
|
||
store.addEntity(BigInt(idMapper.getOrAssign('b1')), { category: true })
|
||
expect(store.getFieldKinds('category')).toEqual(['number', 'string', 'boolean'])
|
||
|
||
// And the field is still ONE field by name.
|
||
expect(store.getIndexedFields()).toEqual(['category'])
|
||
expect(store.hasField('category')).toBe(true)
|
||
})
|
||
|
||
it('reports an unknown field as holding nothing', () => {
|
||
expect(store.getFieldKinds('never-written')).toEqual([])
|
||
})
|
||
})
|
||
|
||
describe('an integer column widens rather than rounding', () => {
|
||
it('keeps a non-integer written after integers as itself', async () => {
|
||
store.addEntity(BigInt(idMapper.getOrAssign('a')), { score: 4 })
|
||
store.addEntity(BigInt(idMapper.getOrAssign('b')), { score: 4.5 })
|
||
store.addEntity(BigInt(idMapper.getOrAssign('c')), { score: 5 })
|
||
await store.flush()
|
||
|
||
// 4.5 used to round to 5 and answer `score === 5` alongside c.
|
||
expect(await uuidsOf('score', 4.5)).toEqual(['b'])
|
||
expect(await uuidsOf('score', 5)).toEqual(['c'])
|
||
expect(await uuidsOf('score', 4)).toEqual(['a'])
|
||
})
|
||
})
|
||
|
||
describe('close then reopen', () => {
|
||
it('keeps every typed posting, on the same storage', async () => {
|
||
store.addEntity(BigInt(idMapper.getOrAssign('s1')), { category: 'electronics' })
|
||
store.addEntity(BigInt(idMapper.getOrAssign('n1')), { category: 5 })
|
||
store.addEntity(BigInt(idMapper.getOrAssign('b1')), { category: true })
|
||
store.addEntity(BigInt(idMapper.getOrAssign('f1')), { score: 1.5 })
|
||
await store.flush()
|
||
await store.close()
|
||
|
||
store = new ColumnStore({ flushThreshold: 10 })
|
||
await store.init(storage, idMapper)
|
||
|
||
expect(store.getFieldKinds('category')).toEqual(['number', 'string', 'boolean'])
|
||
expect(await uuidsOf('category', 'electronics')).toEqual(['s1'])
|
||
expect(await uuidsOf('category', 5)).toEqual(['n1'])
|
||
expect(await uuidsOf('category', true)).toEqual(['b1'])
|
||
expect(await uuidsOf('score', 1.5)).toEqual(['f1'])
|
||
})
|
||
|
||
it('accepts new values of every kind after the reopen', async () => {
|
||
store.addEntity(BigInt(idMapper.getOrAssign('s1')), { category: 'electronics' })
|
||
store.addEntity(BigInt(idMapper.getOrAssign('n1')), { category: 5 })
|
||
await store.flush()
|
||
await store.close()
|
||
|
||
store = new ColumnStore({ flushThreshold: 10 })
|
||
await store.init(storage, idMapper)
|
||
|
||
store.addEntity(BigInt(idMapper.getOrAssign('s2')), { category: 'electronics' })
|
||
store.addEntity(BigInt(idMapper.getOrAssign('n2')), { category: 5 })
|
||
store.addEntity(BigInt(idMapper.getOrAssign('b1')), { category: false })
|
||
await store.flush()
|
||
|
||
expect(await uuidsOf('category', 'electronics')).toEqual(['s1', 's2'])
|
||
expect(await uuidsOf('category', 5)).toEqual(['n1', 'n2'])
|
||
expect(await uuidsOf('category', false)).toEqual(['b1'])
|
||
})
|
||
|
||
it('opens an index written by the pre-typed-postings shape and reads it unchanged', async () => {
|
||
// A single-kind field is byte-identical to what the old writer produced:
|
||
// one manifest at `_column_index/<field>/MANIFEST.json`, no kind
|
||
// subdirectory anywhere. That IS the old on-disk shape, so proving the
|
||
// new reader serves it proves an old index still opens.
|
||
store.addEntity(BigInt(idMapper.getOrAssign('a')), { status: 'active' })
|
||
store.addEntity(BigInt(idMapper.getOrAssign('b')), { status: 'archived' })
|
||
await store.flush()
|
||
|
||
const keys = await (storage as unknown as {
|
||
listObjectsUnderPath: (prefix: string) => Promise<string[]>
|
||
}).listObjectsUnderPath('_column_index/')
|
||
expect(keys.some((k) => k.includes('/k/'))).toBe(false)
|
||
|
||
await store.close()
|
||
store = new ColumnStore({ flushThreshold: 10 })
|
||
await store.init(storage, idMapper)
|
||
|
||
expect(store.getFieldKinds('status')).toEqual(['string'])
|
||
expect(await uuidsOf('status', 'active')).toEqual(['a'])
|
||
})
|
||
})
|
||
})
|