David Snelling
2427bb7960
GraphIndexProvider now speaks BigInt at the boundary (D.2 mirror): - getNeighbors/getVerbIdsBySource/getVerbIdsByTarget take entity ints and return entity/verb ints as bigint[] - new REQUIRED verbIntsToIds(bigint[]) batch reverse resolver (L.7 identity-fingerprint design — verb ids are UUIDs by contract, so the provider-side interning is losslessly reversible) - addVerb(verb, sourceInt, targetInt) returns the interned verb int; removeVerb(verbId) joins the contract Coordinator (brainy.ts) owns ALL UUID <-> int conversion: getOrAssign on writes, getInt on reads (unmapped UUID -> empty result without calling the provider), getUuid / verbIntsToIds on returns, plus a bounded ~100k-entry insertion-order warm cache for verb-int -> verb-id pairs fed by addVerb returns and resolver results. GraphVerb gains derived sourceInt/targetInt (populated at add time, never persisted). findConnectedSubtype gains a native fast path that routes single-type single-subtype outgoing BFS through the provider when available. JS GraphAdjacencyIndex satisfies the contract while staying string/u32-keyed internally: entity ints resolve through the shared entity-id mapper (threaded in by the coordinator on init/fork/checkout), verb ints come from an in-process append-only interning map re-derived from storage on rebuild/cold-start. ColumnStoreProvider widens the same way: addEntity/ removeEntity take bigint, sortTopK/filteredSortTopK return bigint[]. relate() now rejects a caller-supplied id with a teaching error — verb ids are brainy-generated UUIDs by contract in 8.0 (previously a passed id was silently ignored). No Roaring64 provider-boundary decode site exists yet; the JS-internal column store stays Roaring32 and the Treemap decoder lands with the first consumer of provider-returned filter buffers. Public brain API unchanged. 1413 tests green (+10 new BigInt contract tests).
295 lines
11 KiB
TypeScript
295 lines
11 KiB
TypeScript
/**
|
|
* @module column-store.test
|
|
* @description Tests for the ColumnStore coordinator: full lifecycle including
|
|
* add, flush, reload, filter, sort, range query, and removal.
|
|
*/
|
|
|
|
import { describe, it, expect, beforeEach, afterEach } from 'vitest'
|
|
import { ColumnStore } from '../../../../src/indexes/columnStore/ColumnStore.js'
|
|
import { MemoryStorage } from '../../../../src/storage/adapters/memoryStorage.js'
|
|
import { EntityIdMapper } from '../../../../src/utils/entityIdMapper.js'
|
|
import { RoaringBitmap32 } from '../../../../src/utils/roaring/index.js'
|
|
|
|
describe('ColumnStore', () => {
|
|
let storage: MemoryStorage
|
|
let idMapper: EntityIdMapper
|
|
let store: ColumnStore
|
|
|
|
beforeEach(async () => {
|
|
storage = new MemoryStorage()
|
|
await storage.init()
|
|
idMapper = new EntityIdMapper({ storage, storageKey: 'test:idMapper' })
|
|
await idMapper.init()
|
|
|
|
store = new ColumnStore({ flushThreshold: 10 }) // low threshold for tests
|
|
await store.init(storage, idMapper)
|
|
})
|
|
|
|
afterEach(async () => {
|
|
await store.close()
|
|
})
|
|
|
|
// -----------------------------------------------------------------------
|
|
// Basic add + sort
|
|
// -----------------------------------------------------------------------
|
|
|
|
describe('sortTopK', () => {
|
|
it('returns entities sorted by numeric field ascending', async () => {
|
|
store.addEntity(BigInt(idMapper.getOrAssign('a')), { createdAt: 300 })
|
|
store.addEntity(BigInt(idMapper.getOrAssign('b')), { createdAt: 100 })
|
|
store.addEntity(BigInt(idMapper.getOrAssign('c')), { createdAt: 200 })
|
|
await store.flush()
|
|
|
|
const result = await store.sortTopK('createdAt', 'asc', 10)
|
|
const uuids = result.map(id => idMapper.getUuid(Number(id)))
|
|
expect(uuids).toEqual(['b', 'c', 'a'])
|
|
})
|
|
|
|
it('returns entities sorted by numeric field descending', async () => {
|
|
store.addEntity(BigInt(idMapper.getOrAssign('a')), { createdAt: 300 })
|
|
store.addEntity(BigInt(idMapper.getOrAssign('b')), { createdAt: 100 })
|
|
store.addEntity(BigInt(idMapper.getOrAssign('c')), { createdAt: 200 })
|
|
await store.flush()
|
|
|
|
const result = await store.sortTopK('createdAt', 'desc', 10)
|
|
const uuids = result.map(id => idMapper.getUuid(Number(id)))
|
|
expect(uuids).toEqual(['a', 'c', 'b'])
|
|
})
|
|
|
|
it('respects limit K', async () => {
|
|
for (let i = 0; i < 50; i++) {
|
|
store.addEntity(BigInt(idMapper.getOrAssign(`e${i}`)), { createdAt: i * 1000 })
|
|
}
|
|
await store.flush()
|
|
|
|
const result = await store.sortTopK('createdAt', 'desc', 5)
|
|
expect(result).toHaveLength(5)
|
|
// Should be the 5 highest timestamps
|
|
const uuids = result.map(id => idMapper.getUuid(Number(id)))
|
|
expect(uuids).toEqual(['e49', 'e48', 'e47', 'e46', 'e45'])
|
|
})
|
|
|
|
it('sorts string fields lexicographically', async () => {
|
|
store.addEntity(BigInt(idMapper.getOrAssign('a')), { status: 'pending' })
|
|
store.addEntity(BigInt(idMapper.getOrAssign('b')), { status: 'active' })
|
|
store.addEntity(BigInt(idMapper.getOrAssign('c')), { status: 'inactive' })
|
|
await store.flush()
|
|
|
|
const result = await store.sortTopK('status', 'asc', 10)
|
|
const uuids = result.map(id => idMapper.getUuid(Number(id)))
|
|
expect(uuids).toEqual(['b', 'c', 'a']) // active, inactive, pending
|
|
})
|
|
|
|
it('returns empty for unknown field', async () => {
|
|
const result = await store.sortTopK('nonexistent', 'asc', 10)
|
|
expect(result).toEqual([])
|
|
})
|
|
})
|
|
|
|
// -----------------------------------------------------------------------
|
|
// Filtered sort
|
|
// -----------------------------------------------------------------------
|
|
|
|
describe('filteredSortTopK', () => {
|
|
it('returns only entities in the filter bitmap, sorted', async () => {
|
|
const idA = idMapper.getOrAssign('a')
|
|
const idB = idMapper.getOrAssign('b')
|
|
const idC = idMapper.getOrAssign('c')
|
|
const idD = idMapper.getOrAssign('d')
|
|
|
|
store.addEntity(BigInt(idA), { createdAt: 400 })
|
|
store.addEntity(BigInt(idB), { createdAt: 100 })
|
|
store.addEntity(BigInt(idC), { createdAt: 300 })
|
|
store.addEntity(BigInt(idD), { createdAt: 200 })
|
|
await store.flush()
|
|
|
|
// Filter: only B and C
|
|
const bitmap = new RoaringBitmap32([idB, idC])
|
|
const result = await store.filteredSortTopK(bitmap, 'createdAt', 'desc', 10)
|
|
const uuids = result.map(id => idMapper.getUuid(Number(id)))
|
|
expect(uuids).toEqual(['c', 'b']) // 300, 100
|
|
})
|
|
})
|
|
|
|
// -----------------------------------------------------------------------
|
|
// Point filter
|
|
// -----------------------------------------------------------------------
|
|
|
|
describe('filter', () => {
|
|
it('returns matching entities as roaring bitmap', async () => {
|
|
const idA = idMapper.getOrAssign('a')
|
|
const idB = idMapper.getOrAssign('b')
|
|
const idC = idMapper.getOrAssign('c')
|
|
|
|
store.addEntity(BigInt(idA), { status: 'active' })
|
|
store.addEntity(BigInt(idB), { status: 'inactive' })
|
|
store.addEntity(BigInt(idC), { status: 'active' })
|
|
await store.flush()
|
|
|
|
const bitmap = await store.filter('status', 'active')
|
|
expect(bitmap.has(idA)).toBe(true)
|
|
expect(bitmap.has(idC)).toBe(true)
|
|
expect(bitmap.has(idB)).toBe(false)
|
|
})
|
|
|
|
it('returns empty bitmap for no match', async () => {
|
|
store.addEntity(BigInt(idMapper.getOrAssign('a')), { status: 'active' })
|
|
await store.flush()
|
|
|
|
const bitmap = await store.filter('status', 'deleted')
|
|
expect(bitmap.size).toBe(0)
|
|
})
|
|
})
|
|
|
|
// -----------------------------------------------------------------------
|
|
// Range query
|
|
// -----------------------------------------------------------------------
|
|
|
|
describe('rangeQuery', () => {
|
|
it('returns entities within numeric range', async () => {
|
|
const ids: number[] = []
|
|
for (let i = 0; i < 10; i++) {
|
|
ids.push(idMapper.getOrAssign(`e${i}`))
|
|
store.addEntity(BigInt(ids[i]), { price: i * 10 })
|
|
}
|
|
await store.flush()
|
|
|
|
const bitmap = await store.rangeQuery('price', 30, 70)
|
|
// price 30, 40, 50, 60, 70 → entities e3, e4, e5, e6, e7
|
|
expect(bitmap.size).toBe(5)
|
|
expect(bitmap.has(ids[3])).toBe(true)
|
|
expect(bitmap.has(ids[7])).toBe(true)
|
|
expect(bitmap.has(ids[2])).toBe(false)
|
|
expect(bitmap.has(ids[8])).toBe(false)
|
|
})
|
|
})
|
|
|
|
// -----------------------------------------------------------------------
|
|
// Removal
|
|
// -----------------------------------------------------------------------
|
|
|
|
describe('removeEntity', () => {
|
|
it('excluded from sort after removal', async () => {
|
|
const idA = idMapper.getOrAssign('a')
|
|
const idB = idMapper.getOrAssign('b')
|
|
const idC = idMapper.getOrAssign('c')
|
|
|
|
store.addEntity(BigInt(idA), { createdAt: 100 })
|
|
store.addEntity(BigInt(idB), { createdAt: 200 })
|
|
store.addEntity(BigInt(idC), { createdAt: 300 })
|
|
await store.flush()
|
|
|
|
store.removeEntity(BigInt(idB))
|
|
await store.flush()
|
|
|
|
const result = await store.sortTopK('createdAt', 'asc', 10)
|
|
const uuids = result.map(id => idMapper.getUuid(Number(id)))
|
|
expect(uuids).toEqual(['a', 'c'])
|
|
})
|
|
})
|
|
|
|
// -----------------------------------------------------------------------
|
|
// Multi-segment merge sort
|
|
// -----------------------------------------------------------------------
|
|
|
|
describe('multi-segment', () => {
|
|
it('sorts correctly across multiple L0 segments', async () => {
|
|
// Flush threshold is 10, so 25 entities creates 3 segments
|
|
for (let i = 0; i < 25; i++) {
|
|
store.addEntity(BigInt(idMapper.getOrAssign(`e${i}`)), { ts: i })
|
|
}
|
|
await store.flush()
|
|
|
|
const result = await store.sortTopK('ts', 'desc', 5)
|
|
const uuids = result.map(id => idMapper.getUuid(Number(id)))
|
|
expect(uuids).toEqual(['e24', 'e23', 'e22', 'e21', 'e20'])
|
|
})
|
|
})
|
|
|
|
// -----------------------------------------------------------------------
|
|
// Persistence (flush + reload)
|
|
// -----------------------------------------------------------------------
|
|
|
|
describe('persistence', () => {
|
|
it('survives close + reload', async () => {
|
|
store.addEntity(BigInt(idMapper.getOrAssign('a')), { createdAt: 300 })
|
|
store.addEntity(BigInt(idMapper.getOrAssign('b')), { createdAt: 100 })
|
|
store.addEntity(BigInt(idMapper.getOrAssign('c')), { createdAt: 200 })
|
|
await store.flush()
|
|
await store.close()
|
|
|
|
// Create a new store pointing at the same storage
|
|
const store2 = new ColumnStore({ flushThreshold: 10 })
|
|
await store2.init(storage, idMapper)
|
|
|
|
const result = await store2.sortTopK('createdAt', 'desc', 10)
|
|
const uuids = result.map(id => idMapper.getUuid(Number(id)))
|
|
expect(uuids).toEqual(['a', 'c', 'b'])
|
|
|
|
await store2.close()
|
|
})
|
|
})
|
|
|
|
// -----------------------------------------------------------------------
|
|
// Multi-value fields (words)
|
|
// -----------------------------------------------------------------------
|
|
|
|
describe('multi-value fields', () => {
|
|
it('indexes multiple values per entity', async () => {
|
|
const idA = idMapper.getOrAssign('docA')
|
|
const idB = idMapper.getOrAssign('docB')
|
|
|
|
store.addEntity(BigInt(idA), { __words__: ['machine', 'learning', 'algorithm'] })
|
|
store.addEntity(BigInt(idB), { __words__: ['neural', 'network', 'machine'] })
|
|
await store.flush()
|
|
|
|
// Point filter: "machine" should match both
|
|
const machineBitmap = await store.filter('__words__', 'machine')
|
|
expect(machineBitmap.has(idA)).toBe(true)
|
|
expect(machineBitmap.has(idB)).toBe(true)
|
|
|
|
// "algorithm" should match only docA
|
|
const algoBitmap = await store.filter('__words__', 'algorithm')
|
|
expect(algoBitmap.has(idA)).toBe(true)
|
|
expect(algoBitmap.has(idB)).toBe(false)
|
|
|
|
// AND intersection: "machine" AND "neural" → only docB
|
|
const neuralBitmap = await store.filter('__words__', 'neural')
|
|
const intersection = RoaringBitmap32.and(machineBitmap, neuralBitmap)
|
|
expect(intersection.has(idB)).toBe(true)
|
|
expect(intersection.has(idA)).toBe(false)
|
|
})
|
|
})
|
|
|
|
// -----------------------------------------------------------------------
|
|
// getFilterValues
|
|
// -----------------------------------------------------------------------
|
|
|
|
describe('getFilterValues', () => {
|
|
it('returns distinct values sorted', async () => {
|
|
store.addEntity(BigInt(idMapper.getOrAssign('a')), { status: 'pending' })
|
|
store.addEntity(BigInt(idMapper.getOrAssign('b')), { status: 'active' })
|
|
store.addEntity(BigInt(idMapper.getOrAssign('c')), { status: 'active' })
|
|
store.addEntity(BigInt(idMapper.getOrAssign('d')), { status: 'inactive' })
|
|
await store.flush()
|
|
|
|
const values = await store.getFilterValues('status')
|
|
expect(values).toEqual(['active', 'inactive', 'pending'])
|
|
})
|
|
})
|
|
|
|
// -----------------------------------------------------------------------
|
|
// hasField
|
|
// -----------------------------------------------------------------------
|
|
|
|
describe('hasField', () => {
|
|
it('returns false for unknown field', () => {
|
|
expect(store.hasField('nonexistent')).toBe(false)
|
|
})
|
|
|
|
it('returns true after adding data', async () => {
|
|
store.addEntity(BigInt(idMapper.getOrAssign('a')), { createdAt: 100 })
|
|
expect(store.hasField('createdAt')).toBe(true)
|
|
})
|
|
})
|
|
})
|