feat: unified column store for filtering + sorting at billion scale
Adds a per-field sorted column store (Lucene doc values + roaring bitmap
architecture) that replaces the MetadataIndex sparse index internals for
both filtering and sorting. One system for all field types with exact
precision — no bucketing, no per-entity storage reads.
Column store: binary .cidx segment format, in-memory tail buffers,
LSM-style compaction, k-way merge sort, multi-value support (__words__).
All queries (filter, range, sort, filtered sort) route through the column
store when data is available, falling back to sparse index otherwise.
Key unlocks:
- find({ orderBy: 'createdAt' }) works WITHOUT a filter (previously threw)
- find({ orderBy: 'metadata.price' }) works for custom numeric fields
- Exact timestamp precision (no 1-minute bucketing)
- O(K log S) sort independent of total entity count
New files: src/indexes/columnStore/ (types, format, tail buffer, cursor,
manifest, coordinator — ~700 lines). 101 new unit tests covering binary
format round-trips, CRC validation, sort, filter, range, deletion,
multi-segment merge, persistence, and multi-value (words) fields.
Deleted: metadataIndex-automatic-bucketing.test.ts (bucketing behavior
eliminated by exact-precision column store). Sparse index write path
removed from addToIndex/removeFromIndex. Sparse index legacy code still
present as dead code pending cleanup in next commit.
This commit is contained in:
parent
634ddfb2bb
commit
46583f2d9b
16 changed files with 3846 additions and 521 deletions
|
|
@ -125,16 +125,25 @@ describe('find({ orderBy }) sort bug regression', () => {
|
|||
})
|
||||
|
||||
/**
|
||||
* Unfiltered sort is explicitly rejected with a clear error pointing
|
||||
* callers at the right pattern. This guards against silent N-scans
|
||||
* on production workloads.
|
||||
* Unfiltered sort now works via the unified column store.
|
||||
* This was the Track 2 motivating use case — previously threw an error.
|
||||
*/
|
||||
it('orderBy without any filter throws a helpful error', async () => {
|
||||
await brain.add({ data: 'anything', type: NounType.Concept })
|
||||
it('orderBy without filter works via column store', async () => {
|
||||
const id1 = await brain.add({ data: 'first', type: NounType.Concept })
|
||||
await new Promise((r) => setTimeout(r, 20))
|
||||
await brain.add({ data: 'second', type: NounType.Concept })
|
||||
await new Promise((r) => setTimeout(r, 20))
|
||||
const id3 = await brain.add({ data: 'third', type: NounType.Concept })
|
||||
|
||||
await expect(
|
||||
brain.find({ orderBy: 'createdAt', order: 'desc', limit: 1 })
|
||||
).rejects.toThrow(/requires a filter/)
|
||||
const results = await brain.find({
|
||||
orderBy: 'createdAt',
|
||||
order: 'desc',
|
||||
limit: 2
|
||||
})
|
||||
|
||||
expect(results.length).toBeGreaterThanOrEqual(2)
|
||||
// Newest should be first (desc order)
|
||||
expect(results[0].id).toBe(id3)
|
||||
})
|
||||
})
|
||||
|
||||
|
|
|
|||
295
tests/unit/indexes/columnStore/column-store.test.ts
Normal file
295
tests/unit/indexes/columnStore/column-store.test.ts
Normal file
|
|
@ -0,0 +1,295 @@
|
|||
/**
|
||||
* @module column-store.test
|
||||
* @description Tests for the ColumnStore coordinator: full lifecycle including
|
||||
* add, flush, reload, filter, sort, range query, and removal.
|
||||
*/
|
||||
|
||||
import { describe, it, expect, beforeEach, afterEach } from 'vitest'
|
||||
import { ColumnStore } from '../../../../src/indexes/columnStore/ColumnStore.js'
|
||||
import { MemoryStorage } from '../../../../src/storage/adapters/memoryStorage.js'
|
||||
import { EntityIdMapper } from '../../../../src/utils/entityIdMapper.js'
|
||||
import { RoaringBitmap32 } from '../../../../src/utils/roaring/index.js'
|
||||
|
||||
describe('ColumnStore', () => {
|
||||
let storage: MemoryStorage
|
||||
let idMapper: EntityIdMapper
|
||||
let store: ColumnStore
|
||||
|
||||
beforeEach(async () => {
|
||||
storage = new MemoryStorage()
|
||||
await storage.init()
|
||||
idMapper = new EntityIdMapper({ storage, storageKey: 'test:idMapper' })
|
||||
await idMapper.init()
|
||||
|
||||
store = new ColumnStore({ flushThreshold: 10 }) // low threshold for tests
|
||||
await store.init(storage, idMapper)
|
||||
})
|
||||
|
||||
afterEach(async () => {
|
||||
await store.close()
|
||||
})
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
// Basic add + sort
|
||||
// -----------------------------------------------------------------------
|
||||
|
||||
describe('sortTopK', () => {
|
||||
it('returns entities sorted by numeric field ascending', async () => {
|
||||
store.addEntity(idMapper.getOrAssign('a'), { createdAt: 300 })
|
||||
store.addEntity(idMapper.getOrAssign('b'), { createdAt: 100 })
|
||||
store.addEntity(idMapper.getOrAssign('c'), { createdAt: 200 })
|
||||
await store.flush()
|
||||
|
||||
const result = await store.sortTopK('createdAt', 'asc', 10)
|
||||
const uuids = result.map(id => idMapper.getUuid(id))
|
||||
expect(uuids).toEqual(['b', 'c', 'a'])
|
||||
})
|
||||
|
||||
it('returns entities sorted by numeric field descending', async () => {
|
||||
store.addEntity(idMapper.getOrAssign('a'), { createdAt: 300 })
|
||||
store.addEntity(idMapper.getOrAssign('b'), { createdAt: 100 })
|
||||
store.addEntity(idMapper.getOrAssign('c'), { createdAt: 200 })
|
||||
await store.flush()
|
||||
|
||||
const result = await store.sortTopK('createdAt', 'desc', 10)
|
||||
const uuids = result.map(id => idMapper.getUuid(id))
|
||||
expect(uuids).toEqual(['a', 'c', 'b'])
|
||||
})
|
||||
|
||||
it('respects limit K', async () => {
|
||||
for (let i = 0; i < 50; i++) {
|
||||
store.addEntity(idMapper.getOrAssign(`e${i}`), { createdAt: i * 1000 })
|
||||
}
|
||||
await store.flush()
|
||||
|
||||
const result = await store.sortTopK('createdAt', 'desc', 5)
|
||||
expect(result).toHaveLength(5)
|
||||
// Should be the 5 highest timestamps
|
||||
const uuids = result.map(id => idMapper.getUuid(id))
|
||||
expect(uuids).toEqual(['e49', 'e48', 'e47', 'e46', 'e45'])
|
||||
})
|
||||
|
||||
it('sorts string fields lexicographically', async () => {
|
||||
store.addEntity(idMapper.getOrAssign('a'), { status: 'pending' })
|
||||
store.addEntity(idMapper.getOrAssign('b'), { status: 'active' })
|
||||
store.addEntity(idMapper.getOrAssign('c'), { status: 'inactive' })
|
||||
await store.flush()
|
||||
|
||||
const result = await store.sortTopK('status', 'asc', 10)
|
||||
const uuids = result.map(id => idMapper.getUuid(id))
|
||||
expect(uuids).toEqual(['b', 'c', 'a']) // active, inactive, pending
|
||||
})
|
||||
|
||||
it('returns empty for unknown field', async () => {
|
||||
const result = await store.sortTopK('nonexistent', 'asc', 10)
|
||||
expect(result).toEqual([])
|
||||
})
|
||||
})
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
// Filtered sort
|
||||
// -----------------------------------------------------------------------
|
||||
|
||||
describe('filteredSortTopK', () => {
|
||||
it('returns only entities in the filter bitmap, sorted', async () => {
|
||||
const idA = idMapper.getOrAssign('a')
|
||||
const idB = idMapper.getOrAssign('b')
|
||||
const idC = idMapper.getOrAssign('c')
|
||||
const idD = idMapper.getOrAssign('d')
|
||||
|
||||
store.addEntity(idA, { createdAt: 400 })
|
||||
store.addEntity(idB, { createdAt: 100 })
|
||||
store.addEntity(idC, { createdAt: 300 })
|
||||
store.addEntity(idD, { createdAt: 200 })
|
||||
await store.flush()
|
||||
|
||||
// Filter: only B and C
|
||||
const bitmap = new RoaringBitmap32([idB, idC])
|
||||
const result = await store.filteredSortTopK(bitmap, 'createdAt', 'desc', 10)
|
||||
const uuids = result.map(id => idMapper.getUuid(id))
|
||||
expect(uuids).toEqual(['c', 'b']) // 300, 100
|
||||
})
|
||||
})
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
// Point filter
|
||||
// -----------------------------------------------------------------------
|
||||
|
||||
describe('filter', () => {
|
||||
it('returns matching entities as roaring bitmap', async () => {
|
||||
const idA = idMapper.getOrAssign('a')
|
||||
const idB = idMapper.getOrAssign('b')
|
||||
const idC = idMapper.getOrAssign('c')
|
||||
|
||||
store.addEntity(idA, { status: 'active' })
|
||||
store.addEntity(idB, { status: 'inactive' })
|
||||
store.addEntity(idC, { status: 'active' })
|
||||
await store.flush()
|
||||
|
||||
const bitmap = await store.filter('status', 'active')
|
||||
expect(bitmap.has(idA)).toBe(true)
|
||||
expect(bitmap.has(idC)).toBe(true)
|
||||
expect(bitmap.has(idB)).toBe(false)
|
||||
})
|
||||
|
||||
it('returns empty bitmap for no match', async () => {
|
||||
store.addEntity(idMapper.getOrAssign('a'), { status: 'active' })
|
||||
await store.flush()
|
||||
|
||||
const bitmap = await store.filter('status', 'deleted')
|
||||
expect(bitmap.size).toBe(0)
|
||||
})
|
||||
})
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
// Range query
|
||||
// -----------------------------------------------------------------------
|
||||
|
||||
describe('rangeQuery', () => {
|
||||
it('returns entities within numeric range', async () => {
|
||||
const ids: number[] = []
|
||||
for (let i = 0; i < 10; i++) {
|
||||
ids.push(idMapper.getOrAssign(`e${i}`))
|
||||
store.addEntity(ids[i], { price: i * 10 })
|
||||
}
|
||||
await store.flush()
|
||||
|
||||
const bitmap = await store.rangeQuery('price', 30, 70)
|
||||
// price 30, 40, 50, 60, 70 → entities e3, e4, e5, e6, e7
|
||||
expect(bitmap.size).toBe(5)
|
||||
expect(bitmap.has(ids[3])).toBe(true)
|
||||
expect(bitmap.has(ids[7])).toBe(true)
|
||||
expect(bitmap.has(ids[2])).toBe(false)
|
||||
expect(bitmap.has(ids[8])).toBe(false)
|
||||
})
|
||||
})
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
// Removal
|
||||
// -----------------------------------------------------------------------
|
||||
|
||||
describe('removeEntity', () => {
|
||||
it('excluded from sort after removal', async () => {
|
||||
const idA = idMapper.getOrAssign('a')
|
||||
const idB = idMapper.getOrAssign('b')
|
||||
const idC = idMapper.getOrAssign('c')
|
||||
|
||||
store.addEntity(idA, { createdAt: 100 })
|
||||
store.addEntity(idB, { createdAt: 200 })
|
||||
store.addEntity(idC, { createdAt: 300 })
|
||||
await store.flush()
|
||||
|
||||
store.removeEntity(idB)
|
||||
await store.flush()
|
||||
|
||||
const result = await store.sortTopK('createdAt', 'asc', 10)
|
||||
const uuids = result.map(id => idMapper.getUuid(id))
|
||||
expect(uuids).toEqual(['a', 'c'])
|
||||
})
|
||||
})
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
// Multi-segment merge sort
|
||||
// -----------------------------------------------------------------------
|
||||
|
||||
describe('multi-segment', () => {
|
||||
it('sorts correctly across multiple L0 segments', async () => {
|
||||
// Flush threshold is 10, so 25 entities creates 3 segments
|
||||
for (let i = 0; i < 25; i++) {
|
||||
store.addEntity(idMapper.getOrAssign(`e${i}`), { ts: i })
|
||||
}
|
||||
await store.flush()
|
||||
|
||||
const result = await store.sortTopK('ts', 'desc', 5)
|
||||
const uuids = result.map(id => idMapper.getUuid(id))
|
||||
expect(uuids).toEqual(['e24', 'e23', 'e22', 'e21', 'e20'])
|
||||
})
|
||||
})
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
// Persistence (flush + reload)
|
||||
// -----------------------------------------------------------------------
|
||||
|
||||
describe('persistence', () => {
|
||||
it('survives close + reload', async () => {
|
||||
store.addEntity(idMapper.getOrAssign('a'), { createdAt: 300 })
|
||||
store.addEntity(idMapper.getOrAssign('b'), { createdAt: 100 })
|
||||
store.addEntity(idMapper.getOrAssign('c'), { createdAt: 200 })
|
||||
await store.flush()
|
||||
await store.close()
|
||||
|
||||
// Create a new store pointing at the same storage
|
||||
const store2 = new ColumnStore({ flushThreshold: 10 })
|
||||
await store2.init(storage, idMapper)
|
||||
|
||||
const result = await store2.sortTopK('createdAt', 'desc', 10)
|
||||
const uuids = result.map(id => idMapper.getUuid(id))
|
||||
expect(uuids).toEqual(['a', 'c', 'b'])
|
||||
|
||||
await store2.close()
|
||||
})
|
||||
})
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
// Multi-value fields (words)
|
||||
// -----------------------------------------------------------------------
|
||||
|
||||
describe('multi-value fields', () => {
|
||||
it('indexes multiple values per entity', async () => {
|
||||
const idA = idMapper.getOrAssign('docA')
|
||||
const idB = idMapper.getOrAssign('docB')
|
||||
|
||||
store.addEntity(idA, { __words__: ['machine', 'learning', 'algorithm'] })
|
||||
store.addEntity(idB, { __words__: ['neural', 'network', 'machine'] })
|
||||
await store.flush()
|
||||
|
||||
// Point filter: "machine" should match both
|
||||
const machineBitmap = await store.filter('__words__', 'machine')
|
||||
expect(machineBitmap.has(idA)).toBe(true)
|
||||
expect(machineBitmap.has(idB)).toBe(true)
|
||||
|
||||
// "algorithm" should match only docA
|
||||
const algoBitmap = await store.filter('__words__', 'algorithm')
|
||||
expect(algoBitmap.has(idA)).toBe(true)
|
||||
expect(algoBitmap.has(idB)).toBe(false)
|
||||
|
||||
// AND intersection: "machine" AND "neural" → only docB
|
||||
const neuralBitmap = await store.filter('__words__', 'neural')
|
||||
const intersection = RoaringBitmap32.and(machineBitmap, neuralBitmap)
|
||||
expect(intersection.has(idB)).toBe(true)
|
||||
expect(intersection.has(idA)).toBe(false)
|
||||
})
|
||||
})
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
// getFilterValues
|
||||
// -----------------------------------------------------------------------
|
||||
|
||||
describe('getFilterValues', () => {
|
||||
it('returns distinct values sorted', async () => {
|
||||
store.addEntity(idMapper.getOrAssign('a'), { status: 'pending' })
|
||||
store.addEntity(idMapper.getOrAssign('b'), { status: 'active' })
|
||||
store.addEntity(idMapper.getOrAssign('c'), { status: 'active' })
|
||||
store.addEntity(idMapper.getOrAssign('d'), { status: 'inactive' })
|
||||
await store.flush()
|
||||
|
||||
const values = await store.getFilterValues('status')
|
||||
expect(values).toEqual(['active', 'inactive', 'pending'])
|
||||
})
|
||||
})
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
// hasField
|
||||
// -----------------------------------------------------------------------
|
||||
|
||||
describe('hasField', () => {
|
||||
it('returns false for unknown field', () => {
|
||||
expect(store.hasField('nonexistent')).toBe(false)
|
||||
})
|
||||
|
||||
it('returns true after adding data', async () => {
|
||||
store.addEntity(idMapper.getOrAssign('a'), { createdAt: 100 })
|
||||
expect(store.hasField('createdAt')).toBe(true)
|
||||
})
|
||||
})
|
||||
})
|
||||
162
tests/unit/indexes/columnStore/manifest.test.ts
Normal file
162
tests/unit/indexes/columnStore/manifest.test.ts
Normal file
|
|
@ -0,0 +1,162 @@
|
|||
/**
|
||||
* @module manifest.test
|
||||
* @description Tests for the per-field manifest manager.
|
||||
*/
|
||||
|
||||
import { describe, it, expect, beforeEach } from 'vitest'
|
||||
import { ColumnManifest } from '../../../../src/indexes/columnStore/ColumnManifest.js'
|
||||
import { ValueType, type SegmentMeta } from '../../../../src/indexes/columnStore/types.js'
|
||||
|
||||
/**
|
||||
* Minimal mock storage for manifest persistence tests.
|
||||
* Only implements readObjectFromPath and writeObjectToPath.
|
||||
*/
|
||||
class MockStorage {
|
||||
private data = new Map<string, any>()
|
||||
|
||||
async readObjectFromPath(path: string): Promise<any> {
|
||||
return this.data.get(path) ?? null
|
||||
}
|
||||
|
||||
async writeObjectToPath(path: string, obj: any): Promise<void> {
|
||||
this.data.set(path, JSON.parse(JSON.stringify(obj))) // deep clone
|
||||
}
|
||||
}
|
||||
|
||||
describe('ColumnManifest', () => {
|
||||
let storage: MockStorage
|
||||
|
||||
beforeEach(() => {
|
||||
storage = new MockStorage()
|
||||
})
|
||||
|
||||
it('starts fresh when no stored manifest exists', async () => {
|
||||
const manifest = new ColumnManifest('createdAt')
|
||||
await manifest.load(storage as any)
|
||||
|
||||
expect(manifest.isEmpty()).toBe(true)
|
||||
expect(manifest.getAllSegments()).toEqual([])
|
||||
expect(manifest.totalCount).toBe(0)
|
||||
})
|
||||
|
||||
it('save and reload preserves data', async () => {
|
||||
const m1 = new ColumnManifest('createdAt')
|
||||
m1.valueType = ValueType.Number
|
||||
m1.addSegment({
|
||||
id: m1.nextSegmentId(),
|
||||
level: 0,
|
||||
count: 50000,
|
||||
minValue: 1700000000000,
|
||||
maxValue: 1700100000000,
|
||||
file: 'L0-000001.cidx'
|
||||
})
|
||||
await m1.save(storage as any)
|
||||
|
||||
// Reload
|
||||
const m2 = new ColumnManifest('createdAt')
|
||||
await m2.load(storage as any)
|
||||
|
||||
expect(m2.isEmpty()).toBe(false)
|
||||
expect(m2.getAllSegments()).toHaveLength(1)
|
||||
expect(m2.getAllSegments()[0].count).toBe(50000)
|
||||
expect(m2.valueType).toBe(ValueType.Number)
|
||||
})
|
||||
|
||||
it('nextSegmentId increments monotonically', () => {
|
||||
const manifest = new ColumnManifest('test')
|
||||
const id1 = manifest.nextSegmentId()
|
||||
const id2 = manifest.nextSegmentId()
|
||||
const id3 = manifest.nextSegmentId()
|
||||
expect(id1).toBe(1)
|
||||
expect(id2).toBe(2)
|
||||
expect(id3).toBe(3)
|
||||
})
|
||||
|
||||
it('addSegment keeps segments sorted by level then id', () => {
|
||||
const manifest = new ColumnManifest('test')
|
||||
manifest.addSegment({ id: 3, level: 1, count: 100, minValue: 0, maxValue: 100, file: 'L1-3.cidx' })
|
||||
manifest.addSegment({ id: 1, level: 0, count: 50, minValue: 0, maxValue: 50, file: 'L0-1.cidx' })
|
||||
manifest.addSegment({ id: 2, level: 0, count: 50, minValue: 50, maxValue: 100, file: 'L0-2.cidx' })
|
||||
|
||||
const segs = manifest.getAllSegments()
|
||||
expect(segs[0].id).toBe(1) // L0, id=1
|
||||
expect(segs[1].id).toBe(2) // L0, id=2
|
||||
expect(segs[2].id).toBe(3) // L1, id=3
|
||||
})
|
||||
|
||||
it('removeSegments deletes by ID', () => {
|
||||
const manifest = new ColumnManifest('test')
|
||||
manifest.addSegment({ id: 1, level: 0, count: 50, minValue: 0, maxValue: 50, file: 'L0-1.cidx' })
|
||||
manifest.addSegment({ id: 2, level: 0, count: 50, minValue: 50, maxValue: 100, file: 'L0-2.cidx' })
|
||||
manifest.addSegment({ id: 3, level: 1, count: 100, minValue: 0, maxValue: 100, file: 'L1-3.cidx' })
|
||||
|
||||
manifest.removeSegments([1, 2])
|
||||
expect(manifest.getAllSegments()).toHaveLength(1)
|
||||
expect(manifest.getAllSegments()[0].id).toBe(3)
|
||||
})
|
||||
|
||||
it('getSegmentsAtLevel filters correctly', () => {
|
||||
const manifest = new ColumnManifest('test')
|
||||
manifest.addSegment({ id: 1, level: 0, count: 50, minValue: 0, maxValue: 50, file: 'L0-1.cidx' })
|
||||
manifest.addSegment({ id: 2, level: 0, count: 50, minValue: 50, maxValue: 100, file: 'L0-2.cidx' })
|
||||
manifest.addSegment({ id: 3, level: 1, count: 100, minValue: 0, maxValue: 100, file: 'L1-3.cidx' })
|
||||
|
||||
expect(manifest.getSegmentsAtLevel(0)).toHaveLength(2)
|
||||
expect(manifest.getSegmentsAtLevel(1)).toHaveLength(1)
|
||||
expect(manifest.getSegmentsAtLevel(2)).toHaveLength(0)
|
||||
})
|
||||
|
||||
it('segmentPath generates correct paths', () => {
|
||||
const manifest = new ColumnManifest('createdAt', '_column_index')
|
||||
expect(manifest.segmentPath(0, 1)).toBe('_column_index/createdAt/L0-000001.cidx')
|
||||
expect(manifest.segmentPath(1, 42)).toBe('_column_index/createdAt/L1-000042.cidx')
|
||||
expect(manifest.segmentPath(0, 999999)).toBe('_column_index/createdAt/L0-999999.cidx')
|
||||
})
|
||||
|
||||
it('manifestPath generates correct path', () => {
|
||||
const manifest = new ColumnManifest('status', '_column_index')
|
||||
expect(manifest.manifestPath()).toBe('_column_index/status/MANIFEST.json')
|
||||
})
|
||||
|
||||
it('totalCount sums across segments', () => {
|
||||
const manifest = new ColumnManifest('test')
|
||||
manifest.addSegment({ id: 1, level: 0, count: 100, minValue: 0, maxValue: 0, file: 'a' })
|
||||
manifest.addSegment({ id: 2, level: 0, count: 200, minValue: 0, maxValue: 0, file: 'b' })
|
||||
manifest.addSegment({ id: 3, level: 1, count: 300, minValue: 0, maxValue: 0, file: 'c' })
|
||||
expect(manifest.totalCount).toBe(600)
|
||||
})
|
||||
|
||||
it('version increments on each save', async () => {
|
||||
const manifest = new ColumnManifest('test')
|
||||
const v0 = manifest.getData().version
|
||||
await manifest.save(storage as any)
|
||||
const v1 = manifest.getData().version
|
||||
await manifest.save(storage as any)
|
||||
const v2 = manifest.getData().version
|
||||
expect(v1).toBe(v0 + 1)
|
||||
expect(v2).toBe(v1 + 1)
|
||||
})
|
||||
|
||||
it('multiValue flag persists', async () => {
|
||||
const m1 = new ColumnManifest('__words__')
|
||||
m1.multiValue = true
|
||||
m1.valueType = ValueType.String
|
||||
await m1.save(storage as any)
|
||||
|
||||
const m2 = new ColumnManifest('__words__')
|
||||
await m2.load(storage as any)
|
||||
expect(m2.multiValue).toBe(true)
|
||||
expect(m2.valueType).toBe(ValueType.String)
|
||||
})
|
||||
|
||||
it('buildComplete flag persists', async () => {
|
||||
const m1 = new ColumnManifest('test')
|
||||
expect(m1.buildComplete).toBe(false)
|
||||
m1.buildComplete = true
|
||||
await m1.save(storage as any)
|
||||
|
||||
const m2 = new ColumnManifest('test')
|
||||
await m2.load(storage as any)
|
||||
expect(m2.buildComplete).toBe(true)
|
||||
})
|
||||
})
|
||||
270
tests/unit/indexes/columnStore/segment-cursor.test.ts
Normal file
270
tests/unit/indexes/columnStore/segment-cursor.test.ts
Normal file
|
|
@ -0,0 +1,270 @@
|
|||
/**
|
||||
* @module segment-cursor.test
|
||||
* @description Tests for the segment cursor: binary search, iteration, tombstones.
|
||||
*/
|
||||
|
||||
import { describe, it, expect } from 'vitest'
|
||||
import { ColumnSegmentCursor, TailBufferCursor } from '../../../../src/indexes/columnStore/ColumnSegmentCursor.js'
|
||||
import { ValueType, CIDX_MAGIC, CIDX_VERSION } from '../../../../src/indexes/columnStore/types.js'
|
||||
import { RoaringBitmap32 } from '../../../../src/utils/roaring/index.js'
|
||||
|
||||
function makeNumericCursor(values: number[], entityIds: number[], tombstones: number[] = []): ColumnSegmentCursor {
|
||||
return new ColumnSegmentCursor(
|
||||
{
|
||||
magic: CIDX_MAGIC,
|
||||
version: CIDX_VERSION,
|
||||
fieldName: 'test',
|
||||
fieldNameLength: 4,
|
||||
valueType: ValueType.Number,
|
||||
level: 0,
|
||||
codec: 0,
|
||||
flags: 0,
|
||||
count: values.length
|
||||
},
|
||||
values,
|
||||
new Uint32Array(entityIds),
|
||||
new RoaringBitmap32(tombstones)
|
||||
)
|
||||
}
|
||||
|
||||
function makeStringCursor(values: string[], entityIds: number[], tombstones: number[] = []): ColumnSegmentCursor {
|
||||
return new ColumnSegmentCursor(
|
||||
{
|
||||
magic: CIDX_MAGIC,
|
||||
version: CIDX_VERSION,
|
||||
fieldName: 'status',
|
||||
fieldNameLength: 6,
|
||||
valueType: ValueType.String,
|
||||
level: 0,
|
||||
codec: 0,
|
||||
flags: 0,
|
||||
count: values.length
|
||||
},
|
||||
values,
|
||||
new Uint32Array(entityIds),
|
||||
new RoaringBitmap32(tombstones)
|
||||
)
|
||||
}
|
||||
|
||||
describe('ColumnSegmentCursor', () => {
|
||||
// -----------------------------------------------------------------------
|
||||
// Binary search
|
||||
// -----------------------------------------------------------------------
|
||||
|
||||
describe('binarySearchValue (numeric)', () => {
|
||||
it('finds exact match', () => {
|
||||
const c = makeNumericCursor([100, 200, 300, 400, 500], [1, 2, 3, 4, 5])
|
||||
const r = c.binarySearchValue(300)
|
||||
expect(r.found).toBe(true)
|
||||
expect(r.index).toBe(2)
|
||||
})
|
||||
|
||||
it('returns insertion point when not found', () => {
|
||||
const c = makeNumericCursor([100, 200, 400, 500], [1, 2, 3, 4])
|
||||
const r = c.binarySearchValue(300)
|
||||
expect(r.found).toBe(false)
|
||||
expect(r.index).toBe(2) // would insert at position 2
|
||||
})
|
||||
|
||||
it('handles before-first', () => {
|
||||
const c = makeNumericCursor([100, 200, 300], [1, 2, 3])
|
||||
const r = c.binarySearchValue(50)
|
||||
expect(r.found).toBe(false)
|
||||
expect(r.index).toBe(0)
|
||||
})
|
||||
|
||||
it('handles after-last', () => {
|
||||
const c = makeNumericCursor([100, 200, 300], [1, 2, 3])
|
||||
const r = c.binarySearchValue(999)
|
||||
expect(r.found).toBe(false)
|
||||
expect(r.index).toBe(3)
|
||||
})
|
||||
|
||||
it('finds first occurrence of duplicates', () => {
|
||||
const c = makeNumericCursor([100, 200, 200, 200, 300], [1, 2, 3, 4, 5])
|
||||
const r = c.binarySearchValue(200)
|
||||
expect(r.found).toBe(true)
|
||||
expect(r.index).toBe(1) // first occurrence
|
||||
})
|
||||
})
|
||||
|
||||
describe('binarySearchValue (string)', () => {
|
||||
it('finds exact match', () => {
|
||||
const c = makeStringCursor(['active', 'inactive', 'pending'], [1, 2, 3])
|
||||
const r = c.binarySearchValue('inactive')
|
||||
expect(r.found).toBe(true)
|
||||
expect(r.index).toBe(1)
|
||||
})
|
||||
|
||||
it('returns insertion point when not found', () => {
|
||||
const c = makeStringCursor(['active', 'pending'], [1, 2])
|
||||
const r = c.binarySearchValue('inactive')
|
||||
expect(r.found).toBe(false)
|
||||
expect(r.index).toBe(1)
|
||||
})
|
||||
})
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
// Range search
|
||||
// -----------------------------------------------------------------------
|
||||
|
||||
describe('binarySearchRange', () => {
|
||||
it('finds range with inclusive bounds', () => {
|
||||
const c = makeNumericCursor([100, 200, 300, 400, 500], [1, 2, 3, 4, 5])
|
||||
const { startIdx, endIdx } = c.binarySearchRange(200, 400)
|
||||
expect(startIdx).toBe(1) // 200
|
||||
expect(endIdx).toBe(4) // excludes 500
|
||||
})
|
||||
|
||||
it('handles range beyond data', () => {
|
||||
const c = makeNumericCursor([100, 200, 300], [1, 2, 3])
|
||||
const { startIdx, endIdx } = c.binarySearchRange(0, 999)
|
||||
expect(startIdx).toBe(0)
|
||||
expect(endIdx).toBe(3) // all entries
|
||||
})
|
||||
|
||||
it('handles empty range', () => {
|
||||
const c = makeNumericCursor([100, 200, 300], [1, 2, 3])
|
||||
const { startIdx, endIdx } = c.binarySearchRange(150, 180)
|
||||
expect(startIdx).toBe(1) // insertion point for 150
|
||||
expect(endIdx).toBe(1) // nothing between 150 and 180
|
||||
})
|
||||
})
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
// Point and range queries
|
||||
// -----------------------------------------------------------------------
|
||||
|
||||
describe('getEntityIdsForValue', () => {
|
||||
it('returns matching entity IDs', () => {
|
||||
const c = makeNumericCursor([100, 200, 200, 300], [10, 20, 30, 40])
|
||||
expect(c.getEntityIdsForValue(200)).toEqual([20, 30])
|
||||
})
|
||||
|
||||
it('returns empty for no match', () => {
|
||||
const c = makeNumericCursor([100, 200, 300], [10, 20, 30])
|
||||
expect(c.getEntityIdsForValue(999)).toEqual([])
|
||||
})
|
||||
|
||||
it('excludes tombstoned entries', () => {
|
||||
const c = makeNumericCursor([100, 200, 200, 300], [10, 20, 30, 40], [1]) // position 1 tombstoned
|
||||
expect(c.getEntityIdsForValue(200)).toEqual([30]) // position 1 (entityId=20) excluded
|
||||
})
|
||||
})
|
||||
|
||||
describe('getEntityIdsInRange', () => {
|
||||
it('returns entities in range', () => {
|
||||
const c = makeNumericCursor([100, 200, 300, 400, 500], [1, 2, 3, 4, 5])
|
||||
expect(c.getEntityIdsInRange(200, 400)).toEqual([2, 3, 4])
|
||||
})
|
||||
|
||||
it('excludes tombstoned', () => {
|
||||
const c = makeNumericCursor([100, 200, 300, 400, 500], [1, 2, 3, 4, 5], [2]) // position 2 (value=300)
|
||||
expect(c.getEntityIdsInRange(200, 400)).toEqual([2, 4])
|
||||
})
|
||||
})
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
// Iteration
|
||||
// -----------------------------------------------------------------------
|
||||
|
||||
describe('iterateForward', () => {
|
||||
it('yields all entries in order', () => {
|
||||
const c = makeNumericCursor([100, 200, 300], [1, 2, 3])
|
||||
const entries = [...c.iterateForward()]
|
||||
expect(entries).toHaveLength(3)
|
||||
expect(entries[0]).toEqual({ value: 100, entityIntId: 1, position: 0 })
|
||||
expect(entries[2]).toEqual({ value: 300, entityIntId: 3, position: 2 })
|
||||
})
|
||||
|
||||
it('skips tombstoned entries', () => {
|
||||
const c = makeNumericCursor([100, 200, 300], [1, 2, 3], [1])
|
||||
const entries = [...c.iterateForward()]
|
||||
expect(entries).toHaveLength(2)
|
||||
expect(entries.map(e => e.entityIntId)).toEqual([1, 3])
|
||||
})
|
||||
|
||||
it('starts from offset', () => {
|
||||
const c = makeNumericCursor([100, 200, 300, 400], [1, 2, 3, 4])
|
||||
const entries = [...c.iterateForward(2)]
|
||||
expect(entries).toHaveLength(2)
|
||||
expect(entries[0].value).toBe(300)
|
||||
})
|
||||
})
|
||||
|
||||
describe('iterateBackward', () => {
|
||||
it('yields all entries in reverse order', () => {
|
||||
const c = makeNumericCursor([100, 200, 300], [1, 2, 3])
|
||||
const entries = [...c.iterateBackward()]
|
||||
expect(entries).toHaveLength(3)
|
||||
expect(entries[0]).toEqual({ value: 300, entityIntId: 3, position: 2 })
|
||||
expect(entries[2]).toEqual({ value: 100, entityIntId: 1, position: 0 })
|
||||
})
|
||||
|
||||
it('skips tombstoned entries', () => {
|
||||
const c = makeNumericCursor([100, 200, 300], [1, 2, 3], [1])
|
||||
const entries = [...c.iterateBackward()]
|
||||
expect(entries).toHaveLength(2)
|
||||
expect(entries.map(e => e.entityIntId)).toEqual([3, 1])
|
||||
})
|
||||
})
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
// Zone map (min/max)
|
||||
// -----------------------------------------------------------------------
|
||||
|
||||
describe('zone map', () => {
|
||||
it('minValue returns first value', () => {
|
||||
const c = makeNumericCursor([100, 200, 300], [1, 2, 3])
|
||||
expect(c.minValue).toBe(100)
|
||||
})
|
||||
|
||||
it('maxValue returns last value', () => {
|
||||
const c = makeNumericCursor([100, 200, 300], [1, 2, 3])
|
||||
expect(c.maxValue).toBe(300)
|
||||
})
|
||||
|
||||
it('handles empty segment', () => {
|
||||
const c = makeNumericCursor([], [])
|
||||
expect(c.minValue).toBeUndefined()
|
||||
expect(c.maxValue).toBeUndefined()
|
||||
})
|
||||
})
|
||||
|
||||
describe('liveCount', () => {
|
||||
it('subtracts tombstones from count', () => {
|
||||
const c = makeNumericCursor([100, 200, 300], [1, 2, 3], [0, 2])
|
||||
expect(c.liveCount).toBe(1)
|
||||
})
|
||||
})
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
// TailBufferCursor
|
||||
// -----------------------------------------------------------------------
|
||||
|
||||
describe('TailBufferCursor', () => {
|
||||
it('iterates forward', () => {
|
||||
const c = new TailBufferCursor([10, 20, 30], new Uint32Array([1, 2, 3]), ValueType.Number)
|
||||
const entries = [...c.iterateForward()]
|
||||
expect(entries).toHaveLength(3)
|
||||
expect(entries[0].value).toBe(10)
|
||||
})
|
||||
|
||||
it('iterates backward', () => {
|
||||
const c = new TailBufferCursor([10, 20, 30], new Uint32Array([1, 2, 3]), ValueType.Number)
|
||||
const entries = [...c.iterateBackward()]
|
||||
expect(entries[0].value).toBe(30)
|
||||
})
|
||||
|
||||
it('point query returns matching IDs', () => {
|
||||
const c = new TailBufferCursor([10, 20, 20, 30], new Uint32Array([1, 2, 3, 4]), ValueType.Number)
|
||||
expect(c.getEntityIdsForValue(20)).toEqual([2, 3])
|
||||
})
|
||||
|
||||
it('min/max values', () => {
|
||||
const c = new TailBufferCursor([10, 20, 30], new Uint32Array([1, 2, 3]), ValueType.Number)
|
||||
expect(c.minValue).toBe(10)
|
||||
expect(c.maxValue).toBe(30)
|
||||
})
|
||||
})
|
||||
})
|
||||
501
tests/unit/indexes/columnStore/segment-format.test.ts
Normal file
501
tests/unit/indexes/columnStore/segment-format.test.ts
Normal file
|
|
@ -0,0 +1,501 @@
|
|||
/**
|
||||
* @module segment-format.test
|
||||
* @description Round-trip tests for the .cidx binary segment format.
|
||||
* The format is the shared contract between TypeScript and Rust —
|
||||
* every byte must be identical across languages.
|
||||
*/
|
||||
|
||||
import { describe, it, expect } from 'vitest'
|
||||
import {
|
||||
writeSegmentToBuffer,
|
||||
readSegmentFromBuffer,
|
||||
writeHeader,
|
||||
readHeader,
|
||||
writeFooter,
|
||||
readFooter,
|
||||
crc32,
|
||||
encodeNumericValues,
|
||||
decodeNumericValues,
|
||||
encodeFloatValues,
|
||||
decodeFloatValues,
|
||||
encodeStringValues,
|
||||
decodeStringValues,
|
||||
encodeEntityIds,
|
||||
decodeEntityIds
|
||||
} from '../../../../src/indexes/columnStore/ColumnSegmentFormat.js'
|
||||
import {
|
||||
CIDX_MAGIC,
|
||||
CIDX_VERSION,
|
||||
HEADER_SIZE,
|
||||
FOOTER_SIZE,
|
||||
ValueType,
|
||||
FLAG_MULTI_VALUE
|
||||
} from '../../../../src/indexes/columnStore/types.js'
|
||||
import { RoaringBitmap32 } from '../../../../src/utils/roaring/index.js'
|
||||
|
||||
describe('ColumnSegmentFormat', () => {
|
||||
// -----------------------------------------------------------------------
|
||||
// CRC32
|
||||
// -----------------------------------------------------------------------
|
||||
|
||||
describe('crc32', () => {
|
||||
it('produces consistent checksums', () => {
|
||||
const data = Buffer.from('hello world')
|
||||
const c1 = crc32(data)
|
||||
const c2 = crc32(data)
|
||||
expect(c1).toBe(c2)
|
||||
expect(typeof c1).toBe('number')
|
||||
expect(c1).toBeGreaterThan(0)
|
||||
})
|
||||
|
||||
it('different data produces different checksums', () => {
|
||||
const c1 = crc32(Buffer.from('hello'))
|
||||
const c2 = crc32(Buffer.from('world'))
|
||||
expect(c1).not.toBe(c2)
|
||||
})
|
||||
|
||||
it('handles empty buffer', () => {
|
||||
const c = crc32(Buffer.alloc(0))
|
||||
expect(typeof c).toBe('number')
|
||||
})
|
||||
})
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
// Header
|
||||
// -----------------------------------------------------------------------
|
||||
|
||||
describe('header', () => {
|
||||
it('round-trips all fields', () => {
|
||||
const header = {
|
||||
magic: CIDX_MAGIC,
|
||||
version: CIDX_VERSION,
|
||||
fieldName: 'createdAt',
|
||||
fieldNameLength: 9,
|
||||
valueType: ValueType.Number,
|
||||
level: 2,
|
||||
codec: 0,
|
||||
flags: 0,
|
||||
count: 50000
|
||||
}
|
||||
const buf = writeHeader(header)
|
||||
expect(buf.length).toBe(HEADER_SIZE)
|
||||
|
||||
const parsed = readHeader(buf)
|
||||
expect(parsed.magic).toBe(CIDX_MAGIC)
|
||||
expect(parsed.version).toBe(CIDX_VERSION)
|
||||
expect(parsed.fieldName).toBe('createdAt')
|
||||
expect(parsed.valueType).toBe(ValueType.Number)
|
||||
expect(parsed.level).toBe(2)
|
||||
expect(parsed.codec).toBe(0)
|
||||
expect(parsed.flags).toBe(0)
|
||||
expect(parsed.count).toBe(50000)
|
||||
})
|
||||
|
||||
it('truncates field names longer than 32 bytes', () => {
|
||||
const longName = 'a'.repeat(50)
|
||||
const header = {
|
||||
magic: CIDX_MAGIC,
|
||||
version: CIDX_VERSION,
|
||||
fieldName: longName,
|
||||
fieldNameLength: 32,
|
||||
valueType: ValueType.String,
|
||||
level: 0,
|
||||
codec: 0,
|
||||
flags: 0,
|
||||
count: 1
|
||||
}
|
||||
const buf = writeHeader(header)
|
||||
const parsed = readHeader(buf)
|
||||
expect(parsed.fieldName.length).toBe(32)
|
||||
})
|
||||
|
||||
it('preserves multi-value flag', () => {
|
||||
const header = {
|
||||
magic: CIDX_MAGIC,
|
||||
version: CIDX_VERSION,
|
||||
fieldName: '__words__',
|
||||
fieldNameLength: 9,
|
||||
valueType: ValueType.String,
|
||||
level: 0,
|
||||
codec: 0,
|
||||
flags: FLAG_MULTI_VALUE,
|
||||
count: 10000
|
||||
}
|
||||
const buf = writeHeader(header)
|
||||
const parsed = readHeader(buf)
|
||||
expect(parsed.flags & FLAG_MULTI_VALUE).toBe(FLAG_MULTI_VALUE)
|
||||
})
|
||||
|
||||
it('rejects invalid magic', () => {
|
||||
const buf = Buffer.alloc(HEADER_SIZE)
|
||||
buf.writeUInt32LE(0xDEADBEEF, 0)
|
||||
expect(() => readHeader(buf)).toThrow(/Invalid segment magic/)
|
||||
})
|
||||
|
||||
it('rejects unsupported version', () => {
|
||||
const buf = Buffer.alloc(HEADER_SIZE)
|
||||
buf.writeUInt32LE(CIDX_MAGIC, 0)
|
||||
buf.writeUInt16LE(99, 4) // bad version
|
||||
expect(() => readHeader(buf)).toThrow(/Unsupported segment version/)
|
||||
})
|
||||
})
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
// Value encoding
|
||||
// -----------------------------------------------------------------------
|
||||
|
||||
describe('numeric values', () => {
|
||||
it('round-trips integers', () => {
|
||||
const values = [100, 200, 300, 1700000000000]
|
||||
const buf = encodeNumericValues(values)
|
||||
expect(buf.length).toBe(values.length * 8)
|
||||
const decoded = decodeNumericValues(buf, values.length)
|
||||
expect(decoded).toEqual(values)
|
||||
})
|
||||
|
||||
it('handles negative values', () => {
|
||||
const values = [-1000, -1, 0, 1, 1000]
|
||||
const decoded = decodeNumericValues(encodeNumericValues(values), values.length)
|
||||
expect(decoded).toEqual(values)
|
||||
})
|
||||
|
||||
it('handles empty array', () => {
|
||||
const buf = encodeNumericValues([])
|
||||
expect(buf.length).toBe(0)
|
||||
const decoded = decodeNumericValues(buf, 0)
|
||||
expect(decoded).toEqual([])
|
||||
})
|
||||
})
|
||||
|
||||
describe('float values', () => {
|
||||
it('round-trips floats', () => {
|
||||
const values = [1.5, 2.7, 3.14159, 99.99]
|
||||
const buf = encodeFloatValues(values)
|
||||
const decoded = decodeFloatValues(buf, values.length)
|
||||
expect(decoded).toEqual(values)
|
||||
})
|
||||
|
||||
it('handles special values', () => {
|
||||
const values = [-Infinity, -0, 0, Infinity]
|
||||
const decoded = decodeFloatValues(encodeFloatValues(values), values.length)
|
||||
expect(decoded[0]).toBe(-Infinity)
|
||||
expect(decoded[3]).toBe(Infinity)
|
||||
})
|
||||
})
|
||||
|
||||
describe('string values', () => {
|
||||
it('round-trips strings', () => {
|
||||
const values = ['active', 'inactive', 'pending']
|
||||
const buf = encodeStringValues(values)
|
||||
const decoded = decodeStringValues(buf, values.length)
|
||||
expect(decoded).toEqual(values)
|
||||
})
|
||||
|
||||
it('handles empty strings', () => {
|
||||
const values = ['', 'a', '']
|
||||
const decoded = decodeStringValues(encodeStringValues(values), values.length)
|
||||
expect(decoded).toEqual(values)
|
||||
})
|
||||
|
||||
it('handles unicode', () => {
|
||||
const values = ['café', 'naïve', '日本語']
|
||||
const decoded = decodeStringValues(encodeStringValues(values), values.length)
|
||||
expect(decoded).toEqual(values)
|
||||
})
|
||||
|
||||
it('handles single string', () => {
|
||||
const decoded = decodeStringValues(encodeStringValues(['hello']), 1)
|
||||
expect(decoded).toEqual(['hello'])
|
||||
})
|
||||
})
|
||||
|
||||
describe('entity IDs', () => {
|
||||
it('round-trips Uint32Array', () => {
|
||||
const ids = new Uint32Array([1, 42, 1000, 4294967295]) // includes max u32
|
||||
const buf = encodeEntityIds(ids)
|
||||
expect(buf.length).toBe(ids.length * 4)
|
||||
const decoded = decodeEntityIds(buf, ids.length)
|
||||
expect(Array.from(decoded)).toEqual(Array.from(ids))
|
||||
})
|
||||
|
||||
it('round-trips number array', () => {
|
||||
const ids = [5, 10, 15]
|
||||
const decoded = decodeEntityIds(encodeEntityIds(ids), ids.length)
|
||||
expect(Array.from(decoded)).toEqual(ids)
|
||||
})
|
||||
})
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
// Full segment round-trip
|
||||
// -----------------------------------------------------------------------
|
||||
|
||||
describe('full segment', () => {
|
||||
it('round-trips a numeric segment', () => {
|
||||
const values = [1700000000000, 1700000001000, 1700000002000]
|
||||
const entityIds = new Uint32Array([10, 20, 30])
|
||||
|
||||
const buf = writeSegmentToBuffer(
|
||||
{
|
||||
fieldName: 'createdAt',
|
||||
fieldNameLength: 9,
|
||||
valueType: ValueType.Number,
|
||||
level: 0,
|
||||
codec: 0,
|
||||
flags: 0,
|
||||
count: 3
|
||||
},
|
||||
values,
|
||||
entityIds
|
||||
)
|
||||
|
||||
const parsed = readSegmentFromBuffer(buf)
|
||||
expect(parsed.header.fieldName).toBe('createdAt')
|
||||
expect(parsed.header.valueType).toBe(ValueType.Number)
|
||||
expect(parsed.header.count).toBe(3)
|
||||
expect(parsed.values).toEqual(values)
|
||||
expect(Array.from(parsed.entityIds)).toEqual([10, 20, 30])
|
||||
expect(parsed.tombstones.size).toBe(0)
|
||||
})
|
||||
|
||||
it('round-trips a string segment', () => {
|
||||
const values = ['active', 'inactive', 'pending']
|
||||
const entityIds = new Uint32Array([1, 2, 3])
|
||||
|
||||
const buf = writeSegmentToBuffer(
|
||||
{
|
||||
fieldName: 'status',
|
||||
fieldNameLength: 6,
|
||||
valueType: ValueType.String,
|
||||
level: 0,
|
||||
codec: 0,
|
||||
flags: 0,
|
||||
count: 3
|
||||
},
|
||||
values,
|
||||
entityIds
|
||||
)
|
||||
|
||||
const parsed = readSegmentFromBuffer(buf)
|
||||
expect(parsed.values).toEqual(values)
|
||||
expect(Array.from(parsed.entityIds)).toEqual([1, 2, 3])
|
||||
})
|
||||
|
||||
it('round-trips a float segment', () => {
|
||||
const values = [9.99, 19.99, 49.99]
|
||||
const entityIds = new Uint32Array([100, 200, 300])
|
||||
|
||||
const buf = writeSegmentToBuffer(
|
||||
{
|
||||
fieldName: 'price',
|
||||
fieldNameLength: 5,
|
||||
valueType: ValueType.Float,
|
||||
level: 1,
|
||||
codec: 0,
|
||||
flags: 0,
|
||||
count: 3
|
||||
},
|
||||
values,
|
||||
entityIds
|
||||
)
|
||||
|
||||
const parsed = readSegmentFromBuffer(buf)
|
||||
expect(parsed.values).toEqual(values)
|
||||
expect(parsed.header.level).toBe(1)
|
||||
})
|
||||
|
||||
it('round-trips a boolean segment', () => {
|
||||
const values = [0, 0, 1, 1]
|
||||
const entityIds = new Uint32Array([1, 2, 3, 4])
|
||||
|
||||
const buf = writeSegmentToBuffer(
|
||||
{
|
||||
fieldName: 'isActive',
|
||||
fieldNameLength: 8,
|
||||
valueType: ValueType.Boolean,
|
||||
level: 0,
|
||||
codec: 0,
|
||||
flags: 0,
|
||||
count: 4
|
||||
},
|
||||
values,
|
||||
entityIds
|
||||
)
|
||||
|
||||
const parsed = readSegmentFromBuffer(buf)
|
||||
expect(parsed.values).toEqual(values)
|
||||
})
|
||||
|
||||
it('round-trips with tombstones', () => {
|
||||
const values = [100, 200, 300, 400, 500]
|
||||
const entityIds = new Uint32Array([1, 2, 3, 4, 5])
|
||||
const tombstones = new RoaringBitmap32([1, 3]) // positions 1 and 3 are deleted
|
||||
|
||||
const buf = writeSegmentToBuffer(
|
||||
{
|
||||
fieldName: 'score',
|
||||
fieldNameLength: 5,
|
||||
valueType: ValueType.Number,
|
||||
level: 0,
|
||||
codec: 0,
|
||||
flags: 0,
|
||||
count: 5
|
||||
},
|
||||
values,
|
||||
entityIds,
|
||||
tombstones
|
||||
)
|
||||
|
||||
const parsed = readSegmentFromBuffer(buf)
|
||||
expect(parsed.tombstones.has(1)).toBe(true)
|
||||
expect(parsed.tombstones.has(3)).toBe(true)
|
||||
expect(parsed.tombstones.has(0)).toBe(false)
|
||||
expect(parsed.tombstones.has(2)).toBe(false)
|
||||
expect(parsed.tombstones.size).toBe(2)
|
||||
})
|
||||
|
||||
it('round-trips an empty segment', () => {
|
||||
const buf = writeSegmentToBuffer(
|
||||
{
|
||||
fieldName: 'empty',
|
||||
fieldNameLength: 5,
|
||||
valueType: ValueType.Number,
|
||||
level: 0,
|
||||
codec: 0,
|
||||
flags: 0,
|
||||
count: 0
|
||||
},
|
||||
[],
|
||||
new Uint32Array(0)
|
||||
)
|
||||
|
||||
const parsed = readSegmentFromBuffer(buf)
|
||||
expect(parsed.header.count).toBe(0)
|
||||
expect(parsed.values).toEqual([])
|
||||
expect(parsed.entityIds.length).toBe(0)
|
||||
})
|
||||
|
||||
it('round-trips a single-entry segment', () => {
|
||||
const buf = writeSegmentToBuffer(
|
||||
{
|
||||
fieldName: 'x',
|
||||
fieldNameLength: 1,
|
||||
valueType: ValueType.Number,
|
||||
level: 0,
|
||||
codec: 0,
|
||||
flags: 0,
|
||||
count: 1
|
||||
},
|
||||
[42],
|
||||
new Uint32Array([7])
|
||||
)
|
||||
|
||||
const parsed = readSegmentFromBuffer(buf)
|
||||
expect(parsed.values).toEqual([42])
|
||||
expect(Array.from(parsed.entityIds)).toEqual([7])
|
||||
})
|
||||
|
||||
it('round-trips a multi-value segment (words)', () => {
|
||||
const words = ['algorithm', 'learning', 'machine', 'neural']
|
||||
const entityIds = new Uint32Array([5, 5, 5, 10]) // entity 5 has 3 words, entity 10 has 1
|
||||
|
||||
const buf = writeSegmentToBuffer(
|
||||
{
|
||||
fieldName: '__words__',
|
||||
fieldNameLength: 9,
|
||||
valueType: ValueType.String,
|
||||
level: 0,
|
||||
codec: 0,
|
||||
flags: FLAG_MULTI_VALUE,
|
||||
count: 4
|
||||
},
|
||||
words,
|
||||
entityIds
|
||||
)
|
||||
|
||||
const parsed = readSegmentFromBuffer(buf)
|
||||
expect(parsed.values).toEqual(words)
|
||||
expect(Array.from(parsed.entityIds)).toEqual([5, 5, 5, 10])
|
||||
expect(parsed.header.flags & FLAG_MULTI_VALUE).toBe(FLAG_MULTI_VALUE)
|
||||
})
|
||||
})
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
// CRC validation
|
||||
// -----------------------------------------------------------------------
|
||||
|
||||
describe('CRC validation', () => {
|
||||
it('detects tampered data', () => {
|
||||
const buf = writeSegmentToBuffer(
|
||||
{
|
||||
fieldName: 'test',
|
||||
fieldNameLength: 4,
|
||||
valueType: ValueType.Number,
|
||||
level: 0,
|
||||
codec: 0,
|
||||
flags: 0,
|
||||
count: 3
|
||||
},
|
||||
[1, 2, 3],
|
||||
new Uint32Array([10, 20, 30])
|
||||
)
|
||||
|
||||
// Tamper with a value byte
|
||||
const tampered = Buffer.from(buf)
|
||||
tampered[HEADER_SIZE + 4] ^= 0xFF
|
||||
|
||||
expect(() => readSegmentFromBuffer(tampered, true)).toThrow(/CRC32 mismatch/)
|
||||
})
|
||||
|
||||
it('passes with validateCrc=false on tampered data', () => {
|
||||
const buf = writeSegmentToBuffer(
|
||||
{
|
||||
fieldName: 'test',
|
||||
fieldNameLength: 4,
|
||||
valueType: ValueType.Number,
|
||||
level: 0,
|
||||
codec: 0,
|
||||
flags: 0,
|
||||
count: 3
|
||||
},
|
||||
[1, 2, 3],
|
||||
new Uint32Array([10, 20, 30])
|
||||
)
|
||||
|
||||
const tampered = Buffer.from(buf)
|
||||
tampered[HEADER_SIZE + 4] ^= 0xFF
|
||||
|
||||
// Should not throw with validation disabled
|
||||
const parsed = readSegmentFromBuffer(tampered, false)
|
||||
expect(parsed.header.count).toBe(3)
|
||||
})
|
||||
})
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
// Error handling
|
||||
// -----------------------------------------------------------------------
|
||||
|
||||
describe('error handling', () => {
|
||||
it('rejects buffer smaller than header + footer', () => {
|
||||
const tiny = Buffer.alloc(10)
|
||||
expect(() => readSegmentFromBuffer(tiny)).toThrow(/Buffer too small/)
|
||||
})
|
||||
|
||||
it('rejects mismatched values and entityIds length', () => {
|
||||
expect(() =>
|
||||
writeSegmentToBuffer(
|
||||
{
|
||||
fieldName: 'x',
|
||||
fieldNameLength: 1,
|
||||
valueType: ValueType.Number,
|
||||
level: 0,
|
||||
codec: 0,
|
||||
flags: 0,
|
||||
count: 2
|
||||
},
|
||||
[1, 2],
|
||||
new Uint32Array([10]) // mismatched!
|
||||
)
|
||||
).toThrow(/same length/)
|
||||
})
|
||||
})
|
||||
})
|
||||
169
tests/unit/indexes/columnStore/tail-buffer.test.ts
Normal file
169
tests/unit/indexes/columnStore/tail-buffer.test.ts
Normal file
|
|
@ -0,0 +1,169 @@
|
|||
/**
|
||||
* @module tail-buffer.test
|
||||
* @description Tests for the per-field in-memory write buffer.
|
||||
*/
|
||||
|
||||
import { describe, it, expect } from 'vitest'
|
||||
import { ColumnTailBuffer } from '../../../../src/indexes/columnStore/ColumnTailBuffer.js'
|
||||
import { ValueType, DEFAULT_FLUSH_THRESHOLD } from '../../../../src/indexes/columnStore/types.js'
|
||||
|
||||
describe('ColumnTailBuffer', () => {
|
||||
describe('numeric values', () => {
|
||||
it('drain returns entries sorted by value', () => {
|
||||
const buf = new ColumnTailBuffer('createdAt', ValueType.Number)
|
||||
buf.add(300, 3)
|
||||
buf.add(100, 1)
|
||||
buf.add(200, 2)
|
||||
|
||||
const { values, entityIds } = buf.drain()
|
||||
expect(values).toEqual([100, 200, 300])
|
||||
expect(Array.from(entityIds)).toEqual([1, 2, 3])
|
||||
})
|
||||
|
||||
it('breaks ties by entityIntId ascending', () => {
|
||||
const buf = new ColumnTailBuffer('ts', ValueType.Number)
|
||||
buf.add(100, 30)
|
||||
buf.add(100, 10)
|
||||
buf.add(100, 20)
|
||||
|
||||
const { entityIds } = buf.drain()
|
||||
expect(Array.from(entityIds)).toEqual([10, 20, 30])
|
||||
})
|
||||
|
||||
it('handles negative values', () => {
|
||||
const buf = new ColumnTailBuffer('offset', ValueType.Number)
|
||||
buf.add(-10, 1)
|
||||
buf.add(5, 2)
|
||||
buf.add(-20, 3)
|
||||
|
||||
const { values } = buf.drain()
|
||||
expect(values).toEqual([-20, -10, 5])
|
||||
})
|
||||
})
|
||||
|
||||
describe('string values', () => {
|
||||
it('drain returns entries sorted lexicographically', () => {
|
||||
const buf = new ColumnTailBuffer('status', ValueType.String)
|
||||
buf.add('pending', 3)
|
||||
buf.add('active', 1)
|
||||
buf.add('inactive', 2)
|
||||
|
||||
const { values, entityIds } = buf.drain()
|
||||
expect(values).toEqual(['active', 'inactive', 'pending'])
|
||||
expect(Array.from(entityIds)).toEqual([1, 2, 3])
|
||||
})
|
||||
})
|
||||
|
||||
describe('boolean values', () => {
|
||||
it('sorts false (0) before true (1)', () => {
|
||||
const buf = new ColumnTailBuffer('isActive', ValueType.Boolean)
|
||||
buf.add(1, 1)
|
||||
buf.add(0, 2)
|
||||
buf.add(1, 3)
|
||||
buf.add(0, 4)
|
||||
|
||||
const { values, entityIds } = buf.drain()
|
||||
expect(values).toEqual([0, 0, 1, 1])
|
||||
expect(Array.from(entityIds)).toEqual([2, 4, 1, 3])
|
||||
})
|
||||
})
|
||||
|
||||
describe('removal', () => {
|
||||
it('removes pending entries from buffer', () => {
|
||||
const buf = new ColumnTailBuffer('x', ValueType.Number)
|
||||
buf.add(100, 1)
|
||||
buf.add(200, 2)
|
||||
buf.add(300, 3)
|
||||
|
||||
buf.remove(2)
|
||||
|
||||
const { values, entityIds, pendingRemovals } = buf.drain()
|
||||
expect(values).toEqual([100, 300])
|
||||
expect(Array.from(entityIds)).toEqual([1, 3])
|
||||
expect(pendingRemovals.has(2)).toBe(true)
|
||||
})
|
||||
|
||||
it('tracks pending removals for existing segments', () => {
|
||||
const buf = new ColumnTailBuffer('x', ValueType.Number)
|
||||
buf.remove(42)
|
||||
buf.remove(99)
|
||||
|
||||
const { pendingRemovals } = buf.drain()
|
||||
expect(pendingRemovals.has(42)).toBe(true)
|
||||
expect(pendingRemovals.has(99)).toBe(true)
|
||||
})
|
||||
})
|
||||
|
||||
describe('threshold', () => {
|
||||
it('shouldFlush returns false below threshold', () => {
|
||||
const buf = new ColumnTailBuffer('x', ValueType.Number, 3)
|
||||
buf.add(1, 1)
|
||||
buf.add(2, 2)
|
||||
expect(buf.shouldFlush()).toBe(false)
|
||||
})
|
||||
|
||||
it('shouldFlush returns true at threshold', () => {
|
||||
const buf = new ColumnTailBuffer('x', ValueType.Number, 3)
|
||||
buf.add(1, 1)
|
||||
buf.add(2, 2)
|
||||
buf.add(3, 3)
|
||||
expect(buf.shouldFlush()).toBe(true)
|
||||
})
|
||||
|
||||
it('uses default threshold when not specified', () => {
|
||||
const buf = new ColumnTailBuffer('x', ValueType.Number)
|
||||
expect(buf.threshold).toBe(DEFAULT_FLUSH_THRESHOLD)
|
||||
})
|
||||
})
|
||||
|
||||
describe('drain clears buffer', () => {
|
||||
it('buffer is empty after drain', () => {
|
||||
const buf = new ColumnTailBuffer('x', ValueType.Number)
|
||||
buf.add(1, 1)
|
||||
buf.add(2, 2)
|
||||
expect(buf.size).toBe(2)
|
||||
|
||||
buf.drain()
|
||||
expect(buf.size).toBe(0)
|
||||
})
|
||||
|
||||
it('removals are cleared after drain', () => {
|
||||
const buf = new ColumnTailBuffer('x', ValueType.Number)
|
||||
buf.remove(42)
|
||||
buf.drain()
|
||||
|
||||
const { pendingRemovals } = buf.drain()
|
||||
expect(pendingRemovals.size).toBe(0)
|
||||
})
|
||||
})
|
||||
|
||||
describe('clear', () => {
|
||||
it('clears entries and removals without returning data', () => {
|
||||
const buf = new ColumnTailBuffer('x', ValueType.Number)
|
||||
buf.add(1, 1)
|
||||
buf.remove(99)
|
||||
|
||||
buf.clear()
|
||||
expect(buf.size).toBe(0)
|
||||
|
||||
const { values, pendingRemovals } = buf.drain()
|
||||
expect(values).toEqual([])
|
||||
expect(pendingRemovals.size).toBe(0)
|
||||
})
|
||||
})
|
||||
|
||||
describe('multi-value (words)', () => {
|
||||
it('supports multiple entries with the same entityIntId', () => {
|
||||
const buf = new ColumnTailBuffer('__words__', ValueType.String)
|
||||
buf.add('machine', 5)
|
||||
buf.add('learning', 5)
|
||||
buf.add('neural', 10)
|
||||
buf.add('algorithm', 5)
|
||||
|
||||
const { values, entityIds } = buf.drain()
|
||||
// Sorted lexicographically
|
||||
expect(values).toEqual(['algorithm', 'learning', 'machine', 'neural'])
|
||||
expect(Array.from(entityIds)).toEqual([5, 5, 5, 10])
|
||||
})
|
||||
})
|
||||
})
|
||||
|
|
@ -1,297 +0,0 @@
|
|||
/**
|
||||
* MetadataIndex Automatic Bucketing Tests (v3.41.0)
|
||||
* Verify that temporal fields are automatically bucketed from the start
|
||||
* to prevent file pollution while enabling range queries
|
||||
*/
|
||||
|
||||
import { describe, it, expect, beforeEach, afterEach } from 'vitest'
|
||||
import { MemoryStorage } from '../../../src/storage/adapters/memoryStorage.js'
|
||||
import { MetadataIndexManager } from '../../../src/utils/metadataIndex.js'
|
||||
|
||||
describe('MetadataIndex Automatic Bucketing (v3.41.0)', () => {
|
||||
let storage: MemoryStorage
|
||||
let index: MetadataIndexManager
|
||||
// Use a fixed base timestamp to avoid test pollution from Date.now() bucketing
|
||||
const TEST_BASE_TIME = 1700000000000 // Fixed timestamp (Nov 14, 2023)
|
||||
|
||||
beforeEach(() => {
|
||||
storage = new MemoryStorage()
|
||||
// No excludeFields config - fully automatic!
|
||||
index = new MetadataIndexManager(storage, {})
|
||||
})
|
||||
|
||||
afterEach(async () => {
|
||||
// MemoryStorage doesn't have a close() method
|
||||
})
|
||||
|
||||
it('should automatically bucket timestamps from the first operation', async () => {
|
||||
// Add 100 entities with unique timestamps (every millisecond)
|
||||
const baseTime = TEST_BASE_TIME
|
||||
for (let i = 0; i < 100; i++) {
|
||||
await index.addToIndex(`entity${i}`, {
|
||||
noun: 'character',
|
||||
modified: baseTime + i // 100 unique timestamps
|
||||
})
|
||||
}
|
||||
|
||||
await index.flush()
|
||||
|
||||
// Without bucketing: would create 100 metadata index files
|
||||
// With bucketing: should create only ~2 files (all fit in one 1-minute bucket)
|
||||
|
||||
// Get available values for 'modified' field
|
||||
const modifiedValues = await index.getFilterValues('modified')
|
||||
|
||||
// Should have very few unique values (all bucketed to 1-minute intervals)
|
||||
expect(modifiedValues.length).toBeLessThan(5) // Much less than 100!
|
||||
expect(modifiedValues.length).toBeGreaterThan(0) // But not zero
|
||||
})
|
||||
|
||||
it('should prevent file pollution with high-cardinality timestamps', async () => {
|
||||
// Simulate the bug scenario: 575 relationships with unique timestamps
|
||||
const baseTime = TEST_BASE_TIME + 10000000 // Offset to avoid bucket collision with other tests
|
||||
for (let i = 0; i < 575; i++) {
|
||||
await index.addToIndex(`verb${i}`, {
|
||||
verb: 'relates_to',
|
||||
modified: baseTime + (i * 1000), // Every second
|
||||
accessed: baseTime + (i * 500) // Every half-second
|
||||
})
|
||||
}
|
||||
|
||||
await index.flush()
|
||||
|
||||
// Without bucketing: 575 unique modified + 1150 unique accessed = 1725 index files
|
||||
// With bucketing: ~10 modified buckets + ~20 accessed buckets = ~30 index files
|
||||
|
||||
const modifiedValues = await index.getFilterValues('modified')
|
||||
const accessedValues = await index.getFilterValues('accessed')
|
||||
|
||||
// Should have created FAR fewer index entries due to bucketing
|
||||
expect(modifiedValues.length).toBeLessThan(100) // Not 575!
|
||||
expect(accessedValues.length).toBeLessThan(200) // Not 1150!
|
||||
|
||||
// But should still have indexed them (not excluded)
|
||||
expect(modifiedValues.length).toBeGreaterThan(0)
|
||||
expect(accessedValues.length).toBeGreaterThan(0)
|
||||
})
|
||||
|
||||
it('should enable range queries on bucketed timestamp fields', async () => {
|
||||
// Add entities with timestamps at exact bucket boundaries (every minute)
|
||||
const bucketStart = Math.floor((TEST_BASE_TIME + 20000000) / 60000) * 60000
|
||||
for (let i = 0; i < 100; i++) {
|
||||
await index.addToIndex(`entity${i}`, {
|
||||
noun: 'character',
|
||||
modified: bucketStart + (i * 60000) // Every minute = every bucket
|
||||
})
|
||||
}
|
||||
|
||||
await index.flush()
|
||||
|
||||
// Query: modified >= (bucketStart + 30 minutes)
|
||||
const thirtyMinutesLater = bucketStart + (30 * 60000)
|
||||
const results = await index.getIdsForFilter({
|
||||
modified: { greaterThanOrEqual: thirtyMinutesLater }
|
||||
})
|
||||
|
||||
// Should find entities 30-99 (70 entities)
|
||||
// Note: With bucketing, we expect exactly 70 entities (30-99 inclusive)
|
||||
expect(results.length).toBe(70)
|
||||
|
||||
// Verify results are correct
|
||||
results.forEach(id => {
|
||||
const num = parseInt(id.replace('entity', ''))
|
||||
expect(num).toBeGreaterThanOrEqual(30) // Should be in range
|
||||
})
|
||||
})
|
||||
|
||||
it('should work with zero configuration (no excludeFields, no rangeQueryFields)', async () => {
|
||||
// Create index with NO config at all
|
||||
const autoIndex = new MetadataIndexManager(storage, {})
|
||||
const testTime = TEST_BASE_TIME + 30000000 // Offset to avoid bucket collision
|
||||
|
||||
// Add entity with multiple field types
|
||||
await autoIndex.addToIndex('test1', {
|
||||
id: 'uuid-123', // Should be excluded (in default excludeFields)
|
||||
noun: 'character', // Should be indexed (low cardinality)
|
||||
modified: testTime, // Should be indexed + bucketed (temporal)
|
||||
accessed: testTime, // Should be indexed + bucketed (temporal)
|
||||
content: 'long text content', // Should be excluded (in default excludeFields)
|
||||
customTimestamp: testTime // Should be indexed + bucketed (has 'time' in name)
|
||||
})
|
||||
|
||||
await autoIndex.flush()
|
||||
|
||||
// Verify 'modified' was indexed and bucketed
|
||||
const modifiedValues = await autoIndex.getFilterValues('modified')
|
||||
expect(modifiedValues.length).toBeGreaterThan(0) // Indexed!
|
||||
|
||||
// Verify 'accessed' was indexed and bucketed
|
||||
const accessedValues = await autoIndex.getFilterValues('accessed')
|
||||
expect(accessedValues.length).toBeGreaterThan(0) // Indexed!
|
||||
|
||||
// Verify 'customTimestamp' was indexed and bucketed
|
||||
const customValues = await autoIndex.getFilterValues('customTimestamp')
|
||||
expect(customValues.length).toBeGreaterThan(0) // Indexed!
|
||||
|
||||
// Verify 'id' was excluded
|
||||
const idValues = await autoIndex.getFilterValues('id')
|
||||
expect(idValues.length).toBe(0) // Excluded!
|
||||
|
||||
// Verify 'content' was excluded
|
||||
const contentValues = await autoIndex.getFilterValues('content')
|
||||
expect(contentValues.length).toBe(0) // Excluded!
|
||||
})
|
||||
|
||||
it('should handle fields with "time" or "date" in their names', async () => {
|
||||
// Add entities with various timestamp field names
|
||||
const testTime = TEST_BASE_TIME + 40000000 // Offset to avoid bucket collision
|
||||
await index.addToIndex('entity1', {
|
||||
timestamp: testTime,
|
||||
createdAt: testTime,
|
||||
updatedAt: testTime,
|
||||
birthdate: testTime,
|
||||
eventTime: testTime,
|
||||
lastSeen: testTime // Doesn't match pattern, won't be bucketed
|
||||
})
|
||||
|
||||
await index.flush()
|
||||
|
||||
// Fields with 'time' or 'date' should be bucketed
|
||||
const timestampValues = await index.getFilterValues('timestamp')
|
||||
const createdAtValues = await index.getFilterValues('createdAt')
|
||||
const updatedAtValues = await index.getFilterValues('updatedAt')
|
||||
const birthdateValues = await index.getFilterValues('birthdate')
|
||||
const eventTimeValues = await index.getFilterValues('eventTime')
|
||||
|
||||
// All should be indexed (not excluded)
|
||||
expect(timestampValues.length).toBeGreaterThan(0)
|
||||
expect(createdAtValues.length).toBeGreaterThan(0)
|
||||
expect(updatedAtValues.length).toBeGreaterThan(0)
|
||||
expect(birthdateValues.length).toBeGreaterThan(0)
|
||||
expect(eventTimeValues.length).toBeGreaterThan(0)
|
||||
})
|
||||
|
||||
it('should bucket timestamps to 1-minute intervals by default', async () => {
|
||||
// Use a timestamp at the start of a minute bucket to ensure 30s apart stays in same bucket
|
||||
const bucketStart = Math.floor((TEST_BASE_TIME + 50000000) / 60000) * 60000
|
||||
await index.addToIndex('entity1', { modified: bucketStart })
|
||||
await index.addToIndex('entity2', { modified: bucketStart + 30000 }) // +30 seconds, still in same bucket
|
||||
|
||||
await index.flush()
|
||||
|
||||
// Both should be in the same 1-minute bucket
|
||||
const ids1 = await index.getIds('modified', bucketStart)
|
||||
const ids2 = await index.getIds('modified', bucketStart + 30000)
|
||||
|
||||
// Both queries should return both entities (same bucket)
|
||||
expect(ids1.length).toBe(2)
|
||||
expect(ids2.length).toBe(2)
|
||||
expect(ids1.sort()).toEqual(ids2.sort())
|
||||
})
|
||||
|
||||
it('should create separate buckets for timestamps >1 minute apart', async () => {
|
||||
// Use bucket boundaries to ensure entities are in different buckets
|
||||
const bucket1Start = Math.floor((TEST_BASE_TIME + 60000000) / 60000) * 60000
|
||||
const bucket2Start = bucket1Start + 120000 // +2 minutes (definitely different bucket)
|
||||
await index.addToIndex('entity1', { modified: bucket1Start })
|
||||
await index.addToIndex('entity2', { modified: bucket2Start })
|
||||
|
||||
await index.flush()
|
||||
|
||||
// Should be in different 1-minute buckets
|
||||
const ids1 = await index.getIds('modified', bucket1Start)
|
||||
const ids2 = await index.getIds('modified', bucket2Start)
|
||||
|
||||
// Each query should return only its own entity
|
||||
expect(ids1).toContain('entity1')
|
||||
expect(ids1).not.toContain('entity2')
|
||||
expect(ids2).toContain('entity2')
|
||||
expect(ids2).not.toContain('entity1')
|
||||
})
|
||||
|
||||
it('should handle range queries across bucket boundaries', async () => {
|
||||
// Use explicit bucket boundaries for predictable range queries
|
||||
const bucket0 = Math.floor((TEST_BASE_TIME + 70000000) / 60000) * 60000
|
||||
const bucket1 = bucket0 + 60000 // +1 minute
|
||||
const bucket2 = bucket0 + 120000 // +2 minutes
|
||||
const bucket3 = bucket0 + 180000 // +3 minutes
|
||||
|
||||
await index.addToIndex('entity0', { modified: bucket0 })
|
||||
await index.addToIndex('entity1', { modified: bucket1 })
|
||||
await index.addToIndex('entity2', { modified: bucket2 })
|
||||
await index.addToIndex('entity3', { modified: bucket3 })
|
||||
|
||||
await index.flush()
|
||||
|
||||
// Query: modified > bucket1 (should get entity2 and entity3, not entity0 or entity1)
|
||||
const results = await index.getIdsForFilter({
|
||||
modified: { greaterThan: bucket1 }
|
||||
})
|
||||
|
||||
// Should include entities after bucket1
|
||||
expect(results).toContain('entity2')
|
||||
expect(results).toContain('entity3')
|
||||
|
||||
// Should not include entity0 or entity1 (at or before bucket1)
|
||||
expect(results).not.toContain('entity0')
|
||||
expect(results).not.toContain('entity1')
|
||||
})
|
||||
|
||||
it('should not break non-temporal numeric fields', async () => {
|
||||
// Add entities with regular numeric fields (not timestamps)
|
||||
await index.addToIndex('entity1', {
|
||||
noun: 'item',
|
||||
price: 19.99,
|
||||
quantity: 5,
|
||||
rating: 4.5
|
||||
})
|
||||
|
||||
await index.addToIndex('entity2', {
|
||||
noun: 'item',
|
||||
price: 29.99,
|
||||
quantity: 3,
|
||||
rating: 4.7
|
||||
})
|
||||
|
||||
await index.flush()
|
||||
|
||||
// Non-temporal numeric fields should NOT be bucketed
|
||||
const priceValues = await index.getFilterValues('price')
|
||||
const quantityValues = await index.getFilterValues('quantity')
|
||||
const ratingValues = await index.getFilterValues('rating')
|
||||
|
||||
// Each unique value should create its own index entry
|
||||
expect(priceValues).toContain('19.99')
|
||||
expect(priceValues).toContain('29.99')
|
||||
expect(quantityValues).toContain('5')
|
||||
expect(quantityValues).toContain('3')
|
||||
expect(ratingValues.length).toBeGreaterThan(0)
|
||||
})
|
||||
|
||||
it('should maintain backward compatibility with existing code', async () => {
|
||||
// Existing code that used excludeFields should still work
|
||||
const legacyIndex = new MetadataIndexManager(storage, {
|
||||
excludeFields: ['internalField', 'debugData']
|
||||
})
|
||||
const testTime = TEST_BASE_TIME + 90000000 // Offset to avoid bucket collision
|
||||
|
||||
await legacyIndex.addToIndex('entity1', {
|
||||
noun: 'character',
|
||||
modified: testTime, // Should be indexed + bucketed
|
||||
internalField: 'secret', // Should be excluded
|
||||
debugData: 'verbose logs' // Should be excluded
|
||||
})
|
||||
|
||||
await legacyIndex.flush()
|
||||
|
||||
// Modified should be indexed
|
||||
const modifiedValues = await legacyIndex.getFilterValues('modified')
|
||||
expect(modifiedValues.length).toBeGreaterThan(0)
|
||||
|
||||
// Custom excluded fields should be excluded
|
||||
const internalValues = await legacyIndex.getFilterValues('internalField')
|
||||
const debugValues = await legacyIndex.getFilterValues('debugData')
|
||||
expect(internalValues.length).toBe(0)
|
||||
expect(debugValues.length).toBe(0)
|
||||
})
|
||||
})
|
||||
Loading…
Add table
Add a link
Reference in a new issue