feat: unified column store for filtering + sorting at billion scale
Adds a per-field sorted column store (Lucene doc values + roaring bitmap
architecture) that replaces the MetadataIndex sparse index internals for
both filtering and sorting. One system for all field types with exact
precision — no bucketing, no per-entity storage reads.
Column store: binary .cidx segment format, in-memory tail buffers,
LSM-style compaction, k-way merge sort, multi-value support (__words__).
All queries (filter, range, sort, filtered sort) route through the column
store when data is available, falling back to sparse index otherwise.
Key unlocks:
- find({ orderBy: 'createdAt' }) works WITHOUT a filter (previously threw)
- find({ orderBy: 'metadata.price' }) works for custom numeric fields
- Exact timestamp precision (no 1-minute bucketing)
- O(K log S) sort independent of total entity count
New files: src/indexes/columnStore/ (types, format, tail buffer, cursor,
manifest, coordinator — ~700 lines). 101 new unit tests covering binary
format round-trips, CRC validation, sort, filter, range, deletion,
multi-segment merge, persistence, and multi-value (words) fields.
Deleted: metadataIndex-automatic-bucketing.test.ts (bucketing behavior
eliminated by exact-precision column store). Sparse index write path
removed from addToIndex/removeFromIndex. Sparse index legacy code still
present as dead code pending cleanup in next commit.
This commit is contained in:
parent
634ddfb2bb
commit
46583f2d9b
16 changed files with 3846 additions and 521 deletions
270
tests/unit/indexes/columnStore/segment-cursor.test.ts
Normal file
270
tests/unit/indexes/columnStore/segment-cursor.test.ts
Normal file
|
|
@ -0,0 +1,270 @@
|
|||
/**
|
||||
* @module segment-cursor.test
|
||||
* @description Tests for the segment cursor: binary search, iteration, tombstones.
|
||||
*/
|
||||
|
||||
import { describe, it, expect } from 'vitest'
|
||||
import { ColumnSegmentCursor, TailBufferCursor } from '../../../../src/indexes/columnStore/ColumnSegmentCursor.js'
|
||||
import { ValueType, CIDX_MAGIC, CIDX_VERSION } from '../../../../src/indexes/columnStore/types.js'
|
||||
import { RoaringBitmap32 } from '../../../../src/utils/roaring/index.js'
|
||||
|
||||
function makeNumericCursor(values: number[], entityIds: number[], tombstones: number[] = []): ColumnSegmentCursor {
|
||||
return new ColumnSegmentCursor(
|
||||
{
|
||||
magic: CIDX_MAGIC,
|
||||
version: CIDX_VERSION,
|
||||
fieldName: 'test',
|
||||
fieldNameLength: 4,
|
||||
valueType: ValueType.Number,
|
||||
level: 0,
|
||||
codec: 0,
|
||||
flags: 0,
|
||||
count: values.length
|
||||
},
|
||||
values,
|
||||
new Uint32Array(entityIds),
|
||||
new RoaringBitmap32(tombstones)
|
||||
)
|
||||
}
|
||||
|
||||
function makeStringCursor(values: string[], entityIds: number[], tombstones: number[] = []): ColumnSegmentCursor {
|
||||
return new ColumnSegmentCursor(
|
||||
{
|
||||
magic: CIDX_MAGIC,
|
||||
version: CIDX_VERSION,
|
||||
fieldName: 'status',
|
||||
fieldNameLength: 6,
|
||||
valueType: ValueType.String,
|
||||
level: 0,
|
||||
codec: 0,
|
||||
flags: 0,
|
||||
count: values.length
|
||||
},
|
||||
values,
|
||||
new Uint32Array(entityIds),
|
||||
new RoaringBitmap32(tombstones)
|
||||
)
|
||||
}
|
||||
|
||||
describe('ColumnSegmentCursor', () => {
|
||||
// -----------------------------------------------------------------------
|
||||
// Binary search
|
||||
// -----------------------------------------------------------------------
|
||||
|
||||
describe('binarySearchValue (numeric)', () => {
|
||||
it('finds exact match', () => {
|
||||
const c = makeNumericCursor([100, 200, 300, 400, 500], [1, 2, 3, 4, 5])
|
||||
const r = c.binarySearchValue(300)
|
||||
expect(r.found).toBe(true)
|
||||
expect(r.index).toBe(2)
|
||||
})
|
||||
|
||||
it('returns insertion point when not found', () => {
|
||||
const c = makeNumericCursor([100, 200, 400, 500], [1, 2, 3, 4])
|
||||
const r = c.binarySearchValue(300)
|
||||
expect(r.found).toBe(false)
|
||||
expect(r.index).toBe(2) // would insert at position 2
|
||||
})
|
||||
|
||||
it('handles before-first', () => {
|
||||
const c = makeNumericCursor([100, 200, 300], [1, 2, 3])
|
||||
const r = c.binarySearchValue(50)
|
||||
expect(r.found).toBe(false)
|
||||
expect(r.index).toBe(0)
|
||||
})
|
||||
|
||||
it('handles after-last', () => {
|
||||
const c = makeNumericCursor([100, 200, 300], [1, 2, 3])
|
||||
const r = c.binarySearchValue(999)
|
||||
expect(r.found).toBe(false)
|
||||
expect(r.index).toBe(3)
|
||||
})
|
||||
|
||||
it('finds first occurrence of duplicates', () => {
|
||||
const c = makeNumericCursor([100, 200, 200, 200, 300], [1, 2, 3, 4, 5])
|
||||
const r = c.binarySearchValue(200)
|
||||
expect(r.found).toBe(true)
|
||||
expect(r.index).toBe(1) // first occurrence
|
||||
})
|
||||
})
|
||||
|
||||
describe('binarySearchValue (string)', () => {
|
||||
it('finds exact match', () => {
|
||||
const c = makeStringCursor(['active', 'inactive', 'pending'], [1, 2, 3])
|
||||
const r = c.binarySearchValue('inactive')
|
||||
expect(r.found).toBe(true)
|
||||
expect(r.index).toBe(1)
|
||||
})
|
||||
|
||||
it('returns insertion point when not found', () => {
|
||||
const c = makeStringCursor(['active', 'pending'], [1, 2])
|
||||
const r = c.binarySearchValue('inactive')
|
||||
expect(r.found).toBe(false)
|
||||
expect(r.index).toBe(1)
|
||||
})
|
||||
})
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
// Range search
|
||||
// -----------------------------------------------------------------------
|
||||
|
||||
describe('binarySearchRange', () => {
|
||||
it('finds range with inclusive bounds', () => {
|
||||
const c = makeNumericCursor([100, 200, 300, 400, 500], [1, 2, 3, 4, 5])
|
||||
const { startIdx, endIdx } = c.binarySearchRange(200, 400)
|
||||
expect(startIdx).toBe(1) // 200
|
||||
expect(endIdx).toBe(4) // excludes 500
|
||||
})
|
||||
|
||||
it('handles range beyond data', () => {
|
||||
const c = makeNumericCursor([100, 200, 300], [1, 2, 3])
|
||||
const { startIdx, endIdx } = c.binarySearchRange(0, 999)
|
||||
expect(startIdx).toBe(0)
|
||||
expect(endIdx).toBe(3) // all entries
|
||||
})
|
||||
|
||||
it('handles empty range', () => {
|
||||
const c = makeNumericCursor([100, 200, 300], [1, 2, 3])
|
||||
const { startIdx, endIdx } = c.binarySearchRange(150, 180)
|
||||
expect(startIdx).toBe(1) // insertion point for 150
|
||||
expect(endIdx).toBe(1) // nothing between 150 and 180
|
||||
})
|
||||
})
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
// Point and range queries
|
||||
// -----------------------------------------------------------------------
|
||||
|
||||
describe('getEntityIdsForValue', () => {
|
||||
it('returns matching entity IDs', () => {
|
||||
const c = makeNumericCursor([100, 200, 200, 300], [10, 20, 30, 40])
|
||||
expect(c.getEntityIdsForValue(200)).toEqual([20, 30])
|
||||
})
|
||||
|
||||
it('returns empty for no match', () => {
|
||||
const c = makeNumericCursor([100, 200, 300], [10, 20, 30])
|
||||
expect(c.getEntityIdsForValue(999)).toEqual([])
|
||||
})
|
||||
|
||||
it('excludes tombstoned entries', () => {
|
||||
const c = makeNumericCursor([100, 200, 200, 300], [10, 20, 30, 40], [1]) // position 1 tombstoned
|
||||
expect(c.getEntityIdsForValue(200)).toEqual([30]) // position 1 (entityId=20) excluded
|
||||
})
|
||||
})
|
||||
|
||||
describe('getEntityIdsInRange', () => {
|
||||
it('returns entities in range', () => {
|
||||
const c = makeNumericCursor([100, 200, 300, 400, 500], [1, 2, 3, 4, 5])
|
||||
expect(c.getEntityIdsInRange(200, 400)).toEqual([2, 3, 4])
|
||||
})
|
||||
|
||||
it('excludes tombstoned', () => {
|
||||
const c = makeNumericCursor([100, 200, 300, 400, 500], [1, 2, 3, 4, 5], [2]) // position 2 (value=300)
|
||||
expect(c.getEntityIdsInRange(200, 400)).toEqual([2, 4])
|
||||
})
|
||||
})
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
// Iteration
|
||||
// -----------------------------------------------------------------------
|
||||
|
||||
describe('iterateForward', () => {
|
||||
it('yields all entries in order', () => {
|
||||
const c = makeNumericCursor([100, 200, 300], [1, 2, 3])
|
||||
const entries = [...c.iterateForward()]
|
||||
expect(entries).toHaveLength(3)
|
||||
expect(entries[0]).toEqual({ value: 100, entityIntId: 1, position: 0 })
|
||||
expect(entries[2]).toEqual({ value: 300, entityIntId: 3, position: 2 })
|
||||
})
|
||||
|
||||
it('skips tombstoned entries', () => {
|
||||
const c = makeNumericCursor([100, 200, 300], [1, 2, 3], [1])
|
||||
const entries = [...c.iterateForward()]
|
||||
expect(entries).toHaveLength(2)
|
||||
expect(entries.map(e => e.entityIntId)).toEqual([1, 3])
|
||||
})
|
||||
|
||||
it('starts from offset', () => {
|
||||
const c = makeNumericCursor([100, 200, 300, 400], [1, 2, 3, 4])
|
||||
const entries = [...c.iterateForward(2)]
|
||||
expect(entries).toHaveLength(2)
|
||||
expect(entries[0].value).toBe(300)
|
||||
})
|
||||
})
|
||||
|
||||
describe('iterateBackward', () => {
|
||||
it('yields all entries in reverse order', () => {
|
||||
const c = makeNumericCursor([100, 200, 300], [1, 2, 3])
|
||||
const entries = [...c.iterateBackward()]
|
||||
expect(entries).toHaveLength(3)
|
||||
expect(entries[0]).toEqual({ value: 300, entityIntId: 3, position: 2 })
|
||||
expect(entries[2]).toEqual({ value: 100, entityIntId: 1, position: 0 })
|
||||
})
|
||||
|
||||
it('skips tombstoned entries', () => {
|
||||
const c = makeNumericCursor([100, 200, 300], [1, 2, 3], [1])
|
||||
const entries = [...c.iterateBackward()]
|
||||
expect(entries).toHaveLength(2)
|
||||
expect(entries.map(e => e.entityIntId)).toEqual([3, 1])
|
||||
})
|
||||
})
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
// Zone map (min/max)
|
||||
// -----------------------------------------------------------------------
|
||||
|
||||
describe('zone map', () => {
|
||||
it('minValue returns first value', () => {
|
||||
const c = makeNumericCursor([100, 200, 300], [1, 2, 3])
|
||||
expect(c.minValue).toBe(100)
|
||||
})
|
||||
|
||||
it('maxValue returns last value', () => {
|
||||
const c = makeNumericCursor([100, 200, 300], [1, 2, 3])
|
||||
expect(c.maxValue).toBe(300)
|
||||
})
|
||||
|
||||
it('handles empty segment', () => {
|
||||
const c = makeNumericCursor([], [])
|
||||
expect(c.minValue).toBeUndefined()
|
||||
expect(c.maxValue).toBeUndefined()
|
||||
})
|
||||
})
|
||||
|
||||
describe('liveCount', () => {
|
||||
it('subtracts tombstones from count', () => {
|
||||
const c = makeNumericCursor([100, 200, 300], [1, 2, 3], [0, 2])
|
||||
expect(c.liveCount).toBe(1)
|
||||
})
|
||||
})
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
// TailBufferCursor
|
||||
// -----------------------------------------------------------------------
|
||||
|
||||
describe('TailBufferCursor', () => {
|
||||
it('iterates forward', () => {
|
||||
const c = new TailBufferCursor([10, 20, 30], new Uint32Array([1, 2, 3]), ValueType.Number)
|
||||
const entries = [...c.iterateForward()]
|
||||
expect(entries).toHaveLength(3)
|
||||
expect(entries[0].value).toBe(10)
|
||||
})
|
||||
|
||||
it('iterates backward', () => {
|
||||
const c = new TailBufferCursor([10, 20, 30], new Uint32Array([1, 2, 3]), ValueType.Number)
|
||||
const entries = [...c.iterateBackward()]
|
||||
expect(entries[0].value).toBe(30)
|
||||
})
|
||||
|
||||
it('point query returns matching IDs', () => {
|
||||
const c = new TailBufferCursor([10, 20, 20, 30], new Uint32Array([1, 2, 3, 4]), ValueType.Number)
|
||||
expect(c.getEntityIdsForValue(20)).toEqual([2, 3])
|
||||
})
|
||||
|
||||
it('min/max values', () => {
|
||||
const c = new TailBufferCursor([10, 20, 30], new Uint32Array([1, 2, 3]), ValueType.Number)
|
||||
expect(c.minValue).toBe(10)
|
||||
expect(c.maxValue).toBe(30)
|
||||
})
|
||||
})
|
||||
})
|
||||
Loading…
Add table
Add a link
Reference in a new issue