feat: unified column store for filtering + sorting at billion scale
Adds a per-field sorted column store (Lucene doc values + roaring bitmap
architecture) that replaces the MetadataIndex sparse index internals for
both filtering and sorting. One system for all field types with exact
precision — no bucketing, no per-entity storage reads.
Column store: binary .cidx segment format, in-memory tail buffers,
LSM-style compaction, k-way merge sort, multi-value support (__words__).
All queries (filter, range, sort, filtered sort) route through the column
store when data is available, falling back to sparse index otherwise.
Key unlocks:
- find({ orderBy: 'createdAt' }) works WITHOUT a filter (previously threw)
- find({ orderBy: 'metadata.price' }) works for custom numeric fields
- Exact timestamp precision (no 1-minute bucketing)
- O(K log S) sort independent of total entity count
New files: src/indexes/columnStore/ (types, format, tail buffer, cursor,
manifest, coordinator — ~700 lines). 101 new unit tests covering binary
format round-trips, CRC validation, sort, filter, range, deletion,
multi-segment merge, persistence, and multi-value (words) fields.
Deleted: metadataIndex-automatic-bucketing.test.ts (bucketing behavior
eliminated by exact-precision column store). Sparse index write path
removed from addToIndex/removeFromIndex. Sparse index legacy code still
present as dead code pending cleanup in next commit.
This commit is contained in:
parent
634ddfb2bb
commit
46583f2d9b
16 changed files with 3846 additions and 521 deletions
|
|
@ -1,297 +0,0 @@
|
|||
/**
|
||||
* MetadataIndex Automatic Bucketing Tests (v3.41.0)
|
||||
* Verify that temporal fields are automatically bucketed from the start
|
||||
* to prevent file pollution while enabling range queries
|
||||
*/
|
||||
|
||||
import { describe, it, expect, beforeEach, afterEach } from 'vitest'
|
||||
import { MemoryStorage } from '../../../src/storage/adapters/memoryStorage.js'
|
||||
import { MetadataIndexManager } from '../../../src/utils/metadataIndex.js'
|
||||
|
||||
describe('MetadataIndex Automatic Bucketing (v3.41.0)', () => {
|
||||
let storage: MemoryStorage
|
||||
let index: MetadataIndexManager
|
||||
// Use a fixed base timestamp to avoid test pollution from Date.now() bucketing
|
||||
const TEST_BASE_TIME = 1700000000000 // Fixed timestamp (Nov 14, 2023)
|
||||
|
||||
beforeEach(() => {
|
||||
storage = new MemoryStorage()
|
||||
// No excludeFields config - fully automatic!
|
||||
index = new MetadataIndexManager(storage, {})
|
||||
})
|
||||
|
||||
afterEach(async () => {
|
||||
// MemoryStorage doesn't have a close() method
|
||||
})
|
||||
|
||||
it('should automatically bucket timestamps from the first operation', async () => {
|
||||
// Add 100 entities with unique timestamps (every millisecond)
|
||||
const baseTime = TEST_BASE_TIME
|
||||
for (let i = 0; i < 100; i++) {
|
||||
await index.addToIndex(`entity${i}`, {
|
||||
noun: 'character',
|
||||
modified: baseTime + i // 100 unique timestamps
|
||||
})
|
||||
}
|
||||
|
||||
await index.flush()
|
||||
|
||||
// Without bucketing: would create 100 metadata index files
|
||||
// With bucketing: should create only ~2 files (all fit in one 1-minute bucket)
|
||||
|
||||
// Get available values for 'modified' field
|
||||
const modifiedValues = await index.getFilterValues('modified')
|
||||
|
||||
// Should have very few unique values (all bucketed to 1-minute intervals)
|
||||
expect(modifiedValues.length).toBeLessThan(5) // Much less than 100!
|
||||
expect(modifiedValues.length).toBeGreaterThan(0) // But not zero
|
||||
})
|
||||
|
||||
it('should prevent file pollution with high-cardinality timestamps', async () => {
|
||||
// Simulate the bug scenario: 575 relationships with unique timestamps
|
||||
const baseTime = TEST_BASE_TIME + 10000000 // Offset to avoid bucket collision with other tests
|
||||
for (let i = 0; i < 575; i++) {
|
||||
await index.addToIndex(`verb${i}`, {
|
||||
verb: 'relates_to',
|
||||
modified: baseTime + (i * 1000), // Every second
|
||||
accessed: baseTime + (i * 500) // Every half-second
|
||||
})
|
||||
}
|
||||
|
||||
await index.flush()
|
||||
|
||||
// Without bucketing: 575 unique modified + 1150 unique accessed = 1725 index files
|
||||
// With bucketing: ~10 modified buckets + ~20 accessed buckets = ~30 index files
|
||||
|
||||
const modifiedValues = await index.getFilterValues('modified')
|
||||
const accessedValues = await index.getFilterValues('accessed')
|
||||
|
||||
// Should have created FAR fewer index entries due to bucketing
|
||||
expect(modifiedValues.length).toBeLessThan(100) // Not 575!
|
||||
expect(accessedValues.length).toBeLessThan(200) // Not 1150!
|
||||
|
||||
// But should still have indexed them (not excluded)
|
||||
expect(modifiedValues.length).toBeGreaterThan(0)
|
||||
expect(accessedValues.length).toBeGreaterThan(0)
|
||||
})
|
||||
|
||||
it('should enable range queries on bucketed timestamp fields', async () => {
|
||||
// Add entities with timestamps at exact bucket boundaries (every minute)
|
||||
const bucketStart = Math.floor((TEST_BASE_TIME + 20000000) / 60000) * 60000
|
||||
for (let i = 0; i < 100; i++) {
|
||||
await index.addToIndex(`entity${i}`, {
|
||||
noun: 'character',
|
||||
modified: bucketStart + (i * 60000) // Every minute = every bucket
|
||||
})
|
||||
}
|
||||
|
||||
await index.flush()
|
||||
|
||||
// Query: modified >= (bucketStart + 30 minutes)
|
||||
const thirtyMinutesLater = bucketStart + (30 * 60000)
|
||||
const results = await index.getIdsForFilter({
|
||||
modified: { greaterThanOrEqual: thirtyMinutesLater }
|
||||
})
|
||||
|
||||
// Should find entities 30-99 (70 entities)
|
||||
// Note: With bucketing, we expect exactly 70 entities (30-99 inclusive)
|
||||
expect(results.length).toBe(70)
|
||||
|
||||
// Verify results are correct
|
||||
results.forEach(id => {
|
||||
const num = parseInt(id.replace('entity', ''))
|
||||
expect(num).toBeGreaterThanOrEqual(30) // Should be in range
|
||||
})
|
||||
})
|
||||
|
||||
it('should work with zero configuration (no excludeFields, no rangeQueryFields)', async () => {
|
||||
// Create index with NO config at all
|
||||
const autoIndex = new MetadataIndexManager(storage, {})
|
||||
const testTime = TEST_BASE_TIME + 30000000 // Offset to avoid bucket collision
|
||||
|
||||
// Add entity with multiple field types
|
||||
await autoIndex.addToIndex('test1', {
|
||||
id: 'uuid-123', // Should be excluded (in default excludeFields)
|
||||
noun: 'character', // Should be indexed (low cardinality)
|
||||
modified: testTime, // Should be indexed + bucketed (temporal)
|
||||
accessed: testTime, // Should be indexed + bucketed (temporal)
|
||||
content: 'long text content', // Should be excluded (in default excludeFields)
|
||||
customTimestamp: testTime // Should be indexed + bucketed (has 'time' in name)
|
||||
})
|
||||
|
||||
await autoIndex.flush()
|
||||
|
||||
// Verify 'modified' was indexed and bucketed
|
||||
const modifiedValues = await autoIndex.getFilterValues('modified')
|
||||
expect(modifiedValues.length).toBeGreaterThan(0) // Indexed!
|
||||
|
||||
// Verify 'accessed' was indexed and bucketed
|
||||
const accessedValues = await autoIndex.getFilterValues('accessed')
|
||||
expect(accessedValues.length).toBeGreaterThan(0) // Indexed!
|
||||
|
||||
// Verify 'customTimestamp' was indexed and bucketed
|
||||
const customValues = await autoIndex.getFilterValues('customTimestamp')
|
||||
expect(customValues.length).toBeGreaterThan(0) // Indexed!
|
||||
|
||||
// Verify 'id' was excluded
|
||||
const idValues = await autoIndex.getFilterValues('id')
|
||||
expect(idValues.length).toBe(0) // Excluded!
|
||||
|
||||
// Verify 'content' was excluded
|
||||
const contentValues = await autoIndex.getFilterValues('content')
|
||||
expect(contentValues.length).toBe(0) // Excluded!
|
||||
})
|
||||
|
||||
it('should handle fields with "time" or "date" in their names', async () => {
|
||||
// Add entities with various timestamp field names
|
||||
const testTime = TEST_BASE_TIME + 40000000 // Offset to avoid bucket collision
|
||||
await index.addToIndex('entity1', {
|
||||
timestamp: testTime,
|
||||
createdAt: testTime,
|
||||
updatedAt: testTime,
|
||||
birthdate: testTime,
|
||||
eventTime: testTime,
|
||||
lastSeen: testTime // Doesn't match pattern, won't be bucketed
|
||||
})
|
||||
|
||||
await index.flush()
|
||||
|
||||
// Fields with 'time' or 'date' should be bucketed
|
||||
const timestampValues = await index.getFilterValues('timestamp')
|
||||
const createdAtValues = await index.getFilterValues('createdAt')
|
||||
const updatedAtValues = await index.getFilterValues('updatedAt')
|
||||
const birthdateValues = await index.getFilterValues('birthdate')
|
||||
const eventTimeValues = await index.getFilterValues('eventTime')
|
||||
|
||||
// All should be indexed (not excluded)
|
||||
expect(timestampValues.length).toBeGreaterThan(0)
|
||||
expect(createdAtValues.length).toBeGreaterThan(0)
|
||||
expect(updatedAtValues.length).toBeGreaterThan(0)
|
||||
expect(birthdateValues.length).toBeGreaterThan(0)
|
||||
expect(eventTimeValues.length).toBeGreaterThan(0)
|
||||
})
|
||||
|
||||
it('should bucket timestamps to 1-minute intervals by default', async () => {
|
||||
// Use a timestamp at the start of a minute bucket to ensure 30s apart stays in same bucket
|
||||
const bucketStart = Math.floor((TEST_BASE_TIME + 50000000) / 60000) * 60000
|
||||
await index.addToIndex('entity1', { modified: bucketStart })
|
||||
await index.addToIndex('entity2', { modified: bucketStart + 30000 }) // +30 seconds, still in same bucket
|
||||
|
||||
await index.flush()
|
||||
|
||||
// Both should be in the same 1-minute bucket
|
||||
const ids1 = await index.getIds('modified', bucketStart)
|
||||
const ids2 = await index.getIds('modified', bucketStart + 30000)
|
||||
|
||||
// Both queries should return both entities (same bucket)
|
||||
expect(ids1.length).toBe(2)
|
||||
expect(ids2.length).toBe(2)
|
||||
expect(ids1.sort()).toEqual(ids2.sort())
|
||||
})
|
||||
|
||||
it('should create separate buckets for timestamps >1 minute apart', async () => {
|
||||
// Use bucket boundaries to ensure entities are in different buckets
|
||||
const bucket1Start = Math.floor((TEST_BASE_TIME + 60000000) / 60000) * 60000
|
||||
const bucket2Start = bucket1Start + 120000 // +2 minutes (definitely different bucket)
|
||||
await index.addToIndex('entity1', { modified: bucket1Start })
|
||||
await index.addToIndex('entity2', { modified: bucket2Start })
|
||||
|
||||
await index.flush()
|
||||
|
||||
// Should be in different 1-minute buckets
|
||||
const ids1 = await index.getIds('modified', bucket1Start)
|
||||
const ids2 = await index.getIds('modified', bucket2Start)
|
||||
|
||||
// Each query should return only its own entity
|
||||
expect(ids1).toContain('entity1')
|
||||
expect(ids1).not.toContain('entity2')
|
||||
expect(ids2).toContain('entity2')
|
||||
expect(ids2).not.toContain('entity1')
|
||||
})
|
||||
|
||||
it('should handle range queries across bucket boundaries', async () => {
|
||||
// Use explicit bucket boundaries for predictable range queries
|
||||
const bucket0 = Math.floor((TEST_BASE_TIME + 70000000) / 60000) * 60000
|
||||
const bucket1 = bucket0 + 60000 // +1 minute
|
||||
const bucket2 = bucket0 + 120000 // +2 minutes
|
||||
const bucket3 = bucket0 + 180000 // +3 minutes
|
||||
|
||||
await index.addToIndex('entity0', { modified: bucket0 })
|
||||
await index.addToIndex('entity1', { modified: bucket1 })
|
||||
await index.addToIndex('entity2', { modified: bucket2 })
|
||||
await index.addToIndex('entity3', { modified: bucket3 })
|
||||
|
||||
await index.flush()
|
||||
|
||||
// Query: modified > bucket1 (should get entity2 and entity3, not entity0 or entity1)
|
||||
const results = await index.getIdsForFilter({
|
||||
modified: { greaterThan: bucket1 }
|
||||
})
|
||||
|
||||
// Should include entities after bucket1
|
||||
expect(results).toContain('entity2')
|
||||
expect(results).toContain('entity3')
|
||||
|
||||
// Should not include entity0 or entity1 (at or before bucket1)
|
||||
expect(results).not.toContain('entity0')
|
||||
expect(results).not.toContain('entity1')
|
||||
})
|
||||
|
||||
it('should not break non-temporal numeric fields', async () => {
|
||||
// Add entities with regular numeric fields (not timestamps)
|
||||
await index.addToIndex('entity1', {
|
||||
noun: 'item',
|
||||
price: 19.99,
|
||||
quantity: 5,
|
||||
rating: 4.5
|
||||
})
|
||||
|
||||
await index.addToIndex('entity2', {
|
||||
noun: 'item',
|
||||
price: 29.99,
|
||||
quantity: 3,
|
||||
rating: 4.7
|
||||
})
|
||||
|
||||
await index.flush()
|
||||
|
||||
// Non-temporal numeric fields should NOT be bucketed
|
||||
const priceValues = await index.getFilterValues('price')
|
||||
const quantityValues = await index.getFilterValues('quantity')
|
||||
const ratingValues = await index.getFilterValues('rating')
|
||||
|
||||
// Each unique value should create its own index entry
|
||||
expect(priceValues).toContain('19.99')
|
||||
expect(priceValues).toContain('29.99')
|
||||
expect(quantityValues).toContain('5')
|
||||
expect(quantityValues).toContain('3')
|
||||
expect(ratingValues.length).toBeGreaterThan(0)
|
||||
})
|
||||
|
||||
it('should maintain backward compatibility with existing code', async () => {
|
||||
// Existing code that used excludeFields should still work
|
||||
const legacyIndex = new MetadataIndexManager(storage, {
|
||||
excludeFields: ['internalField', 'debugData']
|
||||
})
|
||||
const testTime = TEST_BASE_TIME + 90000000 // Offset to avoid bucket collision
|
||||
|
||||
await legacyIndex.addToIndex('entity1', {
|
||||
noun: 'character',
|
||||
modified: testTime, // Should be indexed + bucketed
|
||||
internalField: 'secret', // Should be excluded
|
||||
debugData: 'verbose logs' // Should be excluded
|
||||
})
|
||||
|
||||
await legacyIndex.flush()
|
||||
|
||||
// Modified should be indexed
|
||||
const modifiedValues = await legacyIndex.getFilterValues('modified')
|
||||
expect(modifiedValues.length).toBeGreaterThan(0)
|
||||
|
||||
// Custom excluded fields should be excluded
|
||||
const internalValues = await legacyIndex.getFilterValues('internalField')
|
||||
const debugValues = await legacyIndex.getFilterValues('debugData')
|
||||
expect(internalValues.length).toBe(0)
|
||||
expect(debugValues.length).toBe(0)
|
||||
})
|
||||
})
|
||||
Loading…
Add table
Add a link
Reference in a new issue