/** * @module columnStore/ColumnTailBuffer * @description Per-field in-memory write buffer that accumulates (value, entityIntId) * entries before flushing to a sorted L0 segment on disk. * * Follows the MemTable pattern from GraphAdjacencyIndex's LSMTree * (src/graph/lsm/LSMTree.ts:64-143): unsorted writes in memory, * sort on flush, clear after flush. * * For multi-value fields (__words__), one entity can add multiple entries * to the same buffer. The sorted drain merges them naturally. */ import { ValueType, DEFAULT_FLUSH_THRESHOLD } from './types.js' /** * Entry in the tail buffer: a (value, entityIntId) pair. * * For numeric fields, value is a number (timestamp, integer count). * For string fields, value is a string. * For boolean fields, value is 0 or 1 (stored as number). */ export interface TailEntry { value: number | string entityIntId: number } /** * Result of draining the tail buffer: parallel sorted arrays * ready for segment writing. */ export interface DrainResult { values: (number | string)[] entityIds: Uint32Array pendingRemovals: Set } /** * In-memory write buffer for a single field's column data. * * Entries accumulate unsorted. On drain/flush, they are sorted by value * (numeric comparison for numbers, lexicographic for strings). Ties are * broken by entityIntId ascending for deterministic output. * * @example * const buf = new ColumnTailBuffer('createdAt', ValueType.Number) * buf.add(1700000001000, 42) * buf.add(1700000000000, 10) * const { values, entityIds } = buf.drain() * // values: [1700000000000, 1700000001000] * // entityIds: [10, 42] */ export class ColumnTailBuffer { /** Field name this buffer is for. */ readonly fieldName: string /** Value type determines sort comparator. */ readonly valueType: ValueType /** Flush threshold. */ readonly threshold: number /** Unsorted entry accumulator. */ private entries: TailEntry[] = [] /** * Entity int IDs marked for removal (tombstoning). * Applied during segment writes — the removal propagates to existing segments. */ private removals: Set = new Set() /** * @param fieldName - Field name (e.g. 'createdAt', 'status', '__words__') * @param valueType - How values should be sorted and encoded * @param threshold - Number of entries before shouldFlush() returns true */ constructor(fieldName: string, valueType: ValueType, threshold: number = DEFAULT_FLUSH_THRESHOLD) { this.fieldName = fieldName this.valueType = valueType this.threshold = threshold } /** * Add a (value, entityIntId) entry to the buffer. * * @param value - The field value (number for numeric/boolean, string for string) * @param entityIntId - The entity's u32 integer ID from EntityIdMapper */ add(value: number | string, entityIntId: number): void { this.entries.push({ value, entityIntId }) } /** * Mark an entity for removal. The removal is tracked as a pending tombstone * and applied when flushing or during segment compaction. * * @param entityIntId - Entity to remove */ remove(entityIntId: number): void { this.removals.add(entityIntId) // Also remove any pending entries for this entity in the buffer this.entries = this.entries.filter(e => e.entityIntId !== entityIntId) } /** * Number of entries currently in the buffer. */ get size(): number { return this.entries.length } /** * Whether the buffer has reached the flush threshold. */ shouldFlush(): boolean { return this.entries.length >= this.threshold } /** * Drain the buffer: sort entries by value, return parallel arrays, * and clear the buffer. Pending removals are returned separately * for the caller to apply as tombstones to existing segments. * * Sorting: numeric values by numeric comparison, strings lexicographically. * Ties broken by entityIntId ascending for deterministic segment output. * * @returns Sorted parallel arrays and pending removals */ drain(): DrainResult { // Sort entries const sorted = this.entries.slice() if (this.valueType === ValueType.String) { sorted.sort((a, b) => { const cmp = String(a.value).localeCompare(String(b.value)) return cmp !== 0 ? cmp : a.entityIntId - b.entityIntId }) } else { sorted.sort((a, b) => { const va = a.value as number const vb = b.value as number return va !== vb ? va - vb : a.entityIntId - b.entityIntId }) } // Build parallel arrays const values: (number | string)[] = new Array(sorted.length) const entityIds = new Uint32Array(sorted.length) for (let i = 0; i < sorted.length; i++) { values[i] = sorted[i].value entityIds[i] = sorted[i].entityIntId } const pendingRemovals = new Set(this.removals) // Clear buffer this.entries = [] this.removals.clear() return { values, entityIds, pendingRemovals } } /** * Clear the buffer without returning data. */ clear(): void { this.entries = [] this.removals.clear() } }