Compare commits
1 commit
main
...
fix/metada
| Author | SHA1 | Date | |
|---|---|---|---|
| a128f0eda5 |
7 changed files with 1025 additions and 133 deletions
|
|
@ -217,6 +217,40 @@ membership queries at scale:
|
||||||
`__words__` for tokenized text…).
|
`__words__` for tokenized text…).
|
||||||
- `_blobs/_column_index/{field}/L0-NNNNNN.bin` — the actual level-0 run
|
- `_blobs/_column_index/{field}/L0-NNNNNN.bin` — the actual level-0 run
|
||||||
segments, stored through the shared `_blobs/<key>.bin` binary convention.
|
segments, stored through the shared `_blobs/<key>.bin` binary convention.
|
||||||
|
- `_column_index/{field}/k/{kind}/…` — the same two files again, for a
|
||||||
|
**second value kind** on the same field (see below). Absent for a field that
|
||||||
|
holds one kind, which is nearly all of them.
|
||||||
|
|
||||||
|
### One posting column per (field, kind)
|
||||||
|
|
||||||
|
A field is not obliged to hold one type of value. `category` may carry
|
||||||
|
`'electronics'` on some rows and `5` on others, and both are real values of
|
||||||
|
that field. A segment, though, has one encoding — i64, f64, UTF-8, or boolean
|
||||||
|
— so a field that holds several kinds gets **one column per kind**:
|
||||||
|
|
||||||
|
- The first kind a field ever sees owns the plain `_column_index/{field}/`
|
||||||
|
layout above. A single-kind field is therefore byte-identical to what earlier
|
||||||
|
versions wrote, and an index written before typed postings opens unchanged.
|
||||||
|
- Every later kind gets its own column beside it at
|
||||||
|
`_column_index/{field}/k/{kind}/`, where `{kind}` is `number`, `string` or
|
||||||
|
`boolean`.
|
||||||
|
|
||||||
|
What that buys at query time:
|
||||||
|
|
||||||
|
| | |
|
||||||
|
|---|---|
|
||||||
|
| **Equality** | Answered from the column matching the **query value's own kind**. `where {category: 5}` reads the number postings; `where {category: '5'}` reads the string postings. Neither borrows the other's rows — a row written with the number `5` is not a row whose category is the text `'5'`. |
|
||||||
|
| **A kind the field never held** | Matches nothing. That is the true answer, not a coerced one. |
|
||||||
|
| **Ranges** | Routed by the kind of the bounds: numeric bounds read the numeric postings and ignore the field's strings. An **unbounded** range is the "has any value here" probe behind `exists`, and reads every kind. |
|
||||||
|
| **`orderBy`** | A number and a string have no order between them, so a mixed field orders by kind first (number, string, boolean) and by value within a kind. A single-kind field sorts exactly as it always did. |
|
||||||
|
| **Numbers** | One kind, one column: an integer column is written as i64 and widens to f64 the first time a non-integer arrives, so `4.5` is stored as itself rather than rounded. |
|
||||||
|
|
||||||
|
`null` and `undefined` are not kinds and are never posted; their absence is
|
||||||
|
what the `exists` / `missing` operators read.
|
||||||
|
|
||||||
|
Older readers are unaffected by the additional columns: they see the field's
|
||||||
|
primary column exactly where it has always been, and a `k/{kind}` directory is
|
||||||
|
simply a name they never query.
|
||||||
|
|
||||||
Sparse per-field indexes, roaring-bitmap chunks, and zone-map/bloom segments
|
Sparse per-field indexes, roaring-bitmap chunks, and zone-map/bloom segments
|
||||||
additionally live as bucketed keys under `_system/idx/` (see §3). Which path
|
additionally live as bucketed keys under `_system/idx/` (see §3). Which path
|
||||||
|
|
|
||||||
|
|
@ -23,7 +23,10 @@ import type { ColumnStoreProvider, SegmentMeta } from './types.js'
|
||||||
import {
|
import {
|
||||||
ValueType,
|
ValueType,
|
||||||
DEFAULT_FLUSH_THRESHOLD,
|
DEFAULT_FLUSH_THRESHOLD,
|
||||||
FLAG_MULTI_VALUE
|
FLAG_MULTI_VALUE,
|
||||||
|
POSTING_KINDS,
|
||||||
|
KIND_PATH_SEGMENT,
|
||||||
|
type PostingKind
|
||||||
} from './types.js'
|
} from './types.js'
|
||||||
import { ColumnTailBuffer } from './ColumnTailBuffer.js'
|
import { ColumnTailBuffer } from './ColumnTailBuffer.js'
|
||||||
import { ColumnManifest } from './ColumnManifest.js'
|
import { ColumnManifest } from './ColumnManifest.js'
|
||||||
|
|
@ -52,10 +55,89 @@ interface HeapEntry {
|
||||||
value: number | string
|
value: number | string
|
||||||
entityIntId: number
|
entityIntId: number
|
||||||
cursorIndex: number
|
cursorIndex: number
|
||||||
|
/**
|
||||||
|
* Rank of the posting kind this entry came from, from {@link POSTING_KINDS}.
|
||||||
|
* A mixed-kind field has no natural total order, so the merge orders by kind
|
||||||
|
* first and by value within a kind.
|
||||||
|
*/
|
||||||
|
kindRank: number
|
||||||
/** Iterator for the cursor — call next() to advance */
|
/** Iterator for the cursor — call next() to advance */
|
||||||
iterator: Generator<CursorEntry>
|
iterator: Generator<CursorEntry>
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* One physical posting column: a (field, kind) pair and the key every internal
|
||||||
|
* map and every storage path uses for it.
|
||||||
|
*/
|
||||||
|
interface KindColumn {
|
||||||
|
/** The field as the query language names it. */
|
||||||
|
field: string
|
||||||
|
/** The kind of value this column holds. */
|
||||||
|
kind: PostingKind
|
||||||
|
/**
|
||||||
|
* Internal map / storage key. The field's PRIMARY kind uses the bare field
|
||||||
|
* name — the historical layout — and every other kind uses
|
||||||
|
* `<field>/<KIND_PATH_SEGMENT>/<kind>`.
|
||||||
|
*/
|
||||||
|
key: string
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* The KIND a value indexes under — its JavaScript `typeof` class, not its
|
||||||
|
* storage encoding.
|
||||||
|
*
|
||||||
|
* Anything that is not a number, string or boolean indexes as a string, which
|
||||||
|
* is the `String(value)` treatment those values already received. `null` and
|
||||||
|
* `undefined` never reach here: `addEntity` skips them, and their absence is
|
||||||
|
* what the `exists` / `missing` operators read.
|
||||||
|
*
|
||||||
|
* @param value - The value about to be indexed or queried
|
||||||
|
* @returns The posting kind that owns this value
|
||||||
|
*/
|
||||||
|
function kindOfValue(value: unknown): PostingKind {
|
||||||
|
const t = typeof value
|
||||||
|
if (t === 'number') return 'number'
|
||||||
|
if (t === 'boolean') return 'boolean'
|
||||||
|
return 'string'
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* The segment encoding a fresh column of this kind starts with.
|
||||||
|
*
|
||||||
|
* Only the number kind has a choice: an integer column starts as i64 and
|
||||||
|
* widens to f64 the first time a non-integer arrives
|
||||||
|
* ({@link ColumnTailBuffer.promoteToFloat}).
|
||||||
|
*/
|
||||||
|
function initialValueTypeFor(kind: PostingKind, firstValue: unknown): ValueType {
|
||||||
|
switch (kind) {
|
||||||
|
case 'boolean':
|
||||||
|
return ValueType.Boolean
|
||||||
|
case 'string':
|
||||||
|
return ValueType.String
|
||||||
|
case 'number':
|
||||||
|
return Number.isInteger(firstValue) ? ValueType.Number : ValueType.Float
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* The kind a column of this encoding holds — the inverse of
|
||||||
|
* {@link initialValueTypeFor}, used to read a kind back off a manifest written
|
||||||
|
* before typed postings existed.
|
||||||
|
*/
|
||||||
|
function kindOfValueType(valueType: ValueType): PostingKind {
|
||||||
|
switch (valueType) {
|
||||||
|
case ValueType.Boolean:
|
||||||
|
return 'boolean'
|
||||||
|
case ValueType.String:
|
||||||
|
return 'string'
|
||||||
|
case ValueType.Number:
|
||||||
|
case ValueType.Float:
|
||||||
|
return 'number'
|
||||||
|
default:
|
||||||
|
throw new Error(`Unknown ValueType: ${valueType}`)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Unified column store coordinator.
|
* Unified column store coordinator.
|
||||||
*
|
*
|
||||||
|
|
@ -121,9 +203,19 @@ export class ColumnStore implements ColumnStoreProvider {
|
||||||
*/
|
*/
|
||||||
private deletedEntities: Map<string, RoaringBitmap32> = new Map()
|
private deletedEntities: Map<string, RoaringBitmap32> = new Map()
|
||||||
|
|
||||||
/** Known field value types (inferred from first write). */
|
/** Segment encoding per COLUMN key (not per field — a field has one per kind). */
|
||||||
private fieldTypes: Map<string, ValueType> = new Map()
|
private fieldTypes: Map<string, ValueType> = new Map()
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Every posting column a field owns: field → kind → column key.
|
||||||
|
*
|
||||||
|
* This is the map that ends the first-writer type freeze. A field's first
|
||||||
|
* kind takes the bare field name as its column key, keeping the historical
|
||||||
|
* on-disk layout; each later kind takes its own column beside it. Nothing is
|
||||||
|
* coerced across kinds and nothing is dropped for being the wrong type.
|
||||||
|
*/
|
||||||
|
private fieldColumns: Map<string, Map<PostingKind, string>> = new Map()
|
||||||
|
|
||||||
/** Whether init() has completed. */
|
/** Whether init() has completed. */
|
||||||
private initialized = false
|
private initialized = false
|
||||||
|
|
||||||
|
|
@ -140,6 +232,128 @@ export class ColumnStore implements ColumnStoreProvider {
|
||||||
this.l0CompactionTrigger = config?.l0CompactionTrigger ?? 4
|
this.l0CompactionTrigger = config?.l0CompactionTrigger ?? 4
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// =========================================================================
|
||||||
|
// Posting columns: (field, kind) → one physical column
|
||||||
|
// =========================================================================
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Storage / map key for a (field, kind) column.
|
||||||
|
*
|
||||||
|
* `primary` is the kind that owns the bare field name. It is whichever kind
|
||||||
|
* the field saw first, which for an index written before typed postings is
|
||||||
|
* simply the kind of its single manifest — so the historical layout is
|
||||||
|
* preserved rather than migrated.
|
||||||
|
*/
|
||||||
|
private static columnKeyFor(field: string, kind: PostingKind, primary: PostingKind | null): string {
|
||||||
|
return primary === null || kind === primary
|
||||||
|
? field
|
||||||
|
: `${field}/${KIND_PATH_SEGMENT}/${kind}`
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Split a discovered manifest path back into its (field, kind) column, or
|
||||||
|
* `null` when the path names a field's primary column rather than a kind
|
||||||
|
* column. `<field>/k/<kind>` is the only shape that reads as a kind column,
|
||||||
|
* and only for a `<kind>` this version knows.
|
||||||
|
*/
|
||||||
|
private static parseKindColumnKey(key: string): { field: string; kind: PostingKind } | null {
|
||||||
|
const marker = `/${KIND_PATH_SEGMENT}/`
|
||||||
|
const at = key.lastIndexOf(marker)
|
||||||
|
if (at <= 0) return null
|
||||||
|
const kind = key.slice(at + marker.length)
|
||||||
|
if (!POSTING_KINDS.includes(kind as PostingKind)) return null
|
||||||
|
return { field: key.slice(0, at), kind: kind as PostingKind }
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Record a discovered or freshly created column against its field. */
|
||||||
|
private registerColumn(field: string, kind: PostingKind, key: string): void {
|
||||||
|
let byKind = this.fieldColumns.get(field)
|
||||||
|
if (!byKind) {
|
||||||
|
byKind = new Map()
|
||||||
|
this.fieldColumns.set(field, byKind)
|
||||||
|
}
|
||||||
|
const existing = byKind.get(kind)
|
||||||
|
if (existing !== undefined && existing !== key) {
|
||||||
|
// Two columns claiming one (field, kind) means the layout on disk is not
|
||||||
|
// one this writer could have produced. Serving it would silently answer
|
||||||
|
// from half the postings, so say which two and stop.
|
||||||
|
throw new Error(
|
||||||
|
`ColumnStore: field '${field}' has two '${kind}' posting columns on ` +
|
||||||
|
`disk ('${existing}' and '${key}'). The column index layout is ` +
|
||||||
|
`inconsistent — rebuild/repair the metadata index rather than ` +
|
||||||
|
`serving from one half of it.`
|
||||||
|
)
|
||||||
|
}
|
||||||
|
byKind.set(kind, key)
|
||||||
|
}
|
||||||
|
|
||||||
|
/** The column key for this (field, kind), or `null` if the field has no such kind. */
|
||||||
|
private columnKey(field: string, kind: PostingKind): string | null {
|
||||||
|
return this.fieldColumns.get(field)?.get(kind) ?? null
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* The column key for this (field, kind), creating the registration if the
|
||||||
|
* field has not seen this kind before. Write path only.
|
||||||
|
*/
|
||||||
|
private ensureColumnKey(field: string, kind: PostingKind): string {
|
||||||
|
const byKind = this.fieldColumns.get(field)
|
||||||
|
const existing = byKind?.get(kind)
|
||||||
|
if (existing !== undefined) return existing
|
||||||
|
|
||||||
|
// The primary kind is the one already holding the bare field name, if any.
|
||||||
|
let primary: PostingKind | null = null
|
||||||
|
if (byKind) {
|
||||||
|
for (const [k, key] of byKind) {
|
||||||
|
if (key === field) { primary = k; break }
|
||||||
|
}
|
||||||
|
}
|
||||||
|
const key = ColumnStore.columnKeyFor(field, kind, primary)
|
||||||
|
this.registerColumn(field, kind, key)
|
||||||
|
return key
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Every posting column this field owns, in {@link POSTING_KINDS} order.
|
||||||
|
*
|
||||||
|
* Read doors that are not about one particular value — an unbounded range
|
||||||
|
* used as an "any value present" probe, distinct values, sorting — fan out
|
||||||
|
* over all of them.
|
||||||
|
*/
|
||||||
|
private columnsForField(field: string): KindColumn[] {
|
||||||
|
const byKind = this.fieldColumns.get(field)
|
||||||
|
if (!byKind) return []
|
||||||
|
const out: KindColumn[] = []
|
||||||
|
for (const kind of POSTING_KINDS) {
|
||||||
|
const key = byKind.get(kind)
|
||||||
|
if (key !== undefined) out.push({ field, kind, key })
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Which value kinds this field actually holds, in {@link POSTING_KINDS}
|
||||||
|
* order — the honest answer to "what type is this field?".
|
||||||
|
*
|
||||||
|
* A field that carries both `'electronics'` and `5` reports
|
||||||
|
* `['number', 'string']`, not whichever of them was written first.
|
||||||
|
*
|
||||||
|
* @param field - Field name
|
||||||
|
* @returns Every kind with at least one posting, or `[]` for an unknown field
|
||||||
|
*/
|
||||||
|
getFieldKinds(field: string): PostingKind[] {
|
||||||
|
return this.columnsForField(field)
|
||||||
|
.filter((c) => this.columnHasData(c.key))
|
||||||
|
.map((c) => c.kind)
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Does this physical column hold any postings (persisted or buffered)? */
|
||||||
|
private columnHasData(key: string): boolean {
|
||||||
|
const manifest = this.manifests.get(key)
|
||||||
|
const buffer = this.tailBuffers.get(key)
|
||||||
|
return (manifest !== undefined && !manifest.isEmpty()) || (buffer !== undefined && buffer.size > 0)
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Initialize the column store: discover existing field manifests.
|
* Initialize the column store: discover existing field manifests.
|
||||||
*/
|
*/
|
||||||
|
|
@ -157,11 +371,23 @@ export class ColumnStore implements ColumnStoreProvider {
|
||||||
}).listObjectsUnderPath(this.basePath + '/')
|
}).listObjectsUnderPath(this.basePath + '/')
|
||||||
for (const path of paths) {
|
for (const path of paths) {
|
||||||
if (path.endsWith('/MANIFEST.json')) {
|
if (path.endsWith('/MANIFEST.json')) {
|
||||||
const fieldName = path.replace(this.basePath + '/', '').replace('/MANIFEST.json', '')
|
// The discovered name is a COLUMN key: either a bare field (that
|
||||||
const manifest = new ColumnManifest(fieldName, this.basePath)
|
// field's primary kind, which is every column an index written
|
||||||
|
// before typed postings has) or `<field>/k/<kind>` for a second
|
||||||
|
// kind that arrived on a field later.
|
||||||
|
const columnKey = path.replace(this.basePath + '/', '').replace('/MANIFEST.json', '')
|
||||||
|
const manifest = new ColumnManifest(columnKey, this.basePath)
|
||||||
await manifest.load(storage)
|
await manifest.load(storage)
|
||||||
this.manifests.set(fieldName, manifest)
|
this.manifests.set(columnKey, manifest)
|
||||||
this.fieldTypes.set(fieldName, manifest.valueType)
|
this.fieldTypes.set(columnKey, manifest.valueType)
|
||||||
|
|
||||||
|
const parsed = ColumnStore.parseKindColumnKey(columnKey)
|
||||||
|
if (parsed) {
|
||||||
|
this.registerColumn(parsed.field, parsed.kind, columnKey)
|
||||||
|
} else {
|
||||||
|
this.registerColumn(columnKey, kindOfValueType(manifest.valueType), columnKey)
|
||||||
|
}
|
||||||
|
const fieldName = columnKey
|
||||||
|
|
||||||
// Load global deleted bitmap if it exists. Raw blob preferred
|
// Load global deleted bitmap if it exists. Raw blob preferred
|
||||||
// (2.4.0 #4 cortex-shared format); legacy envelope fallback for
|
// (2.4.0 #4 cortex-shared format); legacy envelope fallback for
|
||||||
|
|
@ -264,26 +490,43 @@ export class ColumnStore implements ColumnStoreProvider {
|
||||||
/**
|
/**
|
||||||
* Point filter: find entities where field equals value.
|
* Point filter: find entities where field equals value.
|
||||||
*
|
*
|
||||||
* Searches all segments + tail buffer, returns union as roaring bitmap.
|
* The QUERY VALUE'S OWN KIND picks the posting column, and only that column
|
||||||
* Excludes globally deleted entities.
|
* is read. `where {category: 5}` answers from the number postings and
|
||||||
|
* `where {category: '5'}` from the string postings — neither borrows the
|
||||||
|
* other's rows, because a row written with the number `5` is not a row whose
|
||||||
|
* category is the text `'5'`.
|
||||||
|
*
|
||||||
|
* A field that has never seen this kind matches nothing, which is the true
|
||||||
|
* answer rather than a coerced one.
|
||||||
|
*
|
||||||
|
* Searches all segments + tail buffer of that column, returns the union as a
|
||||||
|
* roaring bitmap. Excludes globally deleted entities.
|
||||||
*/
|
*/
|
||||||
async filter(field: string, value: unknown): Promise<RoaringBitmap32> {
|
async filter(field: string, value: unknown): Promise<RoaringBitmap32> {
|
||||||
const result = new RoaringBitmap32()
|
const result = new RoaringBitmap32()
|
||||||
const deleted = this.deletedEntities.get(field)
|
const columnKey = this.columnKey(field, kindOfValue(value))
|
||||||
|
if (columnKey === null) return result
|
||||||
|
|
||||||
|
// The query value takes the column's encoding — a boolean queried against
|
||||||
|
// a boolean column has to become the 1/0 the column stores.
|
||||||
|
const encoded = this.normalizeValue(value, this.fieldTypes.get(columnKey) ?? ValueType.String)
|
||||||
|
if (encoded === undefined) return result
|
||||||
|
|
||||||
|
const deleted = this.deletedEntities.get(columnKey)
|
||||||
|
|
||||||
// Search segments
|
// Search segments
|
||||||
const cursors = await this.getSegmentCursors(field)
|
const cursors = await this.getSegmentCursors(columnKey)
|
||||||
for (const cursor of cursors) {
|
for (const cursor of cursors) {
|
||||||
const ids = cursor.getEntityIdsForValue(value as number | string)
|
const ids = cursor.getEntityIdsForValue(encoded)
|
||||||
for (const id of ids) {
|
for (const id of ids) {
|
||||||
if (!deleted || !deleted.has(id)) result.add(id)
|
if (!deleted || !deleted.has(id)) result.add(id)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Search tail buffer
|
// Search tail buffer
|
||||||
const tailCursor = this.getTailBufferCursor(field)
|
const tailCursor = this.getTailBufferCursor(columnKey)
|
||||||
if (tailCursor) {
|
if (tailCursor) {
|
||||||
const ids = tailCursor.getEntityIdsForValue(value as number | string)
|
const ids = tailCursor.getEntityIdsForValue(encoded)
|
||||||
for (const id of ids) {
|
for (const id of ids) {
|
||||||
if (!deleted || !deleted.has(id)) result.add(id)
|
if (!deleted || !deleted.has(id)) result.add(id)
|
||||||
}
|
}
|
||||||
|
|
@ -324,22 +567,26 @@ export class ColumnStore implements ColumnStoreProvider {
|
||||||
const out = new Map<number, number | string>()
|
const out = new Map<number, number | string>()
|
||||||
if (wanted.size === 0 || !this.hasField(field)) return out
|
if (wanted.size === 0 || !this.hasField(field)) return out
|
||||||
|
|
||||||
const deleted = this.deletedEntities.get(field)
|
// Every kind the field holds is read, in POSTING_KINDS order — a value an
|
||||||
const take = (entry: { value: number | string; entityIntId: number }): void => {
|
// entity wrote as a string is still that entity's value for this field.
|
||||||
if (!wanted.has(entry.entityIntId)) return
|
for (const column of this.columnsForField(field)) {
|
||||||
if (deleted && deleted.has(entry.entityIntId)) return
|
const deleted = this.deletedEntities.get(column.key)
|
||||||
out.set(entry.entityIntId, entry.value)
|
const take = (entry: { value: number | string; entityIntId: number }): void => {
|
||||||
}
|
if (!wanted.has(entry.entityIntId)) return
|
||||||
|
if (deleted && deleted.has(entry.entityIntId)) return
|
||||||
|
out.set(entry.entityIntId, entry.value)
|
||||||
|
}
|
||||||
|
|
||||||
// Segments oldest -> newest, then the tail: a later write overwrites an
|
// Segments oldest -> newest, then the tail: a later write overwrites an
|
||||||
// earlier one for the same id.
|
// earlier one for the same id.
|
||||||
const cursors = await this.getSegmentCursors(field)
|
const cursors = await this.getSegmentCursors(column.key)
|
||||||
for (const cursor of cursors) {
|
for (const cursor of cursors) {
|
||||||
for (const entry of cursor.iterateForward()) take(entry)
|
for (const entry of cursor.iterateForward()) take(entry)
|
||||||
}
|
}
|
||||||
const tailCursor = this.getTailBufferCursor(field)
|
const tailCursor = this.getTailBufferCursor(column.key)
|
||||||
if (tailCursor) {
|
if (tailCursor) {
|
||||||
for (const entry of tailCursor.iterateForward()) take(entry)
|
for (const entry of tailCursor.iterateForward()) take(entry)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
return out
|
return out
|
||||||
}
|
}
|
||||||
|
|
@ -363,41 +610,59 @@ export class ColumnStore implements ColumnStoreProvider {
|
||||||
includeMax: boolean = true
|
includeMax: boolean = true
|
||||||
): Promise<RoaringBitmap32> {
|
): Promise<RoaringBitmap32> {
|
||||||
const result = new RoaringBitmap32()
|
const result = new RoaringBitmap32()
|
||||||
const cursors = await this.getSegmentCursors(field)
|
|
||||||
|
|
||||||
const hasMin = min !== undefined && min !== null
|
const hasMin = min !== undefined && min !== null
|
||||||
const hasMax = max !== undefined && max !== null
|
const hasMax = max !== undefined && max !== null
|
||||||
|
|
||||||
for (const cursor of cursors) {
|
// The BOUNDS pick the column: numeric bounds read the numeric postings,
|
||||||
const lo = hasMin ? min as number | string : cursor.minValue
|
// string bounds the string postings. An unbounded call is not a range at
|
||||||
const hi = hasMax ? max as number | string : cursor.maxValue
|
// all — it is the "has any value here" probe behind `exists` — so it fans
|
||||||
if (lo === undefined || hi === undefined) continue
|
// out over every kind the field holds.
|
||||||
// Exclusivity applies only to an explicitly provided bound. A bound taken
|
const columns: KindColumn[] = hasMin
|
||||||
// from the segment's own min/max is a real stored value and must stay
|
? this.columnsForKind(field, kindOfValue(min))
|
||||||
// inclusive, or the segment's boundary entities would be wrongly dropped.
|
: hasMax
|
||||||
const ids = cursor.getEntityIdsInRange(
|
? this.columnsForKind(field, kindOfValue(max))
|
||||||
lo,
|
: this.columnsForField(field)
|
||||||
hi,
|
|
||||||
hasMin ? includeMin : true,
|
|
||||||
hasMax ? includeMax : true
|
|
||||||
)
|
|
||||||
for (const id of ids) result.add(id)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Tail buffer range: linear scan (tail is small)
|
for (const column of columns) {
|
||||||
const tailCursor = this.getTailBufferCursor(field)
|
const cursors = await this.getSegmentCursors(column.key)
|
||||||
if (tailCursor) {
|
for (const cursor of cursors) {
|
||||||
for (const entry of tailCursor.iterateForward()) {
|
const lo = hasMin ? min as number | string : cursor.minValue
|
||||||
const v = entry.value as any
|
const hi = hasMax ? max as number | string : cursor.maxValue
|
||||||
const loOk = !hasMin || (includeMin ? v >= (min as any) : v > (min as any))
|
if (lo === undefined || hi === undefined) continue
|
||||||
const hiOk = !hasMax || (includeMax ? v <= (max as any) : v < (max as any))
|
// Exclusivity applies only to an explicitly provided bound. A bound taken
|
||||||
if (loOk && hiOk) result.add(entry.entityIntId)
|
// from the segment's own min/max is a real stored value and must stay
|
||||||
|
// inclusive, or the segment's boundary entities would be wrongly dropped.
|
||||||
|
const ids = cursor.getEntityIdsInRange(
|
||||||
|
lo,
|
||||||
|
hi,
|
||||||
|
hasMin ? includeMin : true,
|
||||||
|
hasMax ? includeMax : true
|
||||||
|
)
|
||||||
|
for (const id of ids) result.add(id)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Tail buffer range: linear scan (tail is small)
|
||||||
|
const tailCursor = this.getTailBufferCursor(column.key)
|
||||||
|
if (tailCursor) {
|
||||||
|
for (const entry of tailCursor.iterateForward()) {
|
||||||
|
const v = entry.value as any
|
||||||
|
const loOk = !hasMin || (includeMin ? v >= (min as any) : v > (min as any))
|
||||||
|
const hiOk = !hasMax || (includeMax ? v <= (max as any) : v < (max as any))
|
||||||
|
if (loOk && hiOk) result.add(entry.entityIntId)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
return result
|
return result
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/** The single column for this (field, kind), as a list, or empty if absent. */
|
||||||
|
private columnsForKind(field: string, kind: PostingKind): KindColumn[] {
|
||||||
|
const key = this.columnKey(field, kind)
|
||||||
|
return key === null ? [] : [{ field, kind, key }]
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Sort top-K: return K entity int IDs in sorted order (u64-safe BigInt).
|
* Sort top-K: return K entity int IDs in sorted order (u64-safe BigInt).
|
||||||
*
|
*
|
||||||
|
|
@ -428,18 +693,21 @@ export class ColumnStore implements ColumnStoreProvider {
|
||||||
*/
|
*/
|
||||||
async getFilterValues(field: string): Promise<string[]> {
|
async getFilterValues(field: string): Promise<string[]> {
|
||||||
const valueSet = new Set<string>()
|
const valueSet = new Set<string>()
|
||||||
const cursors = await this.getSegmentCursors(field)
|
|
||||||
|
|
||||||
for (const cursor of cursors) {
|
for (const column of this.columnsForField(field)) {
|
||||||
for (const entry of cursor.iterateForward()) {
|
const cursors = await this.getSegmentCursors(column.key)
|
||||||
valueSet.add(String(entry.value))
|
|
||||||
|
for (const cursor of cursors) {
|
||||||
|
for (const entry of cursor.iterateForward()) {
|
||||||
|
valueSet.add(String(entry.value))
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
const tailCursor = this.getTailBufferCursor(field)
|
const tailCursor = this.getTailBufferCursor(column.key)
|
||||||
if (tailCursor) {
|
if (tailCursor) {
|
||||||
for (const entry of tailCursor.iterateForward()) {
|
for (const entry of tailCursor.iterateForward()) {
|
||||||
valueSet.add(String(entry.value))
|
valueSet.add(String(entry.value))
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -450,9 +718,7 @@ export class ColumnStore implements ColumnStoreProvider {
|
||||||
* Check if a field has any indexed data.
|
* Check if a field has any indexed data.
|
||||||
*/
|
*/
|
||||||
hasField(field: string): boolean {
|
hasField(field: string): boolean {
|
||||||
const manifest = this.manifests.get(field)
|
return this.columnsForField(field).some((c) => this.columnHasData(c.key))
|
||||||
const buffer = this.tailBuffers.get(field)
|
|
||||||
return (manifest !== undefined && !manifest.isEmpty()) || (buffer !== undefined && buffer.size > 0)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
|
@ -462,12 +728,11 @@ export class ColumnStore implements ColumnStoreProvider {
|
||||||
* store will actually serve queries from.
|
* store will actually serve queries from.
|
||||||
*/
|
*/
|
||||||
getIndexedFields(): string[] {
|
getIndexedFields(): string[] {
|
||||||
|
// Names FIELDS, not columns: a field carrying two kinds is one name here,
|
||||||
|
// the same name a caller queries with.
|
||||||
const fields = new Set<string>()
|
const fields = new Set<string>()
|
||||||
for (const [field, manifest] of this.manifests) {
|
for (const [field] of this.fieldColumns) {
|
||||||
if (!manifest.isEmpty()) fields.add(field)
|
if (this.hasField(field)) fields.add(field)
|
||||||
}
|
|
||||||
for (const [field, buffer] of this.tailBuffers) {
|
|
||||||
if (buffer.size > 0) fields.add(field)
|
|
||||||
}
|
}
|
||||||
return Array.from(fields).sort()
|
return Array.from(fields).sort()
|
||||||
}
|
}
|
||||||
|
|
@ -482,12 +747,16 @@ export class ColumnStore implements ColumnStoreProvider {
|
||||||
getFieldSizeSummary(): Array<{ field: string; segmentCount: number; tailSize: number }> {
|
getFieldSizeSummary(): Array<{ field: string; segmentCount: number; tailSize: number }> {
|
||||||
const summary: Array<{ field: string; segmentCount: number; tailSize: number }> = []
|
const summary: Array<{ field: string; segmentCount: number; tailSize: number }> = []
|
||||||
for (const field of this.getIndexedFields()) {
|
for (const field of this.getIndexedFields()) {
|
||||||
const manifest = this.manifests.get(field)
|
// Summed across the field's kind columns — the caller asked about a
|
||||||
const buffer = this.tailBuffers.get(field)
|
// field, and a field's size is all of the postings under its name.
|
||||||
const segmentCount = manifest && !manifest.isEmpty()
|
let segmentCount = 0
|
||||||
? manifest.getAllSegments().length
|
let tailSize = 0
|
||||||
: 0
|
for (const column of this.columnsForField(field)) {
|
||||||
const tailSize = buffer ? buffer.size : 0
|
const manifest = this.manifests.get(column.key)
|
||||||
|
const buffer = this.tailBuffers.get(column.key)
|
||||||
|
if (manifest && !manifest.isEmpty()) segmentCount += manifest.getAllSegments().length
|
||||||
|
if (buffer) tailSize += buffer.size
|
||||||
|
}
|
||||||
summary.push({ field, segmentCount, tailSize })
|
summary.push({ field, segmentCount, tailSize })
|
||||||
}
|
}
|
||||||
return summary
|
return summary
|
||||||
|
|
@ -515,6 +784,8 @@ export class ColumnStore implements ColumnStoreProvider {
|
||||||
this.segmentCache.clear()
|
this.segmentCache.clear()
|
||||||
this.manifests.clear()
|
this.manifests.clear()
|
||||||
this.deletedEntities.clear()
|
this.deletedEntities.clear()
|
||||||
|
this.fieldColumns.clear()
|
||||||
|
this.fieldTypes.clear()
|
||||||
this.initialized = false
|
this.initialized = false
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -523,32 +794,64 @@ export class ColumnStore implements ColumnStoreProvider {
|
||||||
// =========================================================================
|
// =========================================================================
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Push a single value to a field's tail buffer.
|
* Push a single value to the posting column for its (field, KIND).
|
||||||
* Creates the buffer and manifest if first write to this field.
|
*
|
||||||
* Infers ValueType from the first value seen.
|
* The value's own kind picks the column — a string goes to the field's
|
||||||
|
* string postings, a number to its number postings — so a field carrying
|
||||||
|
* `'electronics'` and `5` keeps both, each answerable by an equality filter
|
||||||
|
* of its own kind. Under the first-writer type freeze this method replaced,
|
||||||
|
* the first value's type became the field's type and every later value of
|
||||||
|
* another kind was coerced to it or, when coercion failed, dropped with no
|
||||||
|
* error at all.
|
||||||
|
*
|
||||||
|
* Creates the column's buffer and manifest on its first value.
|
||||||
*/
|
*/
|
||||||
private pushToBuffer(field: string, value: unknown, entityIntId: number, isMultiValue: boolean): void {
|
private pushToBuffer(field: string, value: unknown, entityIntId: number, isMultiValue: boolean): void {
|
||||||
let buffer = this.tailBuffers.get(field)
|
const kind = kindOfValue(value)
|
||||||
|
const columnKey = this.ensureColumnKey(field, kind)
|
||||||
|
|
||||||
|
let buffer = this.tailBuffers.get(columnKey)
|
||||||
if (!buffer) {
|
if (!buffer) {
|
||||||
const valueType = this.inferValueType(value)
|
// A reopened column takes its encoding from its manifest — an integer
|
||||||
buffer = new ColumnTailBuffer(field, valueType, this.flushThreshold)
|
// column that widened to f64 in an earlier session stays widened.
|
||||||
this.tailBuffers.set(field, buffer)
|
const valueType =
|
||||||
this.fieldTypes.set(field, valueType)
|
this.manifests.get(columnKey)?.valueType ?? initialValueTypeFor(kind, value)
|
||||||
|
buffer = new ColumnTailBuffer(columnKey, valueType, this.flushThreshold)
|
||||||
|
this.tailBuffers.set(columnKey, buffer)
|
||||||
|
this.fieldTypes.set(columnKey, valueType)
|
||||||
|
|
||||||
// Ensure manifest exists
|
// Ensure manifest exists
|
||||||
if (!this.manifests.has(field)) {
|
if (!this.manifests.has(columnKey)) {
|
||||||
const manifest = new ColumnManifest(field, this.basePath)
|
const manifest = new ColumnManifest(columnKey, this.basePath)
|
||||||
manifest.valueType = valueType
|
manifest.valueType = valueType
|
||||||
manifest.multiValue = isMultiValue
|
manifest.multiValue = isMultiValue
|
||||||
this.manifests.set(field, manifest)
|
this.manifests.set(columnKey, manifest)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Normalize value to the column type
|
// An integer column widens the first time a non-integer number arrives, so
|
||||||
const normalizedValue = this.normalizeValue(value, buffer.valueType)
|
// the value is stored as itself instead of rounded to the nearest integer.
|
||||||
if (normalizedValue !== undefined) {
|
if (kind === 'number' && buffer.valueType === ValueType.Number && !Number.isInteger(value)) {
|
||||||
buffer.add(normalizedValue, entityIntId)
|
buffer.promoteToFloat()
|
||||||
|
this.fieldTypes.set(columnKey, ValueType.Float)
|
||||||
|
const manifest = this.manifests.get(columnKey)
|
||||||
|
if (manifest) manifest.valueType = ValueType.Float
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const normalizedValue = this.normalizeValue(value, buffer.valueType)
|
||||||
|
if (normalizedValue === undefined) {
|
||||||
|
// Unreachable by construction: the column was chosen BY this value's
|
||||||
|
// kind, so the encoding always accepts it. Reaching here would mean a
|
||||||
|
// value had been silently dropped from the index — the exact failure
|
||||||
|
// typed postings exist to end — so it is an error, never a skip.
|
||||||
|
throw new Error(
|
||||||
|
`ColumnStore: field '${field}' rejected a ${kind} value for its own ` +
|
||||||
|
`${ValueType[buffer.valueType]} posting column. The value would have ` +
|
||||||
|
`been dropped from the index while the row stayed readable by id — ` +
|
||||||
|
`this is a kind-routing bug, not a value the caller may ignore.`
|
||||||
|
)
|
||||||
|
}
|
||||||
|
buffer.add(normalizedValue, entityIntId)
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
|
@ -677,8 +980,15 @@ export class ColumnStore implements ColumnStoreProvider {
|
||||||
/** Torn-segment quarantine entries for a field (observability + heal input). */
|
/** Torn-segment quarantine entries for a field (observability + heal input). */
|
||||||
quarantinedSegments(field: string): Array<{ segment: string; error: string; hits: number }> {
|
quarantinedSegments(field: string): Array<{ segment: string; error: string; hits: number }> {
|
||||||
const out: Array<{ segment: string; error: string; hits: number }> = []
|
const out: Array<{ segment: string; error: string; hits: number }> = []
|
||||||
for (const [key, q] of this.segmentQuarantine) {
|
// Across every kind column of the field — a torn segment in the string
|
||||||
if (key.startsWith(`${field}:`)) out.push({ segment: key.slice(field.length + 1), error: q.error, hits: q.hits })
|
// postings is this field's torn segment as much as one in the numbers.
|
||||||
|
for (const column of this.columnsForField(field)) {
|
||||||
|
const prefix = `${column.key}:`
|
||||||
|
for (const [key, q] of this.segmentQuarantine) {
|
||||||
|
if (key.startsWith(prefix)) {
|
||||||
|
out.push({ segment: key.slice(prefix.length), error: q.error, hits: q.hits })
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
return out
|
return out
|
||||||
}
|
}
|
||||||
|
|
@ -850,17 +1160,22 @@ export class ColumnStore implements ColumnStoreProvider {
|
||||||
k: number,
|
k: number,
|
||||||
filterBitmap: RoaringBitmap32 | null
|
filterBitmap: RoaringBitmap32 | null
|
||||||
): Promise<number[]> {
|
): Promise<number[]> {
|
||||||
// Collect all cursors (segments + tail buffer)
|
// Collect cursors across EVERY kind the field holds. A single-kind field —
|
||||||
const segCursors = await this.getSegmentCursors(field)
|
// nearly all of them — merges exactly the cursors it always did.
|
||||||
const tailCursor = this.getTailBufferCursor(field)
|
|
||||||
|
|
||||||
// Create iterators for each cursor in the specified direction
|
|
||||||
const iterators: Generator<CursorEntry>[] = []
|
const iterators: Generator<CursorEntry>[] = []
|
||||||
for (const cursor of segCursors) {
|
const iteratorKindRank: number[] = []
|
||||||
iterators.push(order === 'asc' ? cursor.iterateForward() : cursor.iterateBackward())
|
for (const column of this.columnsForField(field)) {
|
||||||
}
|
const kindRank = POSTING_KINDS.indexOf(column.kind)
|
||||||
if (tailCursor) {
|
const segCursors = await this.getSegmentCursors(column.key)
|
||||||
iterators.push(order === 'asc' ? tailCursor.iterateForward() : tailCursor.iterateBackward())
|
for (const cursor of segCursors) {
|
||||||
|
iterators.push(order === 'asc' ? cursor.iterateForward() : cursor.iterateBackward())
|
||||||
|
iteratorKindRank.push(kindRank)
|
||||||
|
}
|
||||||
|
const tailCursor = this.getTailBufferCursor(column.key)
|
||||||
|
if (tailCursor) {
|
||||||
|
iterators.push(order === 'asc' ? tailCursor.iterateForward() : tailCursor.iterateBackward())
|
||||||
|
iteratorKindRank.push(kindRank)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if (iterators.length === 0) return []
|
if (iterators.length === 0) return []
|
||||||
|
|
@ -874,16 +1189,21 @@ export class ColumnStore implements ColumnStoreProvider {
|
||||||
value: next.value.value,
|
value: next.value.value,
|
||||||
entityIntId: next.value.entityIntId,
|
entityIntId: next.value.entityIntId,
|
||||||
cursorIndex: i,
|
cursorIndex: i,
|
||||||
|
kindRank: iteratorKindRank[i],
|
||||||
iterator: iterators[i]
|
iterator: iterators[i]
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Heapify
|
// Heapify. A number and a string have no ordering between them, so a
|
||||||
const isString = (this.fieldTypes.get(field) ?? ValueType.Number) === ValueType.String
|
// mixed-kind field orders by KIND first (POSTING_KINDS order) and by value
|
||||||
|
// within a kind — one defined total order instead of a comparison whose
|
||||||
|
// answer depends on which value happened to be on the left.
|
||||||
const compare = (a: HeapEntry, b: HeapEntry): number => {
|
const compare = (a: HeapEntry, b: HeapEntry): number => {
|
||||||
let cmp: number
|
let cmp: number
|
||||||
if (isString) {
|
if (a.kindRank !== b.kindRank) {
|
||||||
|
cmp = a.kindRank - b.kindRank
|
||||||
|
} else if (POSTING_KINDS[a.kindRank] === 'string') {
|
||||||
cmp = compareCodePoints(String(a.value), String(b.value))
|
cmp = compareCodePoints(String(a.value), String(b.value))
|
||||||
} else {
|
} else {
|
||||||
cmp = (a.value as number) - (b.value as number)
|
cmp = (a.value as number) - (b.value as number)
|
||||||
|
|
@ -915,6 +1235,7 @@ export class ColumnStore implements ColumnStoreProvider {
|
||||||
value: next.value.value,
|
value: next.value.value,
|
||||||
entityIntId: next.value.entityIntId,
|
entityIntId: next.value.entityIntId,
|
||||||
cursorIndex: top.cursorIndex,
|
cursorIndex: top.cursorIndex,
|
||||||
|
kindRank: top.kindRank,
|
||||||
iterator: top.iterator
|
iterator: top.iterator
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
@ -922,8 +1243,11 @@ export class ColumnStore implements ColumnStoreProvider {
|
||||||
this.heapDown(heap, 0, compare)
|
this.heapDown(heap, 0, compare)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Apply global deleted check, filter, and dedup
|
// Apply global deleted check, filter, and dedup. The deleted bitmap is
|
||||||
const deleted = this.deletedEntities.get(field)
|
// per COLUMN, and the entry came from the column its kind names.
|
||||||
|
const deleted = this.deletedEntities.get(
|
||||||
|
this.columnKey(field, POSTING_KINDS[top.kindRank]) ?? field
|
||||||
|
)
|
||||||
if (deleted && deleted.has(top.entityIntId)) continue
|
if (deleted && deleted.has(top.entityIntId)) continue
|
||||||
if (seen.has(top.entityIntId)) continue
|
if (seen.has(top.entityIntId)) continue
|
||||||
if (filterBitmap && !filterBitmap.has(top.entityIntId)) continue
|
if (filterBitmap && !filterBitmap.has(top.entityIntId)) continue
|
||||||
|
|
@ -965,35 +1289,31 @@ export class ColumnStore implements ColumnStoreProvider {
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Infer ValueType from a JavaScript value.
|
* Encode a value for the column its own kind selected.
|
||||||
*/
|
*
|
||||||
private inferValueType(value: unknown): ValueType {
|
* This does NOT convert between kinds. It used to: a string reaching a
|
||||||
if (typeof value === 'boolean') return ValueType.Boolean
|
* numeric column was run through `Number(value)`, and a number reaching a
|
||||||
if (typeof value === 'number') {
|
* numeric column was run through `Math.round`, so `'electronics'` became
|
||||||
return Number.isInteger(value) ? ValueType.Number : ValueType.Float
|
* `NaN` and vanished while `4.5` became `5` and answered the wrong query.
|
||||||
}
|
* Kind routing removes the need for either — the only work left is picking
|
||||||
return ValueType.String
|
* the encoding the column already committed to.
|
||||||
}
|
*
|
||||||
|
* @returns The encoded value, or `undefined` if the value does not belong in
|
||||||
/**
|
* this column at all — which the caller treats as a routing bug and
|
||||||
* Normalize a JavaScript value to the column's ValueType.
|
* raises, never as a value to skip.
|
||||||
*/
|
*/
|
||||||
private normalizeValue(value: unknown, type: ValueType): number | string | undefined {
|
private normalizeValue(value: unknown, type: ValueType): number | string | undefined {
|
||||||
switch (type) {
|
switch (type) {
|
||||||
case ValueType.Number:
|
case ValueType.Number:
|
||||||
if (typeof value === 'number') return Math.round(value)
|
// Integer column. Non-integers widen it to Float before reaching here.
|
||||||
if (typeof value === 'string') { const n = Number(value); return isNaN(n) ? undefined : Math.round(n) }
|
return typeof value === 'number' && Number.isInteger(value) ? value : undefined
|
||||||
if (typeof value === 'boolean') return value ? 1 : 0
|
|
||||||
return undefined
|
|
||||||
case ValueType.Float:
|
case ValueType.Float:
|
||||||
if (typeof value === 'number') return value
|
return typeof value === 'number' ? value : undefined
|
||||||
if (typeof value === 'string') { const n = Number(value); return isNaN(n) ? undefined : n }
|
|
||||||
return undefined
|
|
||||||
case ValueType.Boolean:
|
case ValueType.Boolean:
|
||||||
if (typeof value === 'boolean') return value ? 1 : 0
|
return typeof value === 'boolean' ? (value ? 1 : 0) : undefined
|
||||||
if (typeof value === 'number') return value ? 1 : 0
|
|
||||||
return undefined
|
|
||||||
case ValueType.String:
|
case ValueType.String:
|
||||||
|
// The string kind is also where objects and bigints land, exactly as
|
||||||
|
// they always did.
|
||||||
return String(value)
|
return String(value)
|
||||||
default:
|
default:
|
||||||
return undefined
|
return undefined
|
||||||
|
|
|
||||||
|
|
@ -55,8 +55,12 @@ export class ColumnTailBuffer {
|
||||||
/** Field name this buffer is for. */
|
/** Field name this buffer is for. */
|
||||||
readonly fieldName: string
|
readonly fieldName: string
|
||||||
|
|
||||||
/** Value type determines sort comparator. */
|
/**
|
||||||
readonly valueType: ValueType
|
* Value type determines sort comparator and segment encoding.
|
||||||
|
*
|
||||||
|
* Widened in place by {@link promoteToFloat} — never otherwise reassigned.
|
||||||
|
*/
|
||||||
|
valueType: ValueType
|
||||||
|
|
||||||
/** Flush threshold. */
|
/** Flush threshold. */
|
||||||
readonly threshold: number
|
readonly threshold: number
|
||||||
|
|
@ -81,6 +85,38 @@ export class ColumnTailBuffer {
|
||||||
this.threshold = threshold
|
this.threshold = threshold
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Widen an integer column to floating point, losslessly and in place.
|
||||||
|
*
|
||||||
|
* The number posting kind holds every JavaScript number, but a segment picks
|
||||||
|
* ONE encoding: i64 for integers, f64 for the rest. A column that has only
|
||||||
|
* ever seen integers is written as i64; the first non-integer to arrive
|
||||||
|
* widens it here, so that value is stored as itself instead of being rounded
|
||||||
|
* to the nearest integer with no error — the rounding that made `4.5` and
|
||||||
|
* `5.5` both answer `where {score: 5}` and neither answer its own value.
|
||||||
|
*
|
||||||
|
* Widening is lossless in both directions it has to be: every value already
|
||||||
|
* buffered is an integer, and every integer is exactly representable as f64.
|
||||||
|
* Segments already on disk keep their own i64 encoding in their own headers
|
||||||
|
* and keep decoding by it — only segments written from here on are f64.
|
||||||
|
*
|
||||||
|
* @throws Error if called on a column that is not an integer column — the
|
||||||
|
* only legal widening is Number → Float, and any other request is a bug in
|
||||||
|
* the caller's kind routing rather than something to absorb quietly.
|
||||||
|
*/
|
||||||
|
promoteToFloat(): void {
|
||||||
|
if (this.valueType === ValueType.Float) return
|
||||||
|
if (this.valueType !== ValueType.Number) {
|
||||||
|
throw new Error(
|
||||||
|
`ColumnTailBuffer '${this.fieldName}': cannot widen a ` +
|
||||||
|
`${ValueType[this.valueType]} column to Float — only an integer ` +
|
||||||
|
`(Number) column widens, and this call means a value reached the ` +
|
||||||
|
`wrong kind's column`
|
||||||
|
)
|
||||||
|
}
|
||||||
|
this.valueType = ValueType.Float
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Add a (value, entityIntId) entry to the buffer.
|
* Add a (value, entityIntId) entry to the buffer.
|
||||||
*
|
*
|
||||||
|
|
|
||||||
|
|
@ -58,6 +58,53 @@ export enum ValueType {
|
||||||
Boolean = 3
|
Boolean = 3
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* The KIND of a value, as the query language sees it.
|
||||||
|
*
|
||||||
|
* A kind is a JavaScript `typeof` class, not a storage encoding: `5` and `5.5`
|
||||||
|
* are one kind (`'number'`) held in one posting column, even though they need
|
||||||
|
* different segment encodings (i64 vs f64 — see {@link ValueType}).
|
||||||
|
*
|
||||||
|
* A field holds ONE POSTING COLUMN PER KIND, so `category` may carry string
|
||||||
|
* values and number values at the same time and answer equality on each. This
|
||||||
|
* replaces the first-writer type freeze, under which the first value's type
|
||||||
|
* became the field's type and every later value of another kind was coerced —
|
||||||
|
* or, when coercion failed (`Number('electronics')`), dropped from the index
|
||||||
|
* with no error: the row stayed readable by id and by vector but vanished from
|
||||||
|
* every equality filter on that field.
|
||||||
|
*
|
||||||
|
* Kinds do not coerce into one another at query time either: `where {c: 5}`
|
||||||
|
* matches rows written with the NUMBER `5`, and `where {c: '5'}` matches rows
|
||||||
|
* written with the STRING `'5'`. Neither ever matches the other.
|
||||||
|
*
|
||||||
|
* Values that are none of these three (objects, bigints) index as strings —
|
||||||
|
* the same `String(value)` treatment they received before.
|
||||||
|
*/
|
||||||
|
export type PostingKind = 'number' | 'string' | 'boolean'
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Every posting kind, in the order that defines cross-kind sort position.
|
||||||
|
*
|
||||||
|
* A mixed-kind field has no natural total order — a number does not compare
|
||||||
|
* with a string — so `sortTopK` orders by KIND first (numbers, then strings,
|
||||||
|
* then booleans) and by value within a kind. A single-kind field, which is
|
||||||
|
* nearly every field, sorts exactly as it always did.
|
||||||
|
*/
|
||||||
|
export const POSTING_KINDS: readonly PostingKind[] = ['number', 'string', 'boolean']
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Path segment marking a field's NON-PRIMARY kind columns on disk.
|
||||||
|
*
|
||||||
|
* The first kind a field ever sees keeps the historical layout —
|
||||||
|
* `<base>/<field>/MANIFEST.json` and `<base>/<field>/L0-NNNNNN` — so every
|
||||||
|
* index written before typed postings opens unchanged, and the byte-for-byte
|
||||||
|
* interchange with the native column store is untouched for the single-kind
|
||||||
|
* fields that are nearly all of them. A second kind arriving on the same field
|
||||||
|
* gets its own column at `<base>/<field>/k/<kind>/…` rather than overwriting or
|
||||||
|
* being coerced into the first.
|
||||||
|
*/
|
||||||
|
export const KIND_PATH_SEGMENT = 'k'
|
||||||
|
|
||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
// Segment header and footer
|
// Segment header and footer
|
||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
|
|
@ -267,6 +314,19 @@ export interface ColumnStoreProvider {
|
||||||
*/
|
*/
|
||||||
hasField(field: string): boolean
|
hasField(field: string): boolean
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Which value KINDS this field actually holds, in {@link POSTING_KINDS}
|
||||||
|
* order — the honest answer to "what type is this field?" for a field that
|
||||||
|
* carries more than one.
|
||||||
|
*
|
||||||
|
* OPTIONAL so an implementation written against the pre-typed-postings
|
||||||
|
* contract still satisfies this interface; feature-detect before calling.
|
||||||
|
*
|
||||||
|
* @param field - Field name
|
||||||
|
* @returns Every kind with at least one posting, or `[]` for an unknown field
|
||||||
|
*/
|
||||||
|
getFieldKinds?(field: string): PostingKind[]
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Flush all in-memory tail buffers to L0 segments on disk.
|
* Flush all in-memory tail buffers to L0 segments on disk.
|
||||||
* Saves all manifests.
|
* Saves all manifests.
|
||||||
|
|
|
||||||
|
|
@ -55,8 +55,30 @@ export enum FieldType {
|
||||||
*/
|
*/
|
||||||
export interface FieldTypeInfo {
|
export interface FieldTypeInfo {
|
||||||
field: string
|
field: string
|
||||||
|
/**
|
||||||
|
* The DOMINANT reading of the field — one type, the most specific one every
|
||||||
|
* sampled value satisfies.
|
||||||
|
*
|
||||||
|
* A field is not obliged to hold one kind, so this is not the whole answer
|
||||||
|
* for a field that holds several. Read {@link kinds} beside it: a field
|
||||||
|
* carrying `'electronics'` and `5` infers as STRING here and reports
|
||||||
|
* `['number', 'string']` there, and the metadata index keeps a separate
|
||||||
|
* posting column for each of them.
|
||||||
|
*/
|
||||||
inferredType: FieldType
|
inferredType: FieldType
|
||||||
confidence: number // 0-1 confidence score
|
confidence: number // 0-1 confidence score
|
||||||
|
/**
|
||||||
|
* Every value KIND observed in the sample, in the order
|
||||||
|
* number → string → boolean. More than one entry means a genuinely
|
||||||
|
* mixed field, and every one of those kinds is independently filterable.
|
||||||
|
*
|
||||||
|
* Kinds are JavaScript `typeof` classes, one level coarser than
|
||||||
|
* {@link FieldType}: a UUID and a category name are both `'string'`, and an
|
||||||
|
* integer and a timestamp are both `'number'`.
|
||||||
|
*
|
||||||
|
* Optional only for cached analyses written before this was reported.
|
||||||
|
*/
|
||||||
|
kinds?: Array<'number' | 'string' | 'boolean'>
|
||||||
sampleSize: number // Number of values analyzed
|
sampleSize: number // Number of values analyzed
|
||||||
lastUpdated: number // Timestamp of last analysis
|
lastUpdated: number // Timestamp of last analysis
|
||||||
detectionMethod: 'value' // Always 'value' (no fallbacks!)
|
detectionMethod: 'value' // Always 'value' (no fallbacks!)
|
||||||
|
|
@ -133,14 +155,71 @@ export class FieldTypeInference {
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Analyze values to determine field type
|
* Analyze values to determine field type, and report every KIND the field
|
||||||
|
* actually holds alongside it.
|
||||||
|
*
|
||||||
|
* The classification below picks ONE type, because every one of its
|
||||||
|
* heuristics asks `samples.every(...)`: a field carrying `'electronics'` and
|
||||||
|
* `5` satisfies none of them and lands on STRING. That single answer is true
|
||||||
|
* as far as it goes — string is the dominant reading — but on its own it
|
||||||
|
* says nothing about the numbers also in the field, and a caller that treats
|
||||||
|
* it as the field's only type reproduces the first-writer freeze the index
|
||||||
|
* itself no longer has. {@link FieldTypeInfo.kinds} carries the rest.
|
||||||
|
*/
|
||||||
|
private async analyzeValues(field: string, values: any[]): Promise<FieldTypeInfo> {
|
||||||
|
const info = await this.classifyValues(field, values)
|
||||||
|
info.kinds = FieldTypeInference.observedKinds(values)
|
||||||
|
if (info.kinds.length > 1 && info.metadata) {
|
||||||
|
info.metadata.format = `${info.metadata.format} (field also holds: ${info.kinds
|
||||||
|
.filter((k) => k !== FieldTypeInference.kindOfType(info.inferredType))
|
||||||
|
.join(', ')})`
|
||||||
|
}
|
||||||
|
return info
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* The distinct value kinds present in a sample, in a stable order.
|
||||||
|
*
|
||||||
|
* Kinds are JavaScript `typeof` classes — the same classes the metadata
|
||||||
|
* index keeps separate posting columns for — not the finer
|
||||||
|
* {@link FieldType} readings, which are interpretations layered on top of
|
||||||
|
* them (a UUID and a category name are both the `string` kind).
|
||||||
|
*/
|
||||||
|
private static observedKinds(values: any[]): Array<'number' | 'string' | 'boolean'> {
|
||||||
|
const order: Array<'number' | 'string' | 'boolean'> = ['number', 'string', 'boolean']
|
||||||
|
const seen = new Set<'number' | 'string' | 'boolean'>()
|
||||||
|
for (const v of values) {
|
||||||
|
if (v === null || v === undefined) continue
|
||||||
|
const t = typeof v
|
||||||
|
seen.add(t === 'number' ? 'number' : t === 'boolean' ? 'boolean' : 'string')
|
||||||
|
}
|
||||||
|
return order.filter((k) => seen.has(k))
|
||||||
|
}
|
||||||
|
|
||||||
|
/** The value kind a {@link FieldType} reading is an interpretation of. */
|
||||||
|
private static kindOfType(type: FieldType): 'number' | 'string' | 'boolean' {
|
||||||
|
switch (type) {
|
||||||
|
case FieldType.BOOLEAN:
|
||||||
|
return 'boolean'
|
||||||
|
case FieldType.INTEGER:
|
||||||
|
case FieldType.FLOAT:
|
||||||
|
case FieldType.TIMESTAMP_MS:
|
||||||
|
case FieldType.TIMESTAMP_S:
|
||||||
|
return 'number'
|
||||||
|
default:
|
||||||
|
return 'string'
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Classify values into a single field type.
|
||||||
*
|
*
|
||||||
* Uses DuckDB-inspired type detection order:
|
* Uses DuckDB-inspired type detection order:
|
||||||
* BOOLEAN → INTEGER → FLOAT → DATE → TIMESTAMP → UUID → STRING
|
* BOOLEAN → INTEGER → FLOAT → DATE → TIMESTAMP → UUID → STRING
|
||||||
*
|
*
|
||||||
* No fallbacks - pure value-based detection
|
* No fallbacks - pure value-based detection
|
||||||
*/
|
*/
|
||||||
private async analyzeValues(field: string, values: any[]): Promise<FieldTypeInfo> {
|
private async classifyValues(field: string, values: any[]): Promise<FieldTypeInfo> {
|
||||||
// Filter null/undefined values
|
// Filter null/undefined values
|
||||||
const validValues = values.filter(v => v !== null && v !== undefined)
|
const validValues = values.filter(v => v !== null && v !== undefined)
|
||||||
|
|
||||||
|
|
|
||||||
122
tests/regression/metadata-field-typing.unit.test.ts
Normal file
122
tests/regression/metadata-field-typing.unit.test.ts
Normal file
|
|
@ -0,0 +1,122 @@
|
||||||
|
/**
|
||||||
|
* @module metadata-field-typing.unit.test
|
||||||
|
* @description Regression: a metadata field that holds more than one value
|
||||||
|
* KIND stays fully filterable on every kind it holds.
|
||||||
|
*
|
||||||
|
* The defect this pins, reproduced on the released engine: the metadata index
|
||||||
|
* fixed a field's value type from the FIRST value it saw, and every later value
|
||||||
|
* of a different type was coerced to that type or, when coercion failed,
|
||||||
|
* dropped from the index in silence. Writing `category: 'electronics'` rows and
|
||||||
|
* then `category: 5` rows left `find({ where: { category: 5 } })` returning
|
||||||
|
* nothing — while the same rows in a numbers-only corpus answered correctly.
|
||||||
|
* The rows themselves were never lost: they stayed readable by id and by vector
|
||||||
|
* search, and only ever went missing from equality filters on that one field,
|
||||||
|
* which is what made it so quiet.
|
||||||
|
*
|
||||||
|
* Order is the whole point of these cases. Neither writer owns the field, so
|
||||||
|
* strings-then-numbers and numbers-then-strings must give the same answers.
|
||||||
|
*/
|
||||||
|
|
||||||
|
import { describe, it, expect } from 'vitest'
|
||||||
|
import { Brainy } from '../../src/brainy.js'
|
||||||
|
import { NounType } from '../../src/types/graphTypes.js'
|
||||||
|
|
||||||
|
/** A brain over memory storage, with a corpus written in the given order. */
|
||||||
|
async function brainWith(
|
||||||
|
rows: Array<{ label: string; category: unknown }>
|
||||||
|
): Promise<Brainy> {
|
||||||
|
const brainy = new Brainy({ requireSubtype: false, storage: { type: 'memory' } })
|
||||||
|
await brainy.init()
|
||||||
|
for (const row of rows) {
|
||||||
|
await brainy.add({
|
||||||
|
data: `item ${row.label}`,
|
||||||
|
type: NounType.Thing,
|
||||||
|
metadata: { label: row.label, category: row.category }
|
||||||
|
})
|
||||||
|
}
|
||||||
|
return brainy
|
||||||
|
}
|
||||||
|
|
||||||
|
const labelsOf = (results: Array<{ metadata?: Record<string, unknown> }>): string[] =>
|
||||||
|
results.map((r) => String(r.metadata?.label)).sort()
|
||||||
|
|
||||||
|
describe('regression: a mixed-kind metadata field filters on every kind', { timeout: 180_000 }, () => {
|
||||||
|
it('finds number rows written after string rows', async () => {
|
||||||
|
const brainy = await brainWith([
|
||||||
|
{ label: 'e1', category: 'electronics' },
|
||||||
|
{ label: 'f1', category: 'furniture' },
|
||||||
|
{ label: 'n1', category: 5 },
|
||||||
|
{ label: 'n2', category: 5 },
|
||||||
|
{ label: 'n3', category: 7 }
|
||||||
|
])
|
||||||
|
try {
|
||||||
|
expect(labelsOf(await brainy.find({ where: { category: 5 }, limit: 100 }))).toEqual(['n1', 'n2'])
|
||||||
|
expect(labelsOf(await brainy.find({ where: { category: 7 }, limit: 100 }))).toEqual(['n3'])
|
||||||
|
expect(labelsOf(await brainy.find({ where: { category: 'electronics' }, limit: 100 }))).toEqual(['e1'])
|
||||||
|
expect(labelsOf(await brainy.find({ where: { category: 'furniture' }, limit: 100 }))).toEqual(['f1'])
|
||||||
|
} finally {
|
||||||
|
await brainy.close()
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
it('finds string rows written after number rows', async () => {
|
||||||
|
const brainy = await brainWith([
|
||||||
|
{ label: 'n1', category: 5 },
|
||||||
|
{ label: 'n2', category: 5 },
|
||||||
|
{ label: 'e1', category: 'electronics' },
|
||||||
|
{ label: 'e2', category: 'electronics' }
|
||||||
|
])
|
||||||
|
try {
|
||||||
|
expect(labelsOf(await brainy.find({ where: { category: 'electronics' }, limit: 100 }))).toEqual(['e1', 'e2'])
|
||||||
|
expect(labelsOf(await brainy.find({ where: { category: 5 }, limit: 100 }))).toEqual(['n1', 'n2'])
|
||||||
|
} finally {
|
||||||
|
await brainy.close()
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
it('keeps `5` and `\'5\'` apart — a kind is part of the value, not a formatting detail', async () => {
|
||||||
|
const brainy = await brainWith([
|
||||||
|
{ label: 'num', category: 5 },
|
||||||
|
{ label: 'str', category: '5' }
|
||||||
|
])
|
||||||
|
try {
|
||||||
|
expect(labelsOf(await brainy.find({ where: { category: 5 }, limit: 100 }))).toEqual(['num'])
|
||||||
|
expect(labelsOf(await brainy.find({ where: { category: '5' }, limit: 100 }))).toEqual(['str'])
|
||||||
|
} finally {
|
||||||
|
await brainy.close()
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
it('serves booleans mixed into a field that already holds strings', async () => {
|
||||||
|
const brainy = await brainWith([
|
||||||
|
{ label: 's1', category: 'yes' },
|
||||||
|
{ label: 'b1', category: true },
|
||||||
|
{ label: 'b2', category: false }
|
||||||
|
])
|
||||||
|
try {
|
||||||
|
expect(labelsOf(await brainy.find({ where: { category: true }, limit: 100 }))).toEqual(['b1'])
|
||||||
|
expect(labelsOf(await brainy.find({ where: { category: false }, limit: 100 }))).toEqual(['b2'])
|
||||||
|
expect(labelsOf(await brainy.find({ where: { category: 'yes' }, limit: 100 }))).toEqual(['s1'])
|
||||||
|
} finally {
|
||||||
|
await brainy.close()
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
it('ranges over the numeric part of a mixed field', async () => {
|
||||||
|
const brainy = await brainWith([
|
||||||
|
{ label: 'unpriced', category: 'on request' },
|
||||||
|
{ label: 'cheap', category: 100 },
|
||||||
|
{ label: 'mid', category: 500 },
|
||||||
|
{ label: 'dear', category: 900 }
|
||||||
|
])
|
||||||
|
try {
|
||||||
|
const found = await brainy.find({
|
||||||
|
where: { category: { greaterThan: 200 } },
|
||||||
|
limit: 100
|
||||||
|
})
|
||||||
|
expect(labelsOf(found)).toEqual(['dear', 'mid'])
|
||||||
|
} finally {
|
||||||
|
await brainy.close()
|
||||||
|
}
|
||||||
|
})
|
||||||
|
})
|
||||||
241
tests/unit/indexes/columnStore/column-store-mixed-kind.test.ts
Normal file
241
tests/unit/indexes/columnStore/column-store-mixed-kind.test.ts
Normal file
|
|
@ -0,0 +1,241 @@
|
||||||
|
/**
|
||||||
|
* @module column-store-mixed-kind.test
|
||||||
|
* @description Typed posting lists: one field, several value KINDS, each
|
||||||
|
* answerable on its own.
|
||||||
|
*
|
||||||
|
* The behaviour these pin replaced a first-writer type freeze. The first value
|
||||||
|
* a field ever saw fixed that field's type; every later value of another kind
|
||||||
|
* was coerced to it, and when coercion failed — `Number('electronics')` — the
|
||||||
|
* value was dropped from the index with no error at all. The row stayed
|
||||||
|
* readable by id and by vector and vanished from every equality filter on the
|
||||||
|
* field. These tests therefore care about ORDER: strings-then-numbers and
|
||||||
|
* numbers-then-strings have to behave identically, because neither writer owns
|
||||||
|
* the field.
|
||||||
|
*
|
||||||
|
* Kinds never coerce into one another at query time either. `5` and `'5'` are
|
||||||
|
* different values and match different rows.
|
||||||
|
*/
|
||||||
|
|
||||||
|
import { describe, it, expect, beforeEach, afterEach } from 'vitest'
|
||||||
|
import { ColumnStore } from '../../../../src/indexes/columnStore/ColumnStore.js'
|
||||||
|
import { MemoryStorage } from '../../../../src/storage/adapters/memoryStorage.js'
|
||||||
|
import { EntityIdMapper } from '../../../../src/utils/entityIdMapper.js'
|
||||||
|
|
||||||
|
describe('ColumnStore — typed posting lists per (field, kind)', () => {
|
||||||
|
let storage: MemoryStorage
|
||||||
|
let idMapper: EntityIdMapper
|
||||||
|
let store: ColumnStore
|
||||||
|
|
||||||
|
beforeEach(async () => {
|
||||||
|
storage = new MemoryStorage()
|
||||||
|
await storage.init()
|
||||||
|
idMapper = new EntityIdMapper({ storage, storageKey: 'test:idMapper' })
|
||||||
|
await idMapper.init()
|
||||||
|
|
||||||
|
store = new ColumnStore({ flushThreshold: 10 })
|
||||||
|
await store.init(storage, idMapper)
|
||||||
|
})
|
||||||
|
|
||||||
|
afterEach(async () => {
|
||||||
|
await store.close()
|
||||||
|
})
|
||||||
|
|
||||||
|
/** Resolve a filter to the sorted UUIDs it matched. */
|
||||||
|
const uuidsOf = async (field: string, value: unknown): Promise<string[]> => {
|
||||||
|
const bitmap = await store.filter(field, value)
|
||||||
|
return Array.from(bitmap)
|
||||||
|
.map((id) => idMapper.getUuid(Number(id)))
|
||||||
|
.filter((u): u is string => u !== undefined)
|
||||||
|
.sort()
|
||||||
|
}
|
||||||
|
|
||||||
|
describe('equality answers on the query value’s own kind', () => {
|
||||||
|
it('serves numbers written AFTER strings on the same field', async () => {
|
||||||
|
store.addEntity(BigInt(idMapper.getOrAssign('s1')), { category: 'electronics' })
|
||||||
|
store.addEntity(BigInt(idMapper.getOrAssign('s2')), { category: 'furniture' })
|
||||||
|
store.addEntity(BigInt(idMapper.getOrAssign('n1')), { category: 5 })
|
||||||
|
store.addEntity(BigInt(idMapper.getOrAssign('n2')), { category: 5 })
|
||||||
|
store.addEntity(BigInt(idMapper.getOrAssign('n3')), { category: 7 })
|
||||||
|
|
||||||
|
// The numbers are in the index, though a string got there first.
|
||||||
|
expect(await uuidsOf('category', 5)).toEqual(['n1', 'n2'])
|
||||||
|
expect(await uuidsOf('category', 7)).toEqual(['n3'])
|
||||||
|
// And the strings did not move.
|
||||||
|
expect(await uuidsOf('category', 'electronics')).toEqual(['s1'])
|
||||||
|
expect(await uuidsOf('category', 'furniture')).toEqual(['s2'])
|
||||||
|
})
|
||||||
|
|
||||||
|
it('serves strings written AFTER numbers on the same field', async () => {
|
||||||
|
store.addEntity(BigInt(idMapper.getOrAssign('n1')), { category: 5 })
|
||||||
|
store.addEntity(BigInt(idMapper.getOrAssign('n2')), { category: 5 })
|
||||||
|
store.addEntity(BigInt(idMapper.getOrAssign('s1')), { category: 'electronics' })
|
||||||
|
store.addEntity(BigInt(idMapper.getOrAssign('s2')), { category: 'electronics' })
|
||||||
|
|
||||||
|
// 'electronics' would have become NaN and been dropped under the freeze.
|
||||||
|
expect(await uuidsOf('category', 'electronics')).toEqual(['s1', 's2'])
|
||||||
|
expect(await uuidsOf('category', 5)).toEqual(['n1', 'n2'])
|
||||||
|
})
|
||||||
|
|
||||||
|
it('does not coerce a number query into the string postings, or back', async () => {
|
||||||
|
store.addEntity(BigInt(idMapper.getOrAssign('num')), { code: 5 })
|
||||||
|
store.addEntity(BigInt(idMapper.getOrAssign('str')), { code: '5' })
|
||||||
|
|
||||||
|
expect(await uuidsOf('code', 5)).toEqual(['num'])
|
||||||
|
expect(await uuidsOf('code', '5')).toEqual(['str'])
|
||||||
|
})
|
||||||
|
|
||||||
|
it('serves booleans mixed into a field that already holds strings and numbers', async () => {
|
||||||
|
store.addEntity(BigInt(idMapper.getOrAssign('s1')), { flag: 'yes' })
|
||||||
|
store.addEntity(BigInt(idMapper.getOrAssign('n1')), { flag: 1 })
|
||||||
|
store.addEntity(BigInt(idMapper.getOrAssign('b1')), { flag: true })
|
||||||
|
store.addEntity(BigInt(idMapper.getOrAssign('b2')), { flag: false })
|
||||||
|
|
||||||
|
expect(await uuidsOf('flag', true)).toEqual(['b1'])
|
||||||
|
expect(await uuidsOf('flag', false)).toEqual(['b2'])
|
||||||
|
// `true` stores as 1 internally; that is an encoding, not a value.
|
||||||
|
expect(await uuidsOf('flag', 1)).toEqual(['n1'])
|
||||||
|
expect(await uuidsOf('flag', 'yes')).toEqual(['s1'])
|
||||||
|
})
|
||||||
|
|
||||||
|
it('answers nothing — not something coerced — for a kind the field never held', async () => {
|
||||||
|
store.addEntity(BigInt(idMapper.getOrAssign('s1')), { category: 'electronics' })
|
||||||
|
|
||||||
|
expect(await uuidsOf('category', 5)).toEqual([])
|
||||||
|
expect(await uuidsOf('category', true)).toEqual([])
|
||||||
|
})
|
||||||
|
|
||||||
|
it('holds every kind across a flush, not just the one in the tail buffer', async () => {
|
||||||
|
store.addEntity(BigInt(idMapper.getOrAssign('s1')), { category: 'electronics' })
|
||||||
|
store.addEntity(BigInt(idMapper.getOrAssign('n1')), { category: 5 })
|
||||||
|
await store.flush()
|
||||||
|
store.addEntity(BigInt(idMapper.getOrAssign('s2')), { category: 'electronics' })
|
||||||
|
store.addEntity(BigInt(idMapper.getOrAssign('n2')), { category: 5 })
|
||||||
|
|
||||||
|
expect(await uuidsOf('category', 'electronics')).toEqual(['s1', 's2'])
|
||||||
|
expect(await uuidsOf('category', 5)).toEqual(['n1', 'n2'])
|
||||||
|
})
|
||||||
|
})
|
||||||
|
|
||||||
|
describe('range filters read the numeric postings', () => {
|
||||||
|
it('ranges over the numeric subset of a mixed field, ignoring its strings', async () => {
|
||||||
|
store.addEntity(BigInt(idMapper.getOrAssign('cheap')), { price: 100 })
|
||||||
|
store.addEntity(BigInt(idMapper.getOrAssign('mid')), { price: 500 })
|
||||||
|
store.addEntity(BigInt(idMapper.getOrAssign('dear')), { price: 900 })
|
||||||
|
store.addEntity(BigInt(idMapper.getOrAssign('unpriced')), { price: 'on request' })
|
||||||
|
await store.flush()
|
||||||
|
|
||||||
|
const inRange = await store.rangeQuery('price', 200, 1000)
|
||||||
|
const uuids = Array.from(inRange)
|
||||||
|
.map((id) => idMapper.getUuid(Number(id)))
|
||||||
|
.sort()
|
||||||
|
expect(uuids).toEqual(['dear', 'mid'])
|
||||||
|
})
|
||||||
|
|
||||||
|
it('an unbounded range still reports every kind — it is the “has a value” probe', async () => {
|
||||||
|
store.addEntity(BigInt(idMapper.getOrAssign('n1')), { mixed: 42 })
|
||||||
|
store.addEntity(BigInt(idMapper.getOrAssign('s1')), { mixed: 'text' })
|
||||||
|
store.addEntity(BigInt(idMapper.getOrAssign('b1')), { mixed: true })
|
||||||
|
await store.flush()
|
||||||
|
|
||||||
|
const anyValue = await store.rangeQuery('mixed')
|
||||||
|
const uuids = Array.from(anyValue)
|
||||||
|
.map((id) => idMapper.getUuid(Number(id)))
|
||||||
|
.sort()
|
||||||
|
expect(uuids).toEqual(['b1', 'n1', 's1'])
|
||||||
|
})
|
||||||
|
})
|
||||||
|
|
||||||
|
describe('the index reports what a field actually holds', () => {
|
||||||
|
it('names every kind present, not the one that got there first', async () => {
|
||||||
|
store.addEntity(BigInt(idMapper.getOrAssign('s1')), { category: 'electronics' })
|
||||||
|
expect(store.getFieldKinds('category')).toEqual(['string'])
|
||||||
|
|
||||||
|
store.addEntity(BigInt(idMapper.getOrAssign('n1')), { category: 5 })
|
||||||
|
store.addEntity(BigInt(idMapper.getOrAssign('b1')), { category: true })
|
||||||
|
expect(store.getFieldKinds('category')).toEqual(['number', 'string', 'boolean'])
|
||||||
|
|
||||||
|
// And the field is still ONE field by name.
|
||||||
|
expect(store.getIndexedFields()).toEqual(['category'])
|
||||||
|
expect(store.hasField('category')).toBe(true)
|
||||||
|
})
|
||||||
|
|
||||||
|
it('reports an unknown field as holding nothing', () => {
|
||||||
|
expect(store.getFieldKinds('never-written')).toEqual([])
|
||||||
|
})
|
||||||
|
})
|
||||||
|
|
||||||
|
describe('an integer column widens rather than rounding', () => {
|
||||||
|
it('keeps a non-integer written after integers as itself', async () => {
|
||||||
|
store.addEntity(BigInt(idMapper.getOrAssign('a')), { score: 4 })
|
||||||
|
store.addEntity(BigInt(idMapper.getOrAssign('b')), { score: 4.5 })
|
||||||
|
store.addEntity(BigInt(idMapper.getOrAssign('c')), { score: 5 })
|
||||||
|
await store.flush()
|
||||||
|
|
||||||
|
// 4.5 used to round to 5 and answer `score === 5` alongside c.
|
||||||
|
expect(await uuidsOf('score', 4.5)).toEqual(['b'])
|
||||||
|
expect(await uuidsOf('score', 5)).toEqual(['c'])
|
||||||
|
expect(await uuidsOf('score', 4)).toEqual(['a'])
|
||||||
|
})
|
||||||
|
})
|
||||||
|
|
||||||
|
describe('close then reopen', () => {
|
||||||
|
it('keeps every typed posting, on the same storage', async () => {
|
||||||
|
store.addEntity(BigInt(idMapper.getOrAssign('s1')), { category: 'electronics' })
|
||||||
|
store.addEntity(BigInt(idMapper.getOrAssign('n1')), { category: 5 })
|
||||||
|
store.addEntity(BigInt(idMapper.getOrAssign('b1')), { category: true })
|
||||||
|
store.addEntity(BigInt(idMapper.getOrAssign('f1')), { score: 1.5 })
|
||||||
|
await store.flush()
|
||||||
|
await store.close()
|
||||||
|
|
||||||
|
store = new ColumnStore({ flushThreshold: 10 })
|
||||||
|
await store.init(storage, idMapper)
|
||||||
|
|
||||||
|
expect(store.getFieldKinds('category')).toEqual(['number', 'string', 'boolean'])
|
||||||
|
expect(await uuidsOf('category', 'electronics')).toEqual(['s1'])
|
||||||
|
expect(await uuidsOf('category', 5)).toEqual(['n1'])
|
||||||
|
expect(await uuidsOf('category', true)).toEqual(['b1'])
|
||||||
|
expect(await uuidsOf('score', 1.5)).toEqual(['f1'])
|
||||||
|
})
|
||||||
|
|
||||||
|
it('accepts new values of every kind after the reopen', async () => {
|
||||||
|
store.addEntity(BigInt(idMapper.getOrAssign('s1')), { category: 'electronics' })
|
||||||
|
store.addEntity(BigInt(idMapper.getOrAssign('n1')), { category: 5 })
|
||||||
|
await store.flush()
|
||||||
|
await store.close()
|
||||||
|
|
||||||
|
store = new ColumnStore({ flushThreshold: 10 })
|
||||||
|
await store.init(storage, idMapper)
|
||||||
|
|
||||||
|
store.addEntity(BigInt(idMapper.getOrAssign('s2')), { category: 'electronics' })
|
||||||
|
store.addEntity(BigInt(idMapper.getOrAssign('n2')), { category: 5 })
|
||||||
|
store.addEntity(BigInt(idMapper.getOrAssign('b1')), { category: false })
|
||||||
|
await store.flush()
|
||||||
|
|
||||||
|
expect(await uuidsOf('category', 'electronics')).toEqual(['s1', 's2'])
|
||||||
|
expect(await uuidsOf('category', 5)).toEqual(['n1', 'n2'])
|
||||||
|
expect(await uuidsOf('category', false)).toEqual(['b1'])
|
||||||
|
})
|
||||||
|
|
||||||
|
it('opens an index written by the pre-typed-postings shape and reads it unchanged', async () => {
|
||||||
|
// A single-kind field is byte-identical to what the old writer produced:
|
||||||
|
// one manifest at `_column_index/<field>/MANIFEST.json`, no kind
|
||||||
|
// subdirectory anywhere. That IS the old on-disk shape, so proving the
|
||||||
|
// new reader serves it proves an old index still opens.
|
||||||
|
store.addEntity(BigInt(idMapper.getOrAssign('a')), { status: 'active' })
|
||||||
|
store.addEntity(BigInt(idMapper.getOrAssign('b')), { status: 'archived' })
|
||||||
|
await store.flush()
|
||||||
|
|
||||||
|
const keys = await (storage as unknown as {
|
||||||
|
listObjectsUnderPath: (prefix: string) => Promise<string[]>
|
||||||
|
}).listObjectsUnderPath('_column_index/')
|
||||||
|
expect(keys.some((k) => k.includes('/k/'))).toBe(false)
|
||||||
|
|
||||||
|
await store.close()
|
||||||
|
store = new ColumnStore({ flushThreshold: 10 })
|
||||||
|
await store.init(storage, idMapper)
|
||||||
|
|
||||||
|
expect(store.getFieldKinds('status')).toEqual(['string'])
|
||||||
|
expect(await uuidsOf('status', 'active')).toEqual(['a'])
|
||||||
|
})
|
||||||
|
})
|
||||||
|
})
|
||||||
Reference in a new issue