Compare commits
1 commit
main
...
fix/metada
| Author | SHA1 | Date | |
|---|---|---|---|
| a128f0eda5 |
7 changed files with 1025 additions and 133 deletions
|
|
@ -217,6 +217,40 @@ membership queries at scale:
|
|||
`__words__` for tokenized text…).
|
||||
- `_blobs/_column_index/{field}/L0-NNNNNN.bin` — the actual level-0 run
|
||||
segments, stored through the shared `_blobs/<key>.bin` binary convention.
|
||||
- `_column_index/{field}/k/{kind}/…` — the same two files again, for a
|
||||
**second value kind** on the same field (see below). Absent for a field that
|
||||
holds one kind, which is nearly all of them.
|
||||
|
||||
### One posting column per (field, kind)
|
||||
|
||||
A field is not obliged to hold one type of value. `category` may carry
|
||||
`'electronics'` on some rows and `5` on others, and both are real values of
|
||||
that field. A segment, though, has one encoding — i64, f64, UTF-8, or boolean
|
||||
— so a field that holds several kinds gets **one column per kind**:
|
||||
|
||||
- The first kind a field ever sees owns the plain `_column_index/{field}/`
|
||||
layout above. A single-kind field is therefore byte-identical to what earlier
|
||||
versions wrote, and an index written before typed postings opens unchanged.
|
||||
- Every later kind gets its own column beside it at
|
||||
`_column_index/{field}/k/{kind}/`, where `{kind}` is `number`, `string` or
|
||||
`boolean`.
|
||||
|
||||
What that buys at query time:
|
||||
|
||||
| | |
|
||||
|---|---|
|
||||
| **Equality** | Answered from the column matching the **query value's own kind**. `where {category: 5}` reads the number postings; `where {category: '5'}` reads the string postings. Neither borrows the other's rows — a row written with the number `5` is not a row whose category is the text `'5'`. |
|
||||
| **A kind the field never held** | Matches nothing. That is the true answer, not a coerced one. |
|
||||
| **Ranges** | Routed by the kind of the bounds: numeric bounds read the numeric postings and ignore the field's strings. An **unbounded** range is the "has any value here" probe behind `exists`, and reads every kind. |
|
||||
| **`orderBy`** | A number and a string have no order between them, so a mixed field orders by kind first (number, string, boolean) and by value within a kind. A single-kind field sorts exactly as it always did. |
|
||||
| **Numbers** | One kind, one column: an integer column is written as i64 and widens to f64 the first time a non-integer arrives, so `4.5` is stored as itself rather than rounded. |
|
||||
|
||||
`null` and `undefined` are not kinds and are never posted; their absence is
|
||||
what the `exists` / `missing` operators read.
|
||||
|
||||
Older readers are unaffected by the additional columns: they see the field's
|
||||
primary column exactly where it has always been, and a `k/{kind}` directory is
|
||||
simply a name they never query.
|
||||
|
||||
Sparse per-field indexes, roaring-bitmap chunks, and zone-map/bloom segments
|
||||
additionally live as bucketed keys under `_system/idx/` (see §3). Which path
|
||||
|
|
|
|||
|
|
@ -23,7 +23,10 @@ import type { ColumnStoreProvider, SegmentMeta } from './types.js'
|
|||
import {
|
||||
ValueType,
|
||||
DEFAULT_FLUSH_THRESHOLD,
|
||||
FLAG_MULTI_VALUE
|
||||
FLAG_MULTI_VALUE,
|
||||
POSTING_KINDS,
|
||||
KIND_PATH_SEGMENT,
|
||||
type PostingKind
|
||||
} from './types.js'
|
||||
import { ColumnTailBuffer } from './ColumnTailBuffer.js'
|
||||
import { ColumnManifest } from './ColumnManifest.js'
|
||||
|
|
@ -52,10 +55,89 @@ interface HeapEntry {
|
|||
value: number | string
|
||||
entityIntId: number
|
||||
cursorIndex: number
|
||||
/**
|
||||
* Rank of the posting kind this entry came from, from {@link POSTING_KINDS}.
|
||||
* A mixed-kind field has no natural total order, so the merge orders by kind
|
||||
* first and by value within a kind.
|
||||
*/
|
||||
kindRank: number
|
||||
/** Iterator for the cursor — call next() to advance */
|
||||
iterator: Generator<CursorEntry>
|
||||
}
|
||||
|
||||
/**
|
||||
* One physical posting column: a (field, kind) pair and the key every internal
|
||||
* map and every storage path uses for it.
|
||||
*/
|
||||
interface KindColumn {
|
||||
/** The field as the query language names it. */
|
||||
field: string
|
||||
/** The kind of value this column holds. */
|
||||
kind: PostingKind
|
||||
/**
|
||||
* Internal map / storage key. The field's PRIMARY kind uses the bare field
|
||||
* name — the historical layout — and every other kind uses
|
||||
* `<field>/<KIND_PATH_SEGMENT>/<kind>`.
|
||||
*/
|
||||
key: string
|
||||
}
|
||||
|
||||
/**
|
||||
* The KIND a value indexes under — its JavaScript `typeof` class, not its
|
||||
* storage encoding.
|
||||
*
|
||||
* Anything that is not a number, string or boolean indexes as a string, which
|
||||
* is the `String(value)` treatment those values already received. `null` and
|
||||
* `undefined` never reach here: `addEntity` skips them, and their absence is
|
||||
* what the `exists` / `missing` operators read.
|
||||
*
|
||||
* @param value - The value about to be indexed or queried
|
||||
* @returns The posting kind that owns this value
|
||||
*/
|
||||
function kindOfValue(value: unknown): PostingKind {
|
||||
const t = typeof value
|
||||
if (t === 'number') return 'number'
|
||||
if (t === 'boolean') return 'boolean'
|
||||
return 'string'
|
||||
}
|
||||
|
||||
/**
|
||||
* The segment encoding a fresh column of this kind starts with.
|
||||
*
|
||||
* Only the number kind has a choice: an integer column starts as i64 and
|
||||
* widens to f64 the first time a non-integer arrives
|
||||
* ({@link ColumnTailBuffer.promoteToFloat}).
|
||||
*/
|
||||
function initialValueTypeFor(kind: PostingKind, firstValue: unknown): ValueType {
|
||||
switch (kind) {
|
||||
case 'boolean':
|
||||
return ValueType.Boolean
|
||||
case 'string':
|
||||
return ValueType.String
|
||||
case 'number':
|
||||
return Number.isInteger(firstValue) ? ValueType.Number : ValueType.Float
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* The kind a column of this encoding holds — the inverse of
|
||||
* {@link initialValueTypeFor}, used to read a kind back off a manifest written
|
||||
* before typed postings existed.
|
||||
*/
|
||||
function kindOfValueType(valueType: ValueType): PostingKind {
|
||||
switch (valueType) {
|
||||
case ValueType.Boolean:
|
||||
return 'boolean'
|
||||
case ValueType.String:
|
||||
return 'string'
|
||||
case ValueType.Number:
|
||||
case ValueType.Float:
|
||||
return 'number'
|
||||
default:
|
||||
throw new Error(`Unknown ValueType: ${valueType}`)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Unified column store coordinator.
|
||||
*
|
||||
|
|
@ -121,9 +203,19 @@ export class ColumnStore implements ColumnStoreProvider {
|
|||
*/
|
||||
private deletedEntities: Map<string, RoaringBitmap32> = new Map()
|
||||
|
||||
/** Known field value types (inferred from first write). */
|
||||
/** Segment encoding per COLUMN key (not per field — a field has one per kind). */
|
||||
private fieldTypes: Map<string, ValueType> = new Map()
|
||||
|
||||
/**
|
||||
* Every posting column a field owns: field → kind → column key.
|
||||
*
|
||||
* This is the map that ends the first-writer type freeze. A field's first
|
||||
* kind takes the bare field name as its column key, keeping the historical
|
||||
* on-disk layout; each later kind takes its own column beside it. Nothing is
|
||||
* coerced across kinds and nothing is dropped for being the wrong type.
|
||||
*/
|
||||
private fieldColumns: Map<string, Map<PostingKind, string>> = new Map()
|
||||
|
||||
/** Whether init() has completed. */
|
||||
private initialized = false
|
||||
|
||||
|
|
@ -140,6 +232,128 @@ export class ColumnStore implements ColumnStoreProvider {
|
|||
this.l0CompactionTrigger = config?.l0CompactionTrigger ?? 4
|
||||
}
|
||||
|
||||
// =========================================================================
|
||||
// Posting columns: (field, kind) → one physical column
|
||||
// =========================================================================
|
||||
|
||||
/**
|
||||
* Storage / map key for a (field, kind) column.
|
||||
*
|
||||
* `primary` is the kind that owns the bare field name. It is whichever kind
|
||||
* the field saw first, which for an index written before typed postings is
|
||||
* simply the kind of its single manifest — so the historical layout is
|
||||
* preserved rather than migrated.
|
||||
*/
|
||||
private static columnKeyFor(field: string, kind: PostingKind, primary: PostingKind | null): string {
|
||||
return primary === null || kind === primary
|
||||
? field
|
||||
: `${field}/${KIND_PATH_SEGMENT}/${kind}`
|
||||
}
|
||||
|
||||
/**
|
||||
* Split a discovered manifest path back into its (field, kind) column, or
|
||||
* `null` when the path names a field's primary column rather than a kind
|
||||
* column. `<field>/k/<kind>` is the only shape that reads as a kind column,
|
||||
* and only for a `<kind>` this version knows.
|
||||
*/
|
||||
private static parseKindColumnKey(key: string): { field: string; kind: PostingKind } | null {
|
||||
const marker = `/${KIND_PATH_SEGMENT}/`
|
||||
const at = key.lastIndexOf(marker)
|
||||
if (at <= 0) return null
|
||||
const kind = key.slice(at + marker.length)
|
||||
if (!POSTING_KINDS.includes(kind as PostingKind)) return null
|
||||
return { field: key.slice(0, at), kind: kind as PostingKind }
|
||||
}
|
||||
|
||||
/** Record a discovered or freshly created column against its field. */
|
||||
private registerColumn(field: string, kind: PostingKind, key: string): void {
|
||||
let byKind = this.fieldColumns.get(field)
|
||||
if (!byKind) {
|
||||
byKind = new Map()
|
||||
this.fieldColumns.set(field, byKind)
|
||||
}
|
||||
const existing = byKind.get(kind)
|
||||
if (existing !== undefined && existing !== key) {
|
||||
// Two columns claiming one (field, kind) means the layout on disk is not
|
||||
// one this writer could have produced. Serving it would silently answer
|
||||
// from half the postings, so say which two and stop.
|
||||
throw new Error(
|
||||
`ColumnStore: field '${field}' has two '${kind}' posting columns on ` +
|
||||
`disk ('${existing}' and '${key}'). The column index layout is ` +
|
||||
`inconsistent — rebuild/repair the metadata index rather than ` +
|
||||
`serving from one half of it.`
|
||||
)
|
||||
}
|
||||
byKind.set(kind, key)
|
||||
}
|
||||
|
||||
/** The column key for this (field, kind), or `null` if the field has no such kind. */
|
||||
private columnKey(field: string, kind: PostingKind): string | null {
|
||||
return this.fieldColumns.get(field)?.get(kind) ?? null
|
||||
}
|
||||
|
||||
/**
|
||||
* The column key for this (field, kind), creating the registration if the
|
||||
* field has not seen this kind before. Write path only.
|
||||
*/
|
||||
private ensureColumnKey(field: string, kind: PostingKind): string {
|
||||
const byKind = this.fieldColumns.get(field)
|
||||
const existing = byKind?.get(kind)
|
||||
if (existing !== undefined) return existing
|
||||
|
||||
// The primary kind is the one already holding the bare field name, if any.
|
||||
let primary: PostingKind | null = null
|
||||
if (byKind) {
|
||||
for (const [k, key] of byKind) {
|
||||
if (key === field) { primary = k; break }
|
||||
}
|
||||
}
|
||||
const key = ColumnStore.columnKeyFor(field, kind, primary)
|
||||
this.registerColumn(field, kind, key)
|
||||
return key
|
||||
}
|
||||
|
||||
/**
|
||||
* Every posting column this field owns, in {@link POSTING_KINDS} order.
|
||||
*
|
||||
* Read doors that are not about one particular value — an unbounded range
|
||||
* used as an "any value present" probe, distinct values, sorting — fan out
|
||||
* over all of them.
|
||||
*/
|
||||
private columnsForField(field: string): KindColumn[] {
|
||||
const byKind = this.fieldColumns.get(field)
|
||||
if (!byKind) return []
|
||||
const out: KindColumn[] = []
|
||||
for (const kind of POSTING_KINDS) {
|
||||
const key = byKind.get(kind)
|
||||
if (key !== undefined) out.push({ field, kind, key })
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
/**
|
||||
* Which value kinds this field actually holds, in {@link POSTING_KINDS}
|
||||
* order — the honest answer to "what type is this field?".
|
||||
*
|
||||
* A field that carries both `'electronics'` and `5` reports
|
||||
* `['number', 'string']`, not whichever of them was written first.
|
||||
*
|
||||
* @param field - Field name
|
||||
* @returns Every kind with at least one posting, or `[]` for an unknown field
|
||||
*/
|
||||
getFieldKinds(field: string): PostingKind[] {
|
||||
return this.columnsForField(field)
|
||||
.filter((c) => this.columnHasData(c.key))
|
||||
.map((c) => c.kind)
|
||||
}
|
||||
|
||||
/** Does this physical column hold any postings (persisted or buffered)? */
|
||||
private columnHasData(key: string): boolean {
|
||||
const manifest = this.manifests.get(key)
|
||||
const buffer = this.tailBuffers.get(key)
|
||||
return (manifest !== undefined && !manifest.isEmpty()) || (buffer !== undefined && buffer.size > 0)
|
||||
}
|
||||
|
||||
/**
|
||||
* Initialize the column store: discover existing field manifests.
|
||||
*/
|
||||
|
|
@ -157,11 +371,23 @@ export class ColumnStore implements ColumnStoreProvider {
|
|||
}).listObjectsUnderPath(this.basePath + '/')
|
||||
for (const path of paths) {
|
||||
if (path.endsWith('/MANIFEST.json')) {
|
||||
const fieldName = path.replace(this.basePath + '/', '').replace('/MANIFEST.json', '')
|
||||
const manifest = new ColumnManifest(fieldName, this.basePath)
|
||||
// The discovered name is a COLUMN key: either a bare field (that
|
||||
// field's primary kind, which is every column an index written
|
||||
// before typed postings has) or `<field>/k/<kind>` for a second
|
||||
// kind that arrived on a field later.
|
||||
const columnKey = path.replace(this.basePath + '/', '').replace('/MANIFEST.json', '')
|
||||
const manifest = new ColumnManifest(columnKey, this.basePath)
|
||||
await manifest.load(storage)
|
||||
this.manifests.set(fieldName, manifest)
|
||||
this.fieldTypes.set(fieldName, manifest.valueType)
|
||||
this.manifests.set(columnKey, manifest)
|
||||
this.fieldTypes.set(columnKey, manifest.valueType)
|
||||
|
||||
const parsed = ColumnStore.parseKindColumnKey(columnKey)
|
||||
if (parsed) {
|
||||
this.registerColumn(parsed.field, parsed.kind, columnKey)
|
||||
} else {
|
||||
this.registerColumn(columnKey, kindOfValueType(manifest.valueType), columnKey)
|
||||
}
|
||||
const fieldName = columnKey
|
||||
|
||||
// Load global deleted bitmap if it exists. Raw blob preferred
|
||||
// (2.4.0 #4 cortex-shared format); legacy envelope fallback for
|
||||
|
|
@ -264,26 +490,43 @@ export class ColumnStore implements ColumnStoreProvider {
|
|||
/**
|
||||
* Point filter: find entities where field equals value.
|
||||
*
|
||||
* Searches all segments + tail buffer, returns union as roaring bitmap.
|
||||
* Excludes globally deleted entities.
|
||||
* The QUERY VALUE'S OWN KIND picks the posting column, and only that column
|
||||
* is read. `where {category: 5}` answers from the number postings and
|
||||
* `where {category: '5'}` from the string postings — neither borrows the
|
||||
* other's rows, because a row written with the number `5` is not a row whose
|
||||
* category is the text `'5'`.
|
||||
*
|
||||
* A field that has never seen this kind matches nothing, which is the true
|
||||
* answer rather than a coerced one.
|
||||
*
|
||||
* Searches all segments + tail buffer of that column, returns the union as a
|
||||
* roaring bitmap. Excludes globally deleted entities.
|
||||
*/
|
||||
async filter(field: string, value: unknown): Promise<RoaringBitmap32> {
|
||||
const result = new RoaringBitmap32()
|
||||
const deleted = this.deletedEntities.get(field)
|
||||
const columnKey = this.columnKey(field, kindOfValue(value))
|
||||
if (columnKey === null) return result
|
||||
|
||||
// The query value takes the column's encoding — a boolean queried against
|
||||
// a boolean column has to become the 1/0 the column stores.
|
||||
const encoded = this.normalizeValue(value, this.fieldTypes.get(columnKey) ?? ValueType.String)
|
||||
if (encoded === undefined) return result
|
||||
|
||||
const deleted = this.deletedEntities.get(columnKey)
|
||||
|
||||
// Search segments
|
||||
const cursors = await this.getSegmentCursors(field)
|
||||
const cursors = await this.getSegmentCursors(columnKey)
|
||||
for (const cursor of cursors) {
|
||||
const ids = cursor.getEntityIdsForValue(value as number | string)
|
||||
const ids = cursor.getEntityIdsForValue(encoded)
|
||||
for (const id of ids) {
|
||||
if (!deleted || !deleted.has(id)) result.add(id)
|
||||
}
|
||||
}
|
||||
|
||||
// Search tail buffer
|
||||
const tailCursor = this.getTailBufferCursor(field)
|
||||
const tailCursor = this.getTailBufferCursor(columnKey)
|
||||
if (tailCursor) {
|
||||
const ids = tailCursor.getEntityIdsForValue(value as number | string)
|
||||
const ids = tailCursor.getEntityIdsForValue(encoded)
|
||||
for (const id of ids) {
|
||||
if (!deleted || !deleted.has(id)) result.add(id)
|
||||
}
|
||||
|
|
@ -324,22 +567,26 @@ export class ColumnStore implements ColumnStoreProvider {
|
|||
const out = new Map<number, number | string>()
|
||||
if (wanted.size === 0 || !this.hasField(field)) return out
|
||||
|
||||
const deleted = this.deletedEntities.get(field)
|
||||
const take = (entry: { value: number | string; entityIntId: number }): void => {
|
||||
if (!wanted.has(entry.entityIntId)) return
|
||||
if (deleted && deleted.has(entry.entityIntId)) return
|
||||
out.set(entry.entityIntId, entry.value)
|
||||
}
|
||||
// Every kind the field holds is read, in POSTING_KINDS order — a value an
|
||||
// entity wrote as a string is still that entity's value for this field.
|
||||
for (const column of this.columnsForField(field)) {
|
||||
const deleted = this.deletedEntities.get(column.key)
|
||||
const take = (entry: { value: number | string; entityIntId: number }): void => {
|
||||
if (!wanted.has(entry.entityIntId)) return
|
||||
if (deleted && deleted.has(entry.entityIntId)) return
|
||||
out.set(entry.entityIntId, entry.value)
|
||||
}
|
||||
|
||||
// Segments oldest -> newest, then the tail: a later write overwrites an
|
||||
// earlier one for the same id.
|
||||
const cursors = await this.getSegmentCursors(field)
|
||||
for (const cursor of cursors) {
|
||||
for (const entry of cursor.iterateForward()) take(entry)
|
||||
}
|
||||
const tailCursor = this.getTailBufferCursor(field)
|
||||
if (tailCursor) {
|
||||
for (const entry of tailCursor.iterateForward()) take(entry)
|
||||
// Segments oldest -> newest, then the tail: a later write overwrites an
|
||||
// earlier one for the same id.
|
||||
const cursors = await this.getSegmentCursors(column.key)
|
||||
for (const cursor of cursors) {
|
||||
for (const entry of cursor.iterateForward()) take(entry)
|
||||
}
|
||||
const tailCursor = this.getTailBufferCursor(column.key)
|
||||
if (tailCursor) {
|
||||
for (const entry of tailCursor.iterateForward()) take(entry)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
|
@ -363,41 +610,59 @@ export class ColumnStore implements ColumnStoreProvider {
|
|||
includeMax: boolean = true
|
||||
): Promise<RoaringBitmap32> {
|
||||
const result = new RoaringBitmap32()
|
||||
const cursors = await this.getSegmentCursors(field)
|
||||
|
||||
const hasMin = min !== undefined && min !== null
|
||||
const hasMax = max !== undefined && max !== null
|
||||
|
||||
for (const cursor of cursors) {
|
||||
const lo = hasMin ? min as number | string : cursor.minValue
|
||||
const hi = hasMax ? max as number | string : cursor.maxValue
|
||||
if (lo === undefined || hi === undefined) continue
|
||||
// Exclusivity applies only to an explicitly provided bound. A bound taken
|
||||
// from the segment's own min/max is a real stored value and must stay
|
||||
// inclusive, or the segment's boundary entities would be wrongly dropped.
|
||||
const ids = cursor.getEntityIdsInRange(
|
||||
lo,
|
||||
hi,
|
||||
hasMin ? includeMin : true,
|
||||
hasMax ? includeMax : true
|
||||
)
|
||||
for (const id of ids) result.add(id)
|
||||
}
|
||||
// The BOUNDS pick the column: numeric bounds read the numeric postings,
|
||||
// string bounds the string postings. An unbounded call is not a range at
|
||||
// all — it is the "has any value here" probe behind `exists` — so it fans
|
||||
// out over every kind the field holds.
|
||||
const columns: KindColumn[] = hasMin
|
||||
? this.columnsForKind(field, kindOfValue(min))
|
||||
: hasMax
|
||||
? this.columnsForKind(field, kindOfValue(max))
|
||||
: this.columnsForField(field)
|
||||
|
||||
// Tail buffer range: linear scan (tail is small)
|
||||
const tailCursor = this.getTailBufferCursor(field)
|
||||
if (tailCursor) {
|
||||
for (const entry of tailCursor.iterateForward()) {
|
||||
const v = entry.value as any
|
||||
const loOk = !hasMin || (includeMin ? v >= (min as any) : v > (min as any))
|
||||
const hiOk = !hasMax || (includeMax ? v <= (max as any) : v < (max as any))
|
||||
if (loOk && hiOk) result.add(entry.entityIntId)
|
||||
for (const column of columns) {
|
||||
const cursors = await this.getSegmentCursors(column.key)
|
||||
for (const cursor of cursors) {
|
||||
const lo = hasMin ? min as number | string : cursor.minValue
|
||||
const hi = hasMax ? max as number | string : cursor.maxValue
|
||||
if (lo === undefined || hi === undefined) continue
|
||||
// Exclusivity applies only to an explicitly provided bound. A bound taken
|
||||
// from the segment's own min/max is a real stored value and must stay
|
||||
// inclusive, or the segment's boundary entities would be wrongly dropped.
|
||||
const ids = cursor.getEntityIdsInRange(
|
||||
lo,
|
||||
hi,
|
||||
hasMin ? includeMin : true,
|
||||
hasMax ? includeMax : true
|
||||
)
|
||||
for (const id of ids) result.add(id)
|
||||
}
|
||||
|
||||
// Tail buffer range: linear scan (tail is small)
|
||||
const tailCursor = this.getTailBufferCursor(column.key)
|
||||
if (tailCursor) {
|
||||
for (const entry of tailCursor.iterateForward()) {
|
||||
const v = entry.value as any
|
||||
const loOk = !hasMin || (includeMin ? v >= (min as any) : v > (min as any))
|
||||
const hiOk = !hasMax || (includeMax ? v <= (max as any) : v < (max as any))
|
||||
if (loOk && hiOk) result.add(entry.entityIntId)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return result
|
||||
}
|
||||
|
||||
/** The single column for this (field, kind), as a list, or empty if absent. */
|
||||
private columnsForKind(field: string, kind: PostingKind): KindColumn[] {
|
||||
const key = this.columnKey(field, kind)
|
||||
return key === null ? [] : [{ field, kind, key }]
|
||||
}
|
||||
|
||||
/**
|
||||
* Sort top-K: return K entity int IDs in sorted order (u64-safe BigInt).
|
||||
*
|
||||
|
|
@ -428,18 +693,21 @@ export class ColumnStore implements ColumnStoreProvider {
|
|||
*/
|
||||
async getFilterValues(field: string): Promise<string[]> {
|
||||
const valueSet = new Set<string>()
|
||||
const cursors = await this.getSegmentCursors(field)
|
||||
|
||||
for (const cursor of cursors) {
|
||||
for (const entry of cursor.iterateForward()) {
|
||||
valueSet.add(String(entry.value))
|
||||
for (const column of this.columnsForField(field)) {
|
||||
const cursors = await this.getSegmentCursors(column.key)
|
||||
|
||||
for (const cursor of cursors) {
|
||||
for (const entry of cursor.iterateForward()) {
|
||||
valueSet.add(String(entry.value))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const tailCursor = this.getTailBufferCursor(field)
|
||||
if (tailCursor) {
|
||||
for (const entry of tailCursor.iterateForward()) {
|
||||
valueSet.add(String(entry.value))
|
||||
const tailCursor = this.getTailBufferCursor(column.key)
|
||||
if (tailCursor) {
|
||||
for (const entry of tailCursor.iterateForward()) {
|
||||
valueSet.add(String(entry.value))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -450,9 +718,7 @@ export class ColumnStore implements ColumnStoreProvider {
|
|||
* Check if a field has any indexed data.
|
||||
*/
|
||||
hasField(field: string): boolean {
|
||||
const manifest = this.manifests.get(field)
|
||||
const buffer = this.tailBuffers.get(field)
|
||||
return (manifest !== undefined && !manifest.isEmpty()) || (buffer !== undefined && buffer.size > 0)
|
||||
return this.columnsForField(field).some((c) => this.columnHasData(c.key))
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -462,12 +728,11 @@ export class ColumnStore implements ColumnStoreProvider {
|
|||
* store will actually serve queries from.
|
||||
*/
|
||||
getIndexedFields(): string[] {
|
||||
// Names FIELDS, not columns: a field carrying two kinds is one name here,
|
||||
// the same name a caller queries with.
|
||||
const fields = new Set<string>()
|
||||
for (const [field, manifest] of this.manifests) {
|
||||
if (!manifest.isEmpty()) fields.add(field)
|
||||
}
|
||||
for (const [field, buffer] of this.tailBuffers) {
|
||||
if (buffer.size > 0) fields.add(field)
|
||||
for (const [field] of this.fieldColumns) {
|
||||
if (this.hasField(field)) fields.add(field)
|
||||
}
|
||||
return Array.from(fields).sort()
|
||||
}
|
||||
|
|
@ -482,12 +747,16 @@ export class ColumnStore implements ColumnStoreProvider {
|
|||
getFieldSizeSummary(): Array<{ field: string; segmentCount: number; tailSize: number }> {
|
||||
const summary: Array<{ field: string; segmentCount: number; tailSize: number }> = []
|
||||
for (const field of this.getIndexedFields()) {
|
||||
const manifest = this.manifests.get(field)
|
||||
const buffer = this.tailBuffers.get(field)
|
||||
const segmentCount = manifest && !manifest.isEmpty()
|
||||
? manifest.getAllSegments().length
|
||||
: 0
|
||||
const tailSize = buffer ? buffer.size : 0
|
||||
// Summed across the field's kind columns — the caller asked about a
|
||||
// field, and a field's size is all of the postings under its name.
|
||||
let segmentCount = 0
|
||||
let tailSize = 0
|
||||
for (const column of this.columnsForField(field)) {
|
||||
const manifest = this.manifests.get(column.key)
|
||||
const buffer = this.tailBuffers.get(column.key)
|
||||
if (manifest && !manifest.isEmpty()) segmentCount += manifest.getAllSegments().length
|
||||
if (buffer) tailSize += buffer.size
|
||||
}
|
||||
summary.push({ field, segmentCount, tailSize })
|
||||
}
|
||||
return summary
|
||||
|
|
@ -515,6 +784,8 @@ export class ColumnStore implements ColumnStoreProvider {
|
|||
this.segmentCache.clear()
|
||||
this.manifests.clear()
|
||||
this.deletedEntities.clear()
|
||||
this.fieldColumns.clear()
|
||||
this.fieldTypes.clear()
|
||||
this.initialized = false
|
||||
}
|
||||
|
||||
|
|
@ -523,32 +794,64 @@ export class ColumnStore implements ColumnStoreProvider {
|
|||
// =========================================================================
|
||||
|
||||
/**
|
||||
* Push a single value to a field's tail buffer.
|
||||
* Creates the buffer and manifest if first write to this field.
|
||||
* Infers ValueType from the first value seen.
|
||||
* Push a single value to the posting column for its (field, KIND).
|
||||
*
|
||||
* The value's own kind picks the column — a string goes to the field's
|
||||
* string postings, a number to its number postings — so a field carrying
|
||||
* `'electronics'` and `5` keeps both, each answerable by an equality filter
|
||||
* of its own kind. Under the first-writer type freeze this method replaced,
|
||||
* the first value's type became the field's type and every later value of
|
||||
* another kind was coerced to it or, when coercion failed, dropped with no
|
||||
* error at all.
|
||||
*
|
||||
* Creates the column's buffer and manifest on its first value.
|
||||
*/
|
||||
private pushToBuffer(field: string, value: unknown, entityIntId: number, isMultiValue: boolean): void {
|
||||
let buffer = this.tailBuffers.get(field)
|
||||
const kind = kindOfValue(value)
|
||||
const columnKey = this.ensureColumnKey(field, kind)
|
||||
|
||||
let buffer = this.tailBuffers.get(columnKey)
|
||||
if (!buffer) {
|
||||
const valueType = this.inferValueType(value)
|
||||
buffer = new ColumnTailBuffer(field, valueType, this.flushThreshold)
|
||||
this.tailBuffers.set(field, buffer)
|
||||
this.fieldTypes.set(field, valueType)
|
||||
// A reopened column takes its encoding from its manifest — an integer
|
||||
// column that widened to f64 in an earlier session stays widened.
|
||||
const valueType =
|
||||
this.manifests.get(columnKey)?.valueType ?? initialValueTypeFor(kind, value)
|
||||
buffer = new ColumnTailBuffer(columnKey, valueType, this.flushThreshold)
|
||||
this.tailBuffers.set(columnKey, buffer)
|
||||
this.fieldTypes.set(columnKey, valueType)
|
||||
|
||||
// Ensure manifest exists
|
||||
if (!this.manifests.has(field)) {
|
||||
const manifest = new ColumnManifest(field, this.basePath)
|
||||
if (!this.manifests.has(columnKey)) {
|
||||
const manifest = new ColumnManifest(columnKey, this.basePath)
|
||||
manifest.valueType = valueType
|
||||
manifest.multiValue = isMultiValue
|
||||
this.manifests.set(field, manifest)
|
||||
this.manifests.set(columnKey, manifest)
|
||||
}
|
||||
}
|
||||
|
||||
// Normalize value to the column type
|
||||
const normalizedValue = this.normalizeValue(value, buffer.valueType)
|
||||
if (normalizedValue !== undefined) {
|
||||
buffer.add(normalizedValue, entityIntId)
|
||||
// An integer column widens the first time a non-integer number arrives, so
|
||||
// the value is stored as itself instead of rounded to the nearest integer.
|
||||
if (kind === 'number' && buffer.valueType === ValueType.Number && !Number.isInteger(value)) {
|
||||
buffer.promoteToFloat()
|
||||
this.fieldTypes.set(columnKey, ValueType.Float)
|
||||
const manifest = this.manifests.get(columnKey)
|
||||
if (manifest) manifest.valueType = ValueType.Float
|
||||
}
|
||||
|
||||
const normalizedValue = this.normalizeValue(value, buffer.valueType)
|
||||
if (normalizedValue === undefined) {
|
||||
// Unreachable by construction: the column was chosen BY this value's
|
||||
// kind, so the encoding always accepts it. Reaching here would mean a
|
||||
// value had been silently dropped from the index — the exact failure
|
||||
// typed postings exist to end — so it is an error, never a skip.
|
||||
throw new Error(
|
||||
`ColumnStore: field '${field}' rejected a ${kind} value for its own ` +
|
||||
`${ValueType[buffer.valueType]} posting column. The value would have ` +
|
||||
`been dropped from the index while the row stayed readable by id — ` +
|
||||
`this is a kind-routing bug, not a value the caller may ignore.`
|
||||
)
|
||||
}
|
||||
buffer.add(normalizedValue, entityIntId)
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -677,8 +980,15 @@ export class ColumnStore implements ColumnStoreProvider {
|
|||
/** Torn-segment quarantine entries for a field (observability + heal input). */
|
||||
quarantinedSegments(field: string): Array<{ segment: string; error: string; hits: number }> {
|
||||
const out: Array<{ segment: string; error: string; hits: number }> = []
|
||||
for (const [key, q] of this.segmentQuarantine) {
|
||||
if (key.startsWith(`${field}:`)) out.push({ segment: key.slice(field.length + 1), error: q.error, hits: q.hits })
|
||||
// Across every kind column of the field — a torn segment in the string
|
||||
// postings is this field's torn segment as much as one in the numbers.
|
||||
for (const column of this.columnsForField(field)) {
|
||||
const prefix = `${column.key}:`
|
||||
for (const [key, q] of this.segmentQuarantine) {
|
||||
if (key.startsWith(prefix)) {
|
||||
out.push({ segment: key.slice(prefix.length), error: q.error, hits: q.hits })
|
||||
}
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
|
@ -850,17 +1160,22 @@ export class ColumnStore implements ColumnStoreProvider {
|
|||
k: number,
|
||||
filterBitmap: RoaringBitmap32 | null
|
||||
): Promise<number[]> {
|
||||
// Collect all cursors (segments + tail buffer)
|
||||
const segCursors = await this.getSegmentCursors(field)
|
||||
const tailCursor = this.getTailBufferCursor(field)
|
||||
|
||||
// Create iterators for each cursor in the specified direction
|
||||
// Collect cursors across EVERY kind the field holds. A single-kind field —
|
||||
// nearly all of them — merges exactly the cursors it always did.
|
||||
const iterators: Generator<CursorEntry>[] = []
|
||||
for (const cursor of segCursors) {
|
||||
iterators.push(order === 'asc' ? cursor.iterateForward() : cursor.iterateBackward())
|
||||
}
|
||||
if (tailCursor) {
|
||||
iterators.push(order === 'asc' ? tailCursor.iterateForward() : tailCursor.iterateBackward())
|
||||
const iteratorKindRank: number[] = []
|
||||
for (const column of this.columnsForField(field)) {
|
||||
const kindRank = POSTING_KINDS.indexOf(column.kind)
|
||||
const segCursors = await this.getSegmentCursors(column.key)
|
||||
for (const cursor of segCursors) {
|
||||
iterators.push(order === 'asc' ? cursor.iterateForward() : cursor.iterateBackward())
|
||||
iteratorKindRank.push(kindRank)
|
||||
}
|
||||
const tailCursor = this.getTailBufferCursor(column.key)
|
||||
if (tailCursor) {
|
||||
iterators.push(order === 'asc' ? tailCursor.iterateForward() : tailCursor.iterateBackward())
|
||||
iteratorKindRank.push(kindRank)
|
||||
}
|
||||
}
|
||||
|
||||
if (iterators.length === 0) return []
|
||||
|
|
@ -874,16 +1189,21 @@ export class ColumnStore implements ColumnStoreProvider {
|
|||
value: next.value.value,
|
||||
entityIntId: next.value.entityIntId,
|
||||
cursorIndex: i,
|
||||
kindRank: iteratorKindRank[i],
|
||||
iterator: iterators[i]
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// Heapify
|
||||
const isString = (this.fieldTypes.get(field) ?? ValueType.Number) === ValueType.String
|
||||
// Heapify. A number and a string have no ordering between them, so a
|
||||
// mixed-kind field orders by KIND first (POSTING_KINDS order) and by value
|
||||
// within a kind — one defined total order instead of a comparison whose
|
||||
// answer depends on which value happened to be on the left.
|
||||
const compare = (a: HeapEntry, b: HeapEntry): number => {
|
||||
let cmp: number
|
||||
if (isString) {
|
||||
if (a.kindRank !== b.kindRank) {
|
||||
cmp = a.kindRank - b.kindRank
|
||||
} else if (POSTING_KINDS[a.kindRank] === 'string') {
|
||||
cmp = compareCodePoints(String(a.value), String(b.value))
|
||||
} else {
|
||||
cmp = (a.value as number) - (b.value as number)
|
||||
|
|
@ -915,6 +1235,7 @@ export class ColumnStore implements ColumnStoreProvider {
|
|||
value: next.value.value,
|
||||
entityIntId: next.value.entityIntId,
|
||||
cursorIndex: top.cursorIndex,
|
||||
kindRank: top.kindRank,
|
||||
iterator: top.iterator
|
||||
}
|
||||
}
|
||||
|
|
@ -922,8 +1243,11 @@ export class ColumnStore implements ColumnStoreProvider {
|
|||
this.heapDown(heap, 0, compare)
|
||||
}
|
||||
|
||||
// Apply global deleted check, filter, and dedup
|
||||
const deleted = this.deletedEntities.get(field)
|
||||
// Apply global deleted check, filter, and dedup. The deleted bitmap is
|
||||
// per COLUMN, and the entry came from the column its kind names.
|
||||
const deleted = this.deletedEntities.get(
|
||||
this.columnKey(field, POSTING_KINDS[top.kindRank]) ?? field
|
||||
)
|
||||
if (deleted && deleted.has(top.entityIntId)) continue
|
||||
if (seen.has(top.entityIntId)) continue
|
||||
if (filterBitmap && !filterBitmap.has(top.entityIntId)) continue
|
||||
|
|
@ -965,35 +1289,31 @@ export class ColumnStore implements ColumnStoreProvider {
|
|||
}
|
||||
|
||||
/**
|
||||
* Infer ValueType from a JavaScript value.
|
||||
*/
|
||||
private inferValueType(value: unknown): ValueType {
|
||||
if (typeof value === 'boolean') return ValueType.Boolean
|
||||
if (typeof value === 'number') {
|
||||
return Number.isInteger(value) ? ValueType.Number : ValueType.Float
|
||||
}
|
||||
return ValueType.String
|
||||
}
|
||||
|
||||
/**
|
||||
* Normalize a JavaScript value to the column's ValueType.
|
||||
* Encode a value for the column its own kind selected.
|
||||
*
|
||||
* This does NOT convert between kinds. It used to: a string reaching a
|
||||
* numeric column was run through `Number(value)`, and a number reaching a
|
||||
* numeric column was run through `Math.round`, so `'electronics'` became
|
||||
* `NaN` and vanished while `4.5` became `5` and answered the wrong query.
|
||||
* Kind routing removes the need for either — the only work left is picking
|
||||
* the encoding the column already committed to.
|
||||
*
|
||||
* @returns The encoded value, or `undefined` if the value does not belong in
|
||||
* this column at all — which the caller treats as a routing bug and
|
||||
* raises, never as a value to skip.
|
||||
*/
|
||||
private normalizeValue(value: unknown, type: ValueType): number | string | undefined {
|
||||
switch (type) {
|
||||
case ValueType.Number:
|
||||
if (typeof value === 'number') return Math.round(value)
|
||||
if (typeof value === 'string') { const n = Number(value); return isNaN(n) ? undefined : Math.round(n) }
|
||||
if (typeof value === 'boolean') return value ? 1 : 0
|
||||
return undefined
|
||||
// Integer column. Non-integers widen it to Float before reaching here.
|
||||
return typeof value === 'number' && Number.isInteger(value) ? value : undefined
|
||||
case ValueType.Float:
|
||||
if (typeof value === 'number') return value
|
||||
if (typeof value === 'string') { const n = Number(value); return isNaN(n) ? undefined : n }
|
||||
return undefined
|
||||
return typeof value === 'number' ? value : undefined
|
||||
case ValueType.Boolean:
|
||||
if (typeof value === 'boolean') return value ? 1 : 0
|
||||
if (typeof value === 'number') return value ? 1 : 0
|
||||
return undefined
|
||||
return typeof value === 'boolean' ? (value ? 1 : 0) : undefined
|
||||
case ValueType.String:
|
||||
// The string kind is also where objects and bigints land, exactly as
|
||||
// they always did.
|
||||
return String(value)
|
||||
default:
|
||||
return undefined
|
||||
|
|
|
|||
|
|
@ -55,8 +55,12 @@ export class ColumnTailBuffer {
|
|||
/** Field name this buffer is for. */
|
||||
readonly fieldName: string
|
||||
|
||||
/** Value type determines sort comparator. */
|
||||
readonly valueType: ValueType
|
||||
/**
|
||||
* Value type determines sort comparator and segment encoding.
|
||||
*
|
||||
* Widened in place by {@link promoteToFloat} — never otherwise reassigned.
|
||||
*/
|
||||
valueType: ValueType
|
||||
|
||||
/** Flush threshold. */
|
||||
readonly threshold: number
|
||||
|
|
@ -81,6 +85,38 @@ export class ColumnTailBuffer {
|
|||
this.threshold = threshold
|
||||
}
|
||||
|
||||
/**
|
||||
* Widen an integer column to floating point, losslessly and in place.
|
||||
*
|
||||
* The number posting kind holds every JavaScript number, but a segment picks
|
||||
* ONE encoding: i64 for integers, f64 for the rest. A column that has only
|
||||
* ever seen integers is written as i64; the first non-integer to arrive
|
||||
* widens it here, so that value is stored as itself instead of being rounded
|
||||
* to the nearest integer with no error — the rounding that made `4.5` and
|
||||
* `5.5` both answer `where {score: 5}` and neither answer its own value.
|
||||
*
|
||||
* Widening is lossless in both directions it has to be: every value already
|
||||
* buffered is an integer, and every integer is exactly representable as f64.
|
||||
* Segments already on disk keep their own i64 encoding in their own headers
|
||||
* and keep decoding by it — only segments written from here on are f64.
|
||||
*
|
||||
* @throws Error if called on a column that is not an integer column — the
|
||||
* only legal widening is Number → Float, and any other request is a bug in
|
||||
* the caller's kind routing rather than something to absorb quietly.
|
||||
*/
|
||||
promoteToFloat(): void {
|
||||
if (this.valueType === ValueType.Float) return
|
||||
if (this.valueType !== ValueType.Number) {
|
||||
throw new Error(
|
||||
`ColumnTailBuffer '${this.fieldName}': cannot widen a ` +
|
||||
`${ValueType[this.valueType]} column to Float — only an integer ` +
|
||||
`(Number) column widens, and this call means a value reached the ` +
|
||||
`wrong kind's column`
|
||||
)
|
||||
}
|
||||
this.valueType = ValueType.Float
|
||||
}
|
||||
|
||||
/**
|
||||
* Add a (value, entityIntId) entry to the buffer.
|
||||
*
|
||||
|
|
|
|||
|
|
@ -58,6 +58,53 @@ export enum ValueType {
|
|||
Boolean = 3
|
||||
}
|
||||
|
||||
/**
|
||||
* The KIND of a value, as the query language sees it.
|
||||
*
|
||||
* A kind is a JavaScript `typeof` class, not a storage encoding: `5` and `5.5`
|
||||
* are one kind (`'number'`) held in one posting column, even though they need
|
||||
* different segment encodings (i64 vs f64 — see {@link ValueType}).
|
||||
*
|
||||
* A field holds ONE POSTING COLUMN PER KIND, so `category` may carry string
|
||||
* values and number values at the same time and answer equality on each. This
|
||||
* replaces the first-writer type freeze, under which the first value's type
|
||||
* became the field's type and every later value of another kind was coerced —
|
||||
* or, when coercion failed (`Number('electronics')`), dropped from the index
|
||||
* with no error: the row stayed readable by id and by vector but vanished from
|
||||
* every equality filter on that field.
|
||||
*
|
||||
* Kinds do not coerce into one another at query time either: `where {c: 5}`
|
||||
* matches rows written with the NUMBER `5`, and `where {c: '5'}` matches rows
|
||||
* written with the STRING `'5'`. Neither ever matches the other.
|
||||
*
|
||||
* Values that are none of these three (objects, bigints) index as strings —
|
||||
* the same `String(value)` treatment they received before.
|
||||
*/
|
||||
export type PostingKind = 'number' | 'string' | 'boolean'
|
||||
|
||||
/**
|
||||
* Every posting kind, in the order that defines cross-kind sort position.
|
||||
*
|
||||
* A mixed-kind field has no natural total order — a number does not compare
|
||||
* with a string — so `sortTopK` orders by KIND first (numbers, then strings,
|
||||
* then booleans) and by value within a kind. A single-kind field, which is
|
||||
* nearly every field, sorts exactly as it always did.
|
||||
*/
|
||||
export const POSTING_KINDS: readonly PostingKind[] = ['number', 'string', 'boolean']
|
||||
|
||||
/**
|
||||
* Path segment marking a field's NON-PRIMARY kind columns on disk.
|
||||
*
|
||||
* The first kind a field ever sees keeps the historical layout —
|
||||
* `<base>/<field>/MANIFEST.json` and `<base>/<field>/L0-NNNNNN` — so every
|
||||
* index written before typed postings opens unchanged, and the byte-for-byte
|
||||
* interchange with the native column store is untouched for the single-kind
|
||||
* fields that are nearly all of them. A second kind arriving on the same field
|
||||
* gets its own column at `<base>/<field>/k/<kind>/…` rather than overwriting or
|
||||
* being coerced into the first.
|
||||
*/
|
||||
export const KIND_PATH_SEGMENT = 'k'
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Segment header and footer
|
||||
// ---------------------------------------------------------------------------
|
||||
|
|
@ -267,6 +314,19 @@ export interface ColumnStoreProvider {
|
|||
*/
|
||||
hasField(field: string): boolean
|
||||
|
||||
/**
|
||||
* Which value KINDS this field actually holds, in {@link POSTING_KINDS}
|
||||
* order — the honest answer to "what type is this field?" for a field that
|
||||
* carries more than one.
|
||||
*
|
||||
* OPTIONAL so an implementation written against the pre-typed-postings
|
||||
* contract still satisfies this interface; feature-detect before calling.
|
||||
*
|
||||
* @param field - Field name
|
||||
* @returns Every kind with at least one posting, or `[]` for an unknown field
|
||||
*/
|
||||
getFieldKinds?(field: string): PostingKind[]
|
||||
|
||||
/**
|
||||
* Flush all in-memory tail buffers to L0 segments on disk.
|
||||
* Saves all manifests.
|
||||
|
|
|
|||
|
|
@ -55,8 +55,30 @@ export enum FieldType {
|
|||
*/
|
||||
export interface FieldTypeInfo {
|
||||
field: string
|
||||
/**
|
||||
* The DOMINANT reading of the field — one type, the most specific one every
|
||||
* sampled value satisfies.
|
||||
*
|
||||
* A field is not obliged to hold one kind, so this is not the whole answer
|
||||
* for a field that holds several. Read {@link kinds} beside it: a field
|
||||
* carrying `'electronics'` and `5` infers as STRING here and reports
|
||||
* `['number', 'string']` there, and the metadata index keeps a separate
|
||||
* posting column for each of them.
|
||||
*/
|
||||
inferredType: FieldType
|
||||
confidence: number // 0-1 confidence score
|
||||
/**
|
||||
* Every value KIND observed in the sample, in the order
|
||||
* number → string → boolean. More than one entry means a genuinely
|
||||
* mixed field, and every one of those kinds is independently filterable.
|
||||
*
|
||||
* Kinds are JavaScript `typeof` classes, one level coarser than
|
||||
* {@link FieldType}: a UUID and a category name are both `'string'`, and an
|
||||
* integer and a timestamp are both `'number'`.
|
||||
*
|
||||
* Optional only for cached analyses written before this was reported.
|
||||
*/
|
||||
kinds?: Array<'number' | 'string' | 'boolean'>
|
||||
sampleSize: number // Number of values analyzed
|
||||
lastUpdated: number // Timestamp of last analysis
|
||||
detectionMethod: 'value' // Always 'value' (no fallbacks!)
|
||||
|
|
@ -133,14 +155,71 @@ export class FieldTypeInference {
|
|||
}
|
||||
|
||||
/**
|
||||
* Analyze values to determine field type
|
||||
* Analyze values to determine field type, and report every KIND the field
|
||||
* actually holds alongside it.
|
||||
*
|
||||
* The classification below picks ONE type, because every one of its
|
||||
* heuristics asks `samples.every(...)`: a field carrying `'electronics'` and
|
||||
* `5` satisfies none of them and lands on STRING. That single answer is true
|
||||
* as far as it goes — string is the dominant reading — but on its own it
|
||||
* says nothing about the numbers also in the field, and a caller that treats
|
||||
* it as the field's only type reproduces the first-writer freeze the index
|
||||
* itself no longer has. {@link FieldTypeInfo.kinds} carries the rest.
|
||||
*/
|
||||
private async analyzeValues(field: string, values: any[]): Promise<FieldTypeInfo> {
|
||||
const info = await this.classifyValues(field, values)
|
||||
info.kinds = FieldTypeInference.observedKinds(values)
|
||||
if (info.kinds.length > 1 && info.metadata) {
|
||||
info.metadata.format = `${info.metadata.format} (field also holds: ${info.kinds
|
||||
.filter((k) => k !== FieldTypeInference.kindOfType(info.inferredType))
|
||||
.join(', ')})`
|
||||
}
|
||||
return info
|
||||
}
|
||||
|
||||
/**
|
||||
* The distinct value kinds present in a sample, in a stable order.
|
||||
*
|
||||
* Kinds are JavaScript `typeof` classes — the same classes the metadata
|
||||
* index keeps separate posting columns for — not the finer
|
||||
* {@link FieldType} readings, which are interpretations layered on top of
|
||||
* them (a UUID and a category name are both the `string` kind).
|
||||
*/
|
||||
private static observedKinds(values: any[]): Array<'number' | 'string' | 'boolean'> {
|
||||
const order: Array<'number' | 'string' | 'boolean'> = ['number', 'string', 'boolean']
|
||||
const seen = new Set<'number' | 'string' | 'boolean'>()
|
||||
for (const v of values) {
|
||||
if (v === null || v === undefined) continue
|
||||
const t = typeof v
|
||||
seen.add(t === 'number' ? 'number' : t === 'boolean' ? 'boolean' : 'string')
|
||||
}
|
||||
return order.filter((k) => seen.has(k))
|
||||
}
|
||||
|
||||
/** The value kind a {@link FieldType} reading is an interpretation of. */
|
||||
private static kindOfType(type: FieldType): 'number' | 'string' | 'boolean' {
|
||||
switch (type) {
|
||||
case FieldType.BOOLEAN:
|
||||
return 'boolean'
|
||||
case FieldType.INTEGER:
|
||||
case FieldType.FLOAT:
|
||||
case FieldType.TIMESTAMP_MS:
|
||||
case FieldType.TIMESTAMP_S:
|
||||
return 'number'
|
||||
default:
|
||||
return 'string'
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Classify values into a single field type.
|
||||
*
|
||||
* Uses DuckDB-inspired type detection order:
|
||||
* BOOLEAN → INTEGER → FLOAT → DATE → TIMESTAMP → UUID → STRING
|
||||
*
|
||||
* No fallbacks - pure value-based detection
|
||||
*/
|
||||
private async analyzeValues(field: string, values: any[]): Promise<FieldTypeInfo> {
|
||||
private async classifyValues(field: string, values: any[]): Promise<FieldTypeInfo> {
|
||||
// Filter null/undefined values
|
||||
const validValues = values.filter(v => v !== null && v !== undefined)
|
||||
|
||||
|
|
|
|||
122
tests/regression/metadata-field-typing.unit.test.ts
Normal file
122
tests/regression/metadata-field-typing.unit.test.ts
Normal file
|
|
@ -0,0 +1,122 @@
|
|||
/**
|
||||
* @module metadata-field-typing.unit.test
|
||||
* @description Regression: a metadata field that holds more than one value
|
||||
* KIND stays fully filterable on every kind it holds.
|
||||
*
|
||||
* The defect this pins, reproduced on the released engine: the metadata index
|
||||
* fixed a field's value type from the FIRST value it saw, and every later value
|
||||
* of a different type was coerced to that type or, when coercion failed,
|
||||
* dropped from the index in silence. Writing `category: 'electronics'` rows and
|
||||
* then `category: 5` rows left `find({ where: { category: 5 } })` returning
|
||||
* nothing — while the same rows in a numbers-only corpus answered correctly.
|
||||
* The rows themselves were never lost: they stayed readable by id and by vector
|
||||
* search, and only ever went missing from equality filters on that one field,
|
||||
* which is what made it so quiet.
|
||||
*
|
||||
* Order is the whole point of these cases. Neither writer owns the field, so
|
||||
* strings-then-numbers and numbers-then-strings must give the same answers.
|
||||
*/
|
||||
|
||||
import { describe, it, expect } from 'vitest'
|
||||
import { Brainy } from '../../src/brainy.js'
|
||||
import { NounType } from '../../src/types/graphTypes.js'
|
||||
|
||||
/** A brain over memory storage, with a corpus written in the given order. */
|
||||
async function brainWith(
|
||||
rows: Array<{ label: string; category: unknown }>
|
||||
): Promise<Brainy> {
|
||||
const brainy = new Brainy({ requireSubtype: false, storage: { type: 'memory' } })
|
||||
await brainy.init()
|
||||
for (const row of rows) {
|
||||
await brainy.add({
|
||||
data: `item ${row.label}`,
|
||||
type: NounType.Thing,
|
||||
metadata: { label: row.label, category: row.category }
|
||||
})
|
||||
}
|
||||
return brainy
|
||||
}
|
||||
|
||||
const labelsOf = (results: Array<{ metadata?: Record<string, unknown> }>): string[] =>
|
||||
results.map((r) => String(r.metadata?.label)).sort()
|
||||
|
||||
describe('regression: a mixed-kind metadata field filters on every kind', { timeout: 180_000 }, () => {
|
||||
it('finds number rows written after string rows', async () => {
|
||||
const brainy = await brainWith([
|
||||
{ label: 'e1', category: 'electronics' },
|
||||
{ label: 'f1', category: 'furniture' },
|
||||
{ label: 'n1', category: 5 },
|
||||
{ label: 'n2', category: 5 },
|
||||
{ label: 'n3', category: 7 }
|
||||
])
|
||||
try {
|
||||
expect(labelsOf(await brainy.find({ where: { category: 5 }, limit: 100 }))).toEqual(['n1', 'n2'])
|
||||
expect(labelsOf(await brainy.find({ where: { category: 7 }, limit: 100 }))).toEqual(['n3'])
|
||||
expect(labelsOf(await brainy.find({ where: { category: 'electronics' }, limit: 100 }))).toEqual(['e1'])
|
||||
expect(labelsOf(await brainy.find({ where: { category: 'furniture' }, limit: 100 }))).toEqual(['f1'])
|
||||
} finally {
|
||||
await brainy.close()
|
||||
}
|
||||
})
|
||||
|
||||
it('finds string rows written after number rows', async () => {
|
||||
const brainy = await brainWith([
|
||||
{ label: 'n1', category: 5 },
|
||||
{ label: 'n2', category: 5 },
|
||||
{ label: 'e1', category: 'electronics' },
|
||||
{ label: 'e2', category: 'electronics' }
|
||||
])
|
||||
try {
|
||||
expect(labelsOf(await brainy.find({ where: { category: 'electronics' }, limit: 100 }))).toEqual(['e1', 'e2'])
|
||||
expect(labelsOf(await brainy.find({ where: { category: 5 }, limit: 100 }))).toEqual(['n1', 'n2'])
|
||||
} finally {
|
||||
await brainy.close()
|
||||
}
|
||||
})
|
||||
|
||||
it('keeps `5` and `\'5\'` apart — a kind is part of the value, not a formatting detail', async () => {
|
||||
const brainy = await brainWith([
|
||||
{ label: 'num', category: 5 },
|
||||
{ label: 'str', category: '5' }
|
||||
])
|
||||
try {
|
||||
expect(labelsOf(await brainy.find({ where: { category: 5 }, limit: 100 }))).toEqual(['num'])
|
||||
expect(labelsOf(await brainy.find({ where: { category: '5' }, limit: 100 }))).toEqual(['str'])
|
||||
} finally {
|
||||
await brainy.close()
|
||||
}
|
||||
})
|
||||
|
||||
it('serves booleans mixed into a field that already holds strings', async () => {
|
||||
const brainy = await brainWith([
|
||||
{ label: 's1', category: 'yes' },
|
||||
{ label: 'b1', category: true },
|
||||
{ label: 'b2', category: false }
|
||||
])
|
||||
try {
|
||||
expect(labelsOf(await brainy.find({ where: { category: true }, limit: 100 }))).toEqual(['b1'])
|
||||
expect(labelsOf(await brainy.find({ where: { category: false }, limit: 100 }))).toEqual(['b2'])
|
||||
expect(labelsOf(await brainy.find({ where: { category: 'yes' }, limit: 100 }))).toEqual(['s1'])
|
||||
} finally {
|
||||
await brainy.close()
|
||||
}
|
||||
})
|
||||
|
||||
it('ranges over the numeric part of a mixed field', async () => {
|
||||
const brainy = await brainWith([
|
||||
{ label: 'unpriced', category: 'on request' },
|
||||
{ label: 'cheap', category: 100 },
|
||||
{ label: 'mid', category: 500 },
|
||||
{ label: 'dear', category: 900 }
|
||||
])
|
||||
try {
|
||||
const found = await brainy.find({
|
||||
where: { category: { greaterThan: 200 } },
|
||||
limit: 100
|
||||
})
|
||||
expect(labelsOf(found)).toEqual(['dear', 'mid'])
|
||||
} finally {
|
||||
await brainy.close()
|
||||
}
|
||||
})
|
||||
})
|
||||
241
tests/unit/indexes/columnStore/column-store-mixed-kind.test.ts
Normal file
241
tests/unit/indexes/columnStore/column-store-mixed-kind.test.ts
Normal file
|
|
@ -0,0 +1,241 @@
|
|||
/**
|
||||
* @module column-store-mixed-kind.test
|
||||
* @description Typed posting lists: one field, several value KINDS, each
|
||||
* answerable on its own.
|
||||
*
|
||||
* The behaviour these pin replaced a first-writer type freeze. The first value
|
||||
* a field ever saw fixed that field's type; every later value of another kind
|
||||
* was coerced to it, and when coercion failed — `Number('electronics')` — the
|
||||
* value was dropped from the index with no error at all. The row stayed
|
||||
* readable by id and by vector and vanished from every equality filter on the
|
||||
* field. These tests therefore care about ORDER: strings-then-numbers and
|
||||
* numbers-then-strings have to behave identically, because neither writer owns
|
||||
* the field.
|
||||
*
|
||||
* Kinds never coerce into one another at query time either. `5` and `'5'` are
|
||||
* different values and match different rows.
|
||||
*/
|
||||
|
||||
import { describe, it, expect, beforeEach, afterEach } from 'vitest'
|
||||
import { ColumnStore } from '../../../../src/indexes/columnStore/ColumnStore.js'
|
||||
import { MemoryStorage } from '../../../../src/storage/adapters/memoryStorage.js'
|
||||
import { EntityIdMapper } from '../../../../src/utils/entityIdMapper.js'
|
||||
|
||||
describe('ColumnStore — typed posting lists per (field, kind)', () => {
|
||||
let storage: MemoryStorage
|
||||
let idMapper: EntityIdMapper
|
||||
let store: ColumnStore
|
||||
|
||||
beforeEach(async () => {
|
||||
storage = new MemoryStorage()
|
||||
await storage.init()
|
||||
idMapper = new EntityIdMapper({ storage, storageKey: 'test:idMapper' })
|
||||
await idMapper.init()
|
||||
|
||||
store = new ColumnStore({ flushThreshold: 10 })
|
||||
await store.init(storage, idMapper)
|
||||
})
|
||||
|
||||
afterEach(async () => {
|
||||
await store.close()
|
||||
})
|
||||
|
||||
/** Resolve a filter to the sorted UUIDs it matched. */
|
||||
const uuidsOf = async (field: string, value: unknown): Promise<string[]> => {
|
||||
const bitmap = await store.filter(field, value)
|
||||
return Array.from(bitmap)
|
||||
.map((id) => idMapper.getUuid(Number(id)))
|
||||
.filter((u): u is string => u !== undefined)
|
||||
.sort()
|
||||
}
|
||||
|
||||
describe('equality answers on the query value’s own kind', () => {
|
||||
it('serves numbers written AFTER strings on the same field', async () => {
|
||||
store.addEntity(BigInt(idMapper.getOrAssign('s1')), { category: 'electronics' })
|
||||
store.addEntity(BigInt(idMapper.getOrAssign('s2')), { category: 'furniture' })
|
||||
store.addEntity(BigInt(idMapper.getOrAssign('n1')), { category: 5 })
|
||||
store.addEntity(BigInt(idMapper.getOrAssign('n2')), { category: 5 })
|
||||
store.addEntity(BigInt(idMapper.getOrAssign('n3')), { category: 7 })
|
||||
|
||||
// The numbers are in the index, though a string got there first.
|
||||
expect(await uuidsOf('category', 5)).toEqual(['n1', 'n2'])
|
||||
expect(await uuidsOf('category', 7)).toEqual(['n3'])
|
||||
// And the strings did not move.
|
||||
expect(await uuidsOf('category', 'electronics')).toEqual(['s1'])
|
||||
expect(await uuidsOf('category', 'furniture')).toEqual(['s2'])
|
||||
})
|
||||
|
||||
it('serves strings written AFTER numbers on the same field', async () => {
|
||||
store.addEntity(BigInt(idMapper.getOrAssign('n1')), { category: 5 })
|
||||
store.addEntity(BigInt(idMapper.getOrAssign('n2')), { category: 5 })
|
||||
store.addEntity(BigInt(idMapper.getOrAssign('s1')), { category: 'electronics' })
|
||||
store.addEntity(BigInt(idMapper.getOrAssign('s2')), { category: 'electronics' })
|
||||
|
||||
// 'electronics' would have become NaN and been dropped under the freeze.
|
||||
expect(await uuidsOf('category', 'electronics')).toEqual(['s1', 's2'])
|
||||
expect(await uuidsOf('category', 5)).toEqual(['n1', 'n2'])
|
||||
})
|
||||
|
||||
it('does not coerce a number query into the string postings, or back', async () => {
|
||||
store.addEntity(BigInt(idMapper.getOrAssign('num')), { code: 5 })
|
||||
store.addEntity(BigInt(idMapper.getOrAssign('str')), { code: '5' })
|
||||
|
||||
expect(await uuidsOf('code', 5)).toEqual(['num'])
|
||||
expect(await uuidsOf('code', '5')).toEqual(['str'])
|
||||
})
|
||||
|
||||
it('serves booleans mixed into a field that already holds strings and numbers', async () => {
|
||||
store.addEntity(BigInt(idMapper.getOrAssign('s1')), { flag: 'yes' })
|
||||
store.addEntity(BigInt(idMapper.getOrAssign('n1')), { flag: 1 })
|
||||
store.addEntity(BigInt(idMapper.getOrAssign('b1')), { flag: true })
|
||||
store.addEntity(BigInt(idMapper.getOrAssign('b2')), { flag: false })
|
||||
|
||||
expect(await uuidsOf('flag', true)).toEqual(['b1'])
|
||||
expect(await uuidsOf('flag', false)).toEqual(['b2'])
|
||||
// `true` stores as 1 internally; that is an encoding, not a value.
|
||||
expect(await uuidsOf('flag', 1)).toEqual(['n1'])
|
||||
expect(await uuidsOf('flag', 'yes')).toEqual(['s1'])
|
||||
})
|
||||
|
||||
it('answers nothing — not something coerced — for a kind the field never held', async () => {
|
||||
store.addEntity(BigInt(idMapper.getOrAssign('s1')), { category: 'electronics' })
|
||||
|
||||
expect(await uuidsOf('category', 5)).toEqual([])
|
||||
expect(await uuidsOf('category', true)).toEqual([])
|
||||
})
|
||||
|
||||
it('holds every kind across a flush, not just the one in the tail buffer', async () => {
|
||||
store.addEntity(BigInt(idMapper.getOrAssign('s1')), { category: 'electronics' })
|
||||
store.addEntity(BigInt(idMapper.getOrAssign('n1')), { category: 5 })
|
||||
await store.flush()
|
||||
store.addEntity(BigInt(idMapper.getOrAssign('s2')), { category: 'electronics' })
|
||||
store.addEntity(BigInt(idMapper.getOrAssign('n2')), { category: 5 })
|
||||
|
||||
expect(await uuidsOf('category', 'electronics')).toEqual(['s1', 's2'])
|
||||
expect(await uuidsOf('category', 5)).toEqual(['n1', 'n2'])
|
||||
})
|
||||
})
|
||||
|
||||
describe('range filters read the numeric postings', () => {
|
||||
it('ranges over the numeric subset of a mixed field, ignoring its strings', async () => {
|
||||
store.addEntity(BigInt(idMapper.getOrAssign('cheap')), { price: 100 })
|
||||
store.addEntity(BigInt(idMapper.getOrAssign('mid')), { price: 500 })
|
||||
store.addEntity(BigInt(idMapper.getOrAssign('dear')), { price: 900 })
|
||||
store.addEntity(BigInt(idMapper.getOrAssign('unpriced')), { price: 'on request' })
|
||||
await store.flush()
|
||||
|
||||
const inRange = await store.rangeQuery('price', 200, 1000)
|
||||
const uuids = Array.from(inRange)
|
||||
.map((id) => idMapper.getUuid(Number(id)))
|
||||
.sort()
|
||||
expect(uuids).toEqual(['dear', 'mid'])
|
||||
})
|
||||
|
||||
it('an unbounded range still reports every kind — it is the “has a value” probe', async () => {
|
||||
store.addEntity(BigInt(idMapper.getOrAssign('n1')), { mixed: 42 })
|
||||
store.addEntity(BigInt(idMapper.getOrAssign('s1')), { mixed: 'text' })
|
||||
store.addEntity(BigInt(idMapper.getOrAssign('b1')), { mixed: true })
|
||||
await store.flush()
|
||||
|
||||
const anyValue = await store.rangeQuery('mixed')
|
||||
const uuids = Array.from(anyValue)
|
||||
.map((id) => idMapper.getUuid(Number(id)))
|
||||
.sort()
|
||||
expect(uuids).toEqual(['b1', 'n1', 's1'])
|
||||
})
|
||||
})
|
||||
|
||||
describe('the index reports what a field actually holds', () => {
|
||||
it('names every kind present, not the one that got there first', async () => {
|
||||
store.addEntity(BigInt(idMapper.getOrAssign('s1')), { category: 'electronics' })
|
||||
expect(store.getFieldKinds('category')).toEqual(['string'])
|
||||
|
||||
store.addEntity(BigInt(idMapper.getOrAssign('n1')), { category: 5 })
|
||||
store.addEntity(BigInt(idMapper.getOrAssign('b1')), { category: true })
|
||||
expect(store.getFieldKinds('category')).toEqual(['number', 'string', 'boolean'])
|
||||
|
||||
// And the field is still ONE field by name.
|
||||
expect(store.getIndexedFields()).toEqual(['category'])
|
||||
expect(store.hasField('category')).toBe(true)
|
||||
})
|
||||
|
||||
it('reports an unknown field as holding nothing', () => {
|
||||
expect(store.getFieldKinds('never-written')).toEqual([])
|
||||
})
|
||||
})
|
||||
|
||||
describe('an integer column widens rather than rounding', () => {
|
||||
it('keeps a non-integer written after integers as itself', async () => {
|
||||
store.addEntity(BigInt(idMapper.getOrAssign('a')), { score: 4 })
|
||||
store.addEntity(BigInt(idMapper.getOrAssign('b')), { score: 4.5 })
|
||||
store.addEntity(BigInt(idMapper.getOrAssign('c')), { score: 5 })
|
||||
await store.flush()
|
||||
|
||||
// 4.5 used to round to 5 and answer `score === 5` alongside c.
|
||||
expect(await uuidsOf('score', 4.5)).toEqual(['b'])
|
||||
expect(await uuidsOf('score', 5)).toEqual(['c'])
|
||||
expect(await uuidsOf('score', 4)).toEqual(['a'])
|
||||
})
|
||||
})
|
||||
|
||||
describe('close then reopen', () => {
|
||||
it('keeps every typed posting, on the same storage', async () => {
|
||||
store.addEntity(BigInt(idMapper.getOrAssign('s1')), { category: 'electronics' })
|
||||
store.addEntity(BigInt(idMapper.getOrAssign('n1')), { category: 5 })
|
||||
store.addEntity(BigInt(idMapper.getOrAssign('b1')), { category: true })
|
||||
store.addEntity(BigInt(idMapper.getOrAssign('f1')), { score: 1.5 })
|
||||
await store.flush()
|
||||
await store.close()
|
||||
|
||||
store = new ColumnStore({ flushThreshold: 10 })
|
||||
await store.init(storage, idMapper)
|
||||
|
||||
expect(store.getFieldKinds('category')).toEqual(['number', 'string', 'boolean'])
|
||||
expect(await uuidsOf('category', 'electronics')).toEqual(['s1'])
|
||||
expect(await uuidsOf('category', 5)).toEqual(['n1'])
|
||||
expect(await uuidsOf('category', true)).toEqual(['b1'])
|
||||
expect(await uuidsOf('score', 1.5)).toEqual(['f1'])
|
||||
})
|
||||
|
||||
it('accepts new values of every kind after the reopen', async () => {
|
||||
store.addEntity(BigInt(idMapper.getOrAssign('s1')), { category: 'electronics' })
|
||||
store.addEntity(BigInt(idMapper.getOrAssign('n1')), { category: 5 })
|
||||
await store.flush()
|
||||
await store.close()
|
||||
|
||||
store = new ColumnStore({ flushThreshold: 10 })
|
||||
await store.init(storage, idMapper)
|
||||
|
||||
store.addEntity(BigInt(idMapper.getOrAssign('s2')), { category: 'electronics' })
|
||||
store.addEntity(BigInt(idMapper.getOrAssign('n2')), { category: 5 })
|
||||
store.addEntity(BigInt(idMapper.getOrAssign('b1')), { category: false })
|
||||
await store.flush()
|
||||
|
||||
expect(await uuidsOf('category', 'electronics')).toEqual(['s1', 's2'])
|
||||
expect(await uuidsOf('category', 5)).toEqual(['n1', 'n2'])
|
||||
expect(await uuidsOf('category', false)).toEqual(['b1'])
|
||||
})
|
||||
|
||||
it('opens an index written by the pre-typed-postings shape and reads it unchanged', async () => {
|
||||
// A single-kind field is byte-identical to what the old writer produced:
|
||||
// one manifest at `_column_index/<field>/MANIFEST.json`, no kind
|
||||
// subdirectory anywhere. That IS the old on-disk shape, so proving the
|
||||
// new reader serves it proves an old index still opens.
|
||||
store.addEntity(BigInt(idMapper.getOrAssign('a')), { status: 'active' })
|
||||
store.addEntity(BigInt(idMapper.getOrAssign('b')), { status: 'archived' })
|
||||
await store.flush()
|
||||
|
||||
const keys = await (storage as unknown as {
|
||||
listObjectsUnderPath: (prefix: string) => Promise<string[]>
|
||||
}).listObjectsUnderPath('_column_index/')
|
||||
expect(keys.some((k) => k.includes('/k/'))).toBe(false)
|
||||
|
||||
await store.close()
|
||||
store = new ColumnStore({ flushThreshold: 10 })
|
||||
await store.init(storage, idMapper)
|
||||
|
||||
expect(store.getFieldKinds('status')).toEqual(['string'])
|
||||
expect(await uuidsOf('status', 'active')).toEqual(['a'])
|
||||
})
|
||||
})
|
||||
})
|
||||
Reference in a new issue