diff --git a/docs/api-contract.json b/docs/api-contract.json index 12cb37c8..c4f4e056 100644 --- a/docs/api-contract.json +++ b/docs/api-contract.json @@ -1507,6 +1507,7 @@ "BrainyError", "DerivedArtifactMissingError", "GraphIndexNotReadyError", + "MetadataArrayTooLargeError", "MetadataIndexNotReadyError", "MigrationInProgressError", "ProtectedArtifactError", diff --git a/src/brainy.ts b/src/brainy.ts index 81250144..fe672305 100644 --- a/src/brainy.ts +++ b/src/brainy.ts @@ -8195,7 +8195,23 @@ export class Brainy implements BrainyInterface { // Rank by score (top offset+limit), then drop the offset — identical ordering // to a full `sort((a, b) => b.score - a.score)` + slice, but the native // `sort:topK` provider can compute only the page instead of the full sort. - if (results.length >= offset + limit) { + // + // ONLY when score IS the requested order. An explicit `orderBy` names a + // different ordering key, and this block cannot serve it: it ranks by + // score and CUTS the page, so the tail's `orderBy` sort below either + // never runs at all (the early return, when there is no `connected` / + // `fusion` work left) or runs over a page that score already chose — + // ordering eight rows relevance picked instead of the eight the field + // ordering asks for. Both readings were silent: `find({ query, where, + // orderBy })` answered in score order while `find({ where, orderBy })` + // answered in field order, and nothing said the request had been dropped. + // + // With `orderBy` present the candidate set falls through UNCUT to the + // tail, which orders it in full and pages that ordering — "page last", + // the graph-first law applied to ordering rather than to filtering. The + // set is bounded by the legs (the text matches inside the universe plus + // the beam walk's `limit * 2`), not by the store. + if (!params.orderBy && results.length >= offset + limit) { const k = offset + limit const order = rankIndicesByScore(results.map(r => r.score), k, true) results = reorderByIndices(results, order).slice(offset, k) @@ -13013,7 +13029,7 @@ export class Brainy implements BrainyInterface { } return this._flushQueued } - return this.startFlushLeader() + return this.#startFlushLeader() } /** @@ -13022,16 +13038,23 @@ export class Brainy implements BrainyInterface { * the ONE queued waiter (if any) is promoted. The `finally` callback returns * nothing on purpose: a callback that returned the promoted run's promise * would make the leader await its own follower. + * + * ECMAScript-private (`#`), not TypeScript `private`: `private` is erased at + * compile time, so the method would still land on the prototype — and the + * contract manifest reads the surface the BUILD exposes, so it would emit + * this as a contract door. A door is a promise every engine implementing the + * contract must keep; this is the flush gate's own bookkeeping. `#` keeps it + * off the prototype, where the emitter cannot see it. * @returns The leader's own promise, settling on its own body alone. */ - private startFlushLeader(): Promise { + #startFlushLeader(): Promise { const run = this._runFlush() // `finally` and not `then`: a failed flush must still open the gate, or // one rejection would wedge every later flush behind a promise nobody // will ever settle. const gated: Promise = run.finally(() => { if (this._flushInFlight === gated) this._flushInFlight = null - this.promoteQueuedFlush() + this.#promoteQueuedFlush() }) this._flushInFlight = gated return gated @@ -13042,9 +13065,12 @@ export class Brainy implements BrainyInterface { * leader and settle its deferred from that run. Never throws into the * leader's `finally`: a synchronous failure starting the promoted run is * reported to the waiter, which must be settled on every path. + * + * ECMAScript-private for the same reason as the leader starter above: + * internals are not doors. * @returns Nothing. */ - private promoteQueuedFlush(): void { + #promoteQueuedFlush(): void { const settle = this._flushQueuedSettle if (!settle) return // Clear BEFORE starting, so the promoted run's own joiners queue afresh @@ -13052,7 +13078,7 @@ export class Brainy implements BrainyInterface { this._flushQueued = null this._flushQueuedSettle = null try { - this.startFlushLeader().then(settle.resolve, settle.reject) + this.#startFlushLeader().then(settle.resolve, settle.reject) } catch (error) { settle.reject(error) } @@ -20505,34 +20531,45 @@ export class Brainy implements BrainyInterface { // Phase 1: Flush ALL components in parallel to persist buffered data // This is critical when cor native providers buffer data in Rust memory + // + // READ-ONLY GUARD, applied to EVERY flush here. A flush is a write by + // definition, and a reader has nothing of its own to persist — but these + // calls were not conditional, so a read-only open → read → close REWROTE + // four files under `_system/`: the metadata field registry (whose flush() + // saves it unconditionally, "even with no dirty fields"), and the three + // type/subtype statistics files the storage adapter's count flush stamps. + // Every one of them was re-stamped on a session that committed nothing. + // A reader must leave `_system/` exactly as it found it — the same law the + // clean-shutdown marker already lives under (see the generation-store + // guard below and `Brainy.openReadOnly`). await Promise.all([ // Flush HNSW dirty nodes (deferred persistence mode) (async () => { - if (this.index && typeof this.index.flush === 'function') { + if (this.index && !this.isReadOnly && typeof this.index.flush === 'function') { await this.index.flush() } })(), // Flush metadata index (field indexes + EntityIdMapper) (async () => { - if (this.metadataIndex && typeof this.metadataIndex.flush === 'function') { + if (this.metadataIndex && !this.isReadOnly && typeof this.metadataIndex.flush === 'function') { await this.metadataIndex.flush() } })(), // Flush graph adjacency index (LSM trees) (async () => { - if (this.graphIndex && typeof this.graphIndex.flush === 'function') { + if (this.graphIndex && !this.isReadOnly && typeof this.graphIndex.flush === 'function') { await this.graphIndex.flush() } })(), // Flush storage adapter counts (async () => { - if (this.storage && typeof this.storage.flushCounts === 'function') { + if (this.storage && !this.isReadOnly && typeof this.storage.flushCounts === 'function') { await this.storage.flushCounts() } })(), // Flush aggregation index state (async () => { - if (this._aggregationIndex) { + if (this._aggregationIndex && !this.isReadOnly) { await this._aggregationIndex.flush() } })(), @@ -20581,21 +20618,37 @@ export class Brainy implements BrainyInterface { // Phase 2: Close components to release resources (timers, file handles) // Data is already safe on disk from Phase 1 + // + // READ-ONLY GUARD, same law as Phase 1. Each of these closes is a WRITER: + // the graph index drains both LSM MemTables to SSTables and stamps its + // watermark, and the vector/metadata `close` hooks — optional doors the + // reference engine leaves unimplemented, but which a native provider fills + // in — persist their buffered state. None of that is a reader's to write. + // + // A reader still has to RELEASE what it holds, which is why this is a + // branch rather than a skip: `stopBackgroundFlush()` is the non-writing + // half of the graph index's close, clearing the auto-flush interval that + // would otherwise outlive the session. The optional hooks have no + // non-writing counterpart to call, and a provider that buffers nothing on + // a read-only open has nothing to release. await Promise.all([ (async () => { - if (this.graphIndex && typeof this.graphIndex.close === 'function') { + if (!this.graphIndex) return + if (this.isReadOnly) { + this.graphIndex.stopBackgroundFlush() + } else if (typeof this.graphIndex.close === 'function') { await this.graphIndex.close() } })(), (async () => { const index = this.index as JsHnswVectorIndex & VectorIndexOptionalHooks - if (index && typeof index.close === 'function') { + if (index && !this.isReadOnly && typeof index.close === 'function') { await index.close() } })(), (async () => { const metadataIndex = this.metadataIndex as MetadataIndexManager & MetadataIndexOptionalHooks - if (metadataIndex && typeof metadataIndex.close === 'function') { + if (metadataIndex && !this.isReadOnly && typeof metadataIndex.close === 'function') { await metadataIndex.close() } })(), diff --git a/src/errors/brainyError.ts b/src/errors/brainyError.ts index a58236e3..4301d3f7 100644 --- a/src/errors/brainyError.ts +++ b/src/errors/brainyError.ts @@ -405,3 +405,68 @@ export class MigrationInProgressError extends BrainyError { } } } + +/** + * THE INDEXABLE-ARRAY BOUND. An array-valued metadata field indexes one posting + * per element, so an unbounded array is an unbounded write — a 384-float + * embedding parked in the metadata bag would mint 384 postings for one row. + * The bound exists to keep that out of the index. + * + * 64 is hardcoded on purpose (the zero-config law: no knob). It sits far above + * every legitimate multi-value field the engine has seen — tags, authors, + * categories, labels, participant lists — and far below any real embedding + * width, so the two populations do not overlap and no caller has to tune it. + * + * It replaces a limit of 10 that was applied SILENTLY: a row whose `tags` array + * held eleven entries had that field skipped entirely and dropped out of every + * filtered search on it, with no error, no warning and no way to tell the + * difference from "no row matches". A rule this consequential is a law with a + * name and a refusal, not a `continue`. + */ +export const MAX_INDEXED_ARRAY_LENGTH = 64 + +/** + * A metadata field carries an array longer than {@link MAX_INDEXED_ARRAY_LENGTH}. + * + * Thrown at the WRITE door (`add` / `update` / `relate` / `updateRelation`), so + * the caller learns at the moment of writing that the field will not be + * searchable — rather than discovering it later as rows that quietly fail to + * match. Carries the field, its length and the bound so a handler can report + * or repair without parsing the message. + * + * The cure is one of: store the long array outside the indexed bag (`data` + * carries arbitrary content and is not indexed element-wise); pass an embedding + * as the first-class `vector` parameter, which is where a vector belongs; or + * shorten the field to the values that are actually queried. + */ +export class MetadataArrayTooLargeError extends BrainyError { + /** The metadata field whose array is too long (its full dotted address). */ + public readonly field: string + /** How many elements that array holds. */ + public readonly length: number + /** The bound it exceeded — {@link MAX_INDEXED_ARRAY_LENGTH}. */ + public readonly limit: number + + constructor(site: string, field: string, length: number, limit: number) { + super( + `${site}: metadata field '${field}' holds ${length} array elements, ` + + `over the ${limit}-element indexing bound. An array field indexes one ` + + `posting per element, so an unbounded array is an unbounded write. ` + + `This write is refused rather than indexed partially or skipped silently ` + + `— a skipped field drops the row out of every filtered search on '${field}' ` + + `with no way to tell that from "nothing matched". ` + + `Cures: put the long array in 'data' (stored, not indexed element-wise); ` + + `pass an embedding as the first-class 'vector' parameter; or keep only ` + + `the values you actually query in '${field}'.`, + 'VALIDATION', + false + ) + this.name = 'MetadataArrayTooLargeError' + this.field = field + this.length = length + this.limit = limit + if (Error.captureStackTrace) { + Error.captureStackTrace(this, MetadataArrayTooLargeError) + } + } +} diff --git a/src/graph/graphAdjacencyIndex.ts b/src/graph/graphAdjacencyIndex.ts index ebd3b90c..2c131a30 100644 --- a/src/graph/graphAdjacencyIndex.ts +++ b/src/graph/graphAdjacencyIndex.ts @@ -1105,13 +1105,31 @@ export class GraphAdjacencyIndex implements GraphIndexProvider { } /** - * Clean shutdown + * Stop the auto-flush interval WITHOUT writing anything. + * + * The non-writing half of {@link close}, for a shutdown that must leave the + * store byte-identical — a read-only brain's close. `close()` itself is a + * writer: it drains both LSM MemTables to SSTables and stamps the watermark, + * which is exactly right for a writer and forbidden for a reader. A reader + * still has to release this interval, though: it is the one piece of this + * index that outlives the close and could fire against a store the session no + * longer owns. + * + * @returns Nothing. */ - async close(): Promise { + stopBackgroundFlush(): void { if (this.flushTimer) { clearInterval(this.flushTimer) this.flushTimer = undefined } + } + + /** + * Clean shutdown — drains both trees and stamps the watermark. THIS WRITES; + * a read-only brain must call {@link stopBackgroundFlush} instead. + */ + async close(): Promise { + this.stopBackgroundFlush() // Close both LSM-trees (will flush MemTables to SSTables) if (this.initialized) { diff --git a/src/index.ts b/src/index.ts index e946f15c..673e1e6f 100644 --- a/src/index.ts +++ b/src/index.ts @@ -203,7 +203,7 @@ export { EntityNotFoundError, RelationNotFoundError } from './errors/notFound.js // Base error + typed migration-lock error — thrown by any data-plane call while a // brain runs its one-time 7.x→8.0 upgrade; catch to answer HTTP 503 + Retry-After. -export { BrainyError, MigrationInProgressError, GraphIndexNotReadyError, MetadataIndexNotReadyError, VectorIndexNotReadyError, ProtectedArtifactError, DerivedArtifactMissingError } from './errors/brainyError.js' +export { BrainyError, MigrationInProgressError, GraphIndexNotReadyError, MetadataIndexNotReadyError, VectorIndexNotReadyError, ProtectedArtifactError, DerivedArtifactMissingError, MetadataArrayTooLargeError, MAX_INDEXED_ARRAY_LENGTH } from './errors/brainyError.js' export type { BrainyErrorType } from './errors/brainyError.js' // ============= 8.0 Db API — generational MVCC ============= diff --git a/src/utils/metadataIndex.ts b/src/utils/metadataIndex.ts index a3aa7679..0a5aef75 100644 --- a/src/utils/metadataIndex.ts +++ b/src/utils/metadataIndex.ts @@ -40,7 +40,7 @@ import { import { EntityIdMapper } from './entityIdMapper.js' import { RoaringBitmap32, roaringLibraryInitialize } from './roaring/index.js' import { FieldTypeInference, FieldType } from './fieldTypeInference.js' -import { BrainyError } from '../errors/brainyError.js' +import { BrainyError, MAX_INDEXED_ARRAY_LENGTH } from '../errors/brainyError.js' /** * Fields whose values are stored in the sparse index as BUCKETED values @@ -289,8 +289,10 @@ export class MetadataIndexManager implements MetadataIndexProvider { // No name-based exclude/allow lists — the field-addressing law: every // user field indexes, whatever its name ('content', 'data', 'id', // 'vector', … included). Bulk payloads are kept out by uniform value- - // SHAPE rules in extractIndexableFields (arrays >10 never become - // posting scalars; >100-char values index hashed), never by name. + // SHAPE rules in extractIndexableFields (arrays longer than + // MAX_INDEXED_ARRAY_LENGTH never become posting scalars, and the write + // door refuses them by name; >100-char values index hashed), never by + // field name. } // Initialize metadata cache with similar config to search cache @@ -961,9 +963,41 @@ export class MetadataIndexManager implements MetadataIndexProvider { } /** - * Get IDs for a range using chunked sparse index with zone maps and roaring bitmaps - * Now fully lazy-loaded via UnifiedCache (no local sparseIndices Map) - * Normalize min/max for timestamp bucketing before comparison + * Get IDs for a range using the legacy chunked sparse index (zone maps + + * roaring bitmaps). Lazy-loaded via UnifiedCache. + * + * ORDER IS NOT A KEY. This path compares NORMALIZED values, and + * {@link normalizeValue} carries an escape hatch that is order-destroying by + * design: a string over 100 characters is replaced by {@link hashValue}'s + * digest so it can be used as a filesystem-safe key. Feeding that digest to + * an ORDERING comparison — which is what a `gte` / `lt` / `between` does — + * ranks rows by hash. The result is not empty and not an error: it is a + * confidently ordered wrong answer, and it disagrees with the column-store + * path (`getIdsForRange` above), which compares raw values and is correct. + * + * Two changes hold the line here: + * + * 1. THE BOUNDS ARE NEVER HASHED. They are normalized with `allowHash = + * false`, so a long bound stays comparable instead of collapsing to a + * digest. This alone fixes the common shape — a long bound queried + * against ordinary short values, where the digest sorts below every + * letter and `gte` therefore matched the entire store. + * + * 2. A HASHED KEY IS REFUSED, NEVER GUESSED. The persisted keys are whatever + * the pre-7.20.0 writer normalized them to, so a field whose values ran + * long is stored hashed and its order is simply not recoverable from this + * index. Rather than compare digests, the query throws a typed + * `BrainyError('INVALID_QUERY')` naming the field, the bound and the cure. + * Loud beats wrong. + * + * KNOWN, NAMED DIVERGENCE. The persisted keys are also lower-cased and + * trimmed by `normalizeValue`, so this path's string ranges are + * CASE-INSENSITIVE where the column store's are not. That is a property of + * the bytes a pre-7.20.0 engine wrote, not of the comparison: the raw values + * are not in the index to compare. The bounds are normalized into the same + * case-folded space so the comparison is at least self-consistent, and the + * divergence disappears with the field itself once the column store adopts + * it. See the module note on `getIdsFromChunks` for the path's lifetime. */ private async getIdsFromChunksForRange( field: string, @@ -979,9 +1013,27 @@ export class MetadataIndexManager implements MetadataIndexProvider { } // Normalize min/max for consistent comparison with indexed values - // (indexed values are bucketed for timestamps, so we must bucket the query bounds too) - const normalizedMin = min !== undefined ? this.normalizeValue(min, field) : undefined - const normalizedMax = max !== undefined ? this.normalizeValue(max, field) : undefined + // (indexed values are bucketed for timestamps, so we must bucket the query + // bounds too) — but NEVER through the hash escape hatch, which would make + // the bound incomparable. See the doc comment above. + const normalizedMin = min !== undefined ? this.normalizeValue(min, field, false) : undefined + const normalizedMax = max !== undefined ? this.normalizeValue(max, field, false) : undefined + + // REFUSE BEFORE SELECTING. Chunk selection itself orders values: it tests + // the bounds against each chunk's zone-map min/max. If those are hashes the + // selection is already meaningless — and its failure mode is an EMPTY + // answer (no chunk appears to overlap), which is the quietest wrong answer + // of all. So the key space is checked here, before a single chunk is + // chosen, and again per key below for a chunk whose zone map happens to + // read clean. + for (const chunkId of sparseIndex.getAllChunkIds()) { + const zoneMap = sparseIndex.getChunk(chunkId)?.zoneMap + for (const bound of [zoneMap?.min, zoneMap?.max]) { + if (typeof bound === 'string' && MetadataIndexManager.isHashedValue(bound)) { + throw MetadataIndexManager.rangeOverHashedIndex(field) + } + } + } // Find candidate chunks using zone maps const candidateChunkIds = sparseIndex.findChunksForRange(normalizedMin, normalizedMax) @@ -996,6 +1048,13 @@ export class MetadataIndexManager implements MetadataIndexProvider { const chunk = await this.chunkManager.loadChunk(field, chunkId) if (chunk) { for (const [value, bitmap] of chunk.entries) { + // A hashed key carries no order. Refuse the range rather than rank by + // digest — the whole answer is unsound, so failing on the first one + // is the honest outcome. + if (MetadataIndexManager.isHashedValue(value)) { + throw MetadataIndexManager.rangeOverHashedIndex(field) + } + // Check if value is in range using numeric-aware comparison // (normalizeValue converts numbers to strings, so we must compare numerically) let inRange = true @@ -1024,6 +1083,25 @@ export class MetadataIndexManager implements MetadataIndexProvider { return this.idMapper.intsIterableToUuids(allIntIds) } + /** + * The refusal a range query gets when the legacy sparse index holds hashed + * keys for the field. Names the field and the cure; never a wrong answer. + */ + private static rangeOverHashedIndex(field: string): BrainyError { + return new BrainyError( + `Range query on field "${field}" cannot be served by the legacy sparse index: ` + + `its values were persisted as hashes (values over 100 characters are stored ` + + `hashed to stay within filesystem name limits), and a hash carries no order — ` + + `comparing them would return a confidently ordered wrong answer. ` + + `Equality (\`where: { ${field}: value }\`) still works on this index. ` + + `To range over this field, let the column store adopt it: run ` + + `brain.repairIndex({ rebuild: ['metadata'] }), which rebuilds the field into ` + + `the column store, where ranges compare raw values.`, + 'INVALID_QUERY', + false + ) + } + /** * Get roaring bitmap for a field-value pair without converting to UUIDs * This is used for fast multi-field intersection queries using hardware-accelerated bitmap AND @@ -1191,8 +1269,17 @@ export class MetadataIndexManager implements MetadataIndexProvider { * value-based detection (DuckDB-inspired). Analyzes actual data values, not names. * * NO FALLBACKS - Pure value-based detection only. + * + * @param value - The value to normalize. + * @param field - Optional field name (drives the per-field statistics strategy). + * @param allowHash - Whether the >100-character escape hatch may fire. TRUE + * everywhere a normalized value is used as a KEY (equality postings, chunk + * entries, filenames) — that is what the hash exists for. FALSE on the + * ORDER-comparing path: a hash is deliberately order-destroying, so a + * bound that hashes can only be compared as nonsense. See + * {@link isHashedValue} and `getIdsFromChunksForRange`. */ - private normalizeValue(value: any, field?: string): string { + private normalizeValue(value: any, field?: string, allowHash: boolean = true): string { if (value === null || value === undefined) return '__NULL__' if (typeof value === 'boolean') return value ? '__TRUE__' : '__FALSE__' @@ -1250,21 +1337,34 @@ export class MetadataIndexManager implements MetadataIndexProvider { // Default normalization if (typeof value === 'number') return value.toString() if (Array.isArray(value)) { - const joined = value.map(v => this.normalizeValue(v, field)).join(',') + const joined = value.map(v => this.normalizeValue(v, field, allowHash)).join(',') // Hash very long array values to avoid filesystem limits - if (joined.length > 100) { + if (allowHash && joined.length > 100) { return this.hashValue(joined) } return joined } const stringValue = String(value).toLowerCase().trim() // Hash very long string values to avoid filesystem limits - if (stringValue.length > 100) { + if (allowHash && stringValue.length > 100) { return this.hashValue(stringValue) } return stringValue } + /** + * Is this normalized value a HASH rather than the value itself? + * + * {@link hashValue} is an escape hatch for filesystem name limits, and it is + * deliberately order-destroying: two values whose hashes compare one way + * routinely compare the other way themselves. Anything that ORDERS normalized + * values has to know when it is holding one, because comparing hashes yields + * a confident, wrong answer rather than an error. + */ + private static isHashedValue(normalized: string): boolean { + return normalized.startsWith('__HASH_') + } + /** * Create a short hash for long values to avoid filesystem filename limits */ @@ -1289,9 +1389,10 @@ export class MetadataIndexManager implements MetadataIndexProvider { * 'content', 'vector' in a bag are ordinary user fields) * - Record-frame plumbing (vector, connections, level, data, _rev, id) * never indexes — that is namespace routing, not a name carve-out - * - Value-SHAPE rules apply uniformly to all names: arrays >10 never - * become posting scalars; purely numeric key names (array indices) - * skip; >100-char values index hashed (normalizeValue) + * - Value-SHAPE rules apply uniformly to all names: arrays longer than + * MAX_INDEXED_ARRAY_LENGTH never become posting scalars (and say so — + * the write door refuses them outright); purely numeric key names + * (array indices) skip; >100-char values index hashed (normalizeValue) */ private extractIndexableFields(data: any): Array<{ field: string, value: any }> { const fields: Array<{ field: string, value: any }> = [] @@ -1353,13 +1454,37 @@ export class MetadataIndexManager implements MetadataIndexProvider { // This catches vectors stored as objects: {0: 0.1, 1: 0.2, ...} if (/^\d+$/.test(key)) continue - // Skip large arrays (> 10 elements) - likely vectors or bulk data - if (Array.isArray(value) && value.length > 10) continue + // THE INDEXABLE-ARRAY BOUND ({@link MAX_INDEXED_ARRAY_LENGTH}). An + // array field mints one posting per element, so the index has always + // carried a ceiling — it was 10, and it was applied by this bare + // `continue`: an eleven-element `tags` array had its whole field + // skipped and the row dropped out of every filtered search on it, with + // no error, no warning, and nothing to distinguish that from "no row + // matches". The ceiling is not the defect; the silence was. + // + // The write door refuses this shape by name now + // (`MetadataArrayTooLargeError`, thrown from paramValidation's + // `rejectOversizeIndexArrays`), so a live add/update never reaches + // here over the bound. Reaching it means the row is ALREADY on disk — + // written by an older engine under the old rule — and this is a + // rebuild, a catch-up fold or a remove reading it back. Refusing there + // would make an existing store un-rebuildable, so the row is admitted + // and the skipped field is NARRATED instead. Never silent, either way. + if (Array.isArray(value) && value.length > MAX_INDEXED_ARRAY_LENGTH) { + prodLog.warn( + `[brainy] metadata field '${fullKey}' holds ${value.length} array elements, ` + + `over the ${MAX_INDEXED_ARRAY_LENGTH}-element indexing bound — the field is ` + + `NOT indexed for this row, so it will not match a where-clause on '${fullKey}'. ` + + `This row predates the bound (the write door refuses this shape now). ` + + `Move the long array into 'data', or pass an embedding as the 'vector' parameter.` + ) + continue + } if (value && typeof value === 'object' && !Array.isArray(value)) { // Recurse into nested objects (but not arrays), keeping the frame extract(value, fullKey, frame) - } else if (Array.isArray(value) && value.length <= 10) { + } else if (Array.isArray(value)) { // Small arrays: index as multi-value field (all with same field name) // Example: tags: ["javascript", "node"] → field="tags", value="javascript" + field="tags", value="node" for (const item of value) { diff --git a/src/utils/paramValidation.ts b/src/utils/paramValidation.ts index 00790a4a..f1addb5b 100644 --- a/src/utils/paramValidation.ts +++ b/src/utils/paramValidation.ts @@ -18,6 +18,7 @@ import { findCallerLocation } from './callerLocation.js' import * as os from 'node:os' import * as fs from 'node:fs' import { parseFieldAddress, UnsupportedFindOptionError } from '../db/fieldAddressing.js' +import { MAX_INDEXED_ARRAY_LENGTH, MetadataArrayTooLargeError } from '../errors/brainyError.js' const getSystemMemory = (): number => { if (os) { @@ -538,8 +539,53 @@ function rejectForgedSystemKeys(metadata: Record | undefined, s } } +/** + * THE INDEXABLE-ARRAY BOUND, enforced at the write door. + * + * An array-valued metadata field indexes one posting per element, so the index + * has always carried a ceiling. It used to be 10, and it was applied by a bare + * `continue` deep inside field extraction: a row whose `tags` array held eleven + * entries had that field skipped entirely and dropped out of every filtered + * search on it — no error, no warning, and no way for the caller to tell the + * difference from "no row matches". Silence is the defect; the ceiling is not. + * + * The bound is now {@link MAX_INDEXED_ARRAY_LENGTH}, high enough that every + * legitimate multi-value field clears it, and it REFUSES here instead of + * dropping data downstream. Refusing at the write door is what makes it + * actionable: the caller learns at the moment of writing, with the field, the + * length and the bound in hand. + * + * Scope is the caller's own metadata bag — the values that become postings. + * Nested bags are walked, because a nested field indexes under its dotted + * address exactly like a top-level one. Arrays of OBJECTS are not walked: the + * index only ever makes postings from an array's scalar elements. + * + * @param metadata - The caller's metadata bag (undefined is fine). + * @param site - The write door's name, for the message ('add()', 'update()', …). + * @throws {MetadataArrayTooLargeError} Naming the field, its length and the bound. + */ +function rejectOversizeIndexArrays(metadata: Record | undefined, site: string): void { + if (!metadata) return + + const walk = (bag: Record, prefix: string): void => { + for (const [key, value] of Object.entries(bag)) { + const address = prefix ? `${prefix}.${key}` : key + if (Array.isArray(value)) { + if (value.length > MAX_INDEXED_ARRAY_LENGTH) { + throw new MetadataArrayTooLargeError(site, address, value.length, MAX_INDEXED_ARRAY_LENGTH) + } + } else if (value && typeof value === 'object') { + walk(value as Record, address) + } + } + } + + walk(metadata, '') +} + export function validateAddParams(params: AddParams): void { rejectForgedSystemKeys(params.metadata as Record | undefined, 'add()') + rejectOversizeIndexArrays(params.metadata as Record | undefined, 'add()') // 'data' is ABSENT only when null/undefined — an empty string ('') is real // content (a legitimate empty file's first write) and must not be treated // as missing. Falsy-but-present values (0, false, '') all count as present; @@ -608,6 +654,7 @@ export function validateAddParams(params: AddParams): void { */ export function validateUpdateParams(params: UpdateParams): void { rejectForgedSystemKeys(params.metadata as Record | undefined, 'update()') + rejectOversizeIndexArrays(params.metadata as Record | undefined, 'update()') // Same absent-vs-empty distinction as validateAddParams: '' is a real new // value (e.g. truncating a file to empty content via overwrite), only // null/undefined means "no new data was given". @@ -682,6 +729,7 @@ export function validateUpdateParams(params: UpdateParams): void { */ export function validateRelateParams(params: RelateParams): void { rejectForgedSystemKeys(params.metadata as Record | undefined, 'relate()') + rejectOversizeIndexArrays(params.metadata as Record | undefined, 'relate()') // 8.0 verb-id contract (L.7): verb ids are UUIDs, generated by brainy. // RelateParams has no `id` field — an untyped caller passing one would // previously have it silently ignored (a generated UUID was used instead). @@ -731,6 +779,7 @@ export function validateRelateParams(params: RelateParams): void { */ export function validateUpdateRelationParams(params: UpdateRelationParams): void { rejectForgedSystemKeys(params.metadata as Record | undefined, 'updateRelation()') + rejectOversizeIndexArrays(params.metadata as Record | undefined, 'updateRelation()') if (!params.id) { throw new Error('id is required for updateRelation') } diff --git a/tests/integration/find-orderby-every-path.test.ts b/tests/integration/find-orderby-every-path.test.ts new file mode 100644 index 00000000..7637a79b --- /dev/null +++ b/tests/integration/find-orderby-every-path.test.ts @@ -0,0 +1,244 @@ +/** + * @module tests/integration/find-orderby-every-path + * @description `orderBy` IS THE ORDER — on every find() path, not just the + * metadata-only one. + * + * THE DEFECT. `find({ where, orderBy })` (metadata only) answered in field + * order. `find({ query, where, orderBy })` and `find({ vector, where, orderBy })` + * answered in SCORE order, silently: the vector/filter block ranked the fused + * candidates by score, cut the page, and returned early — the tail's `orderBy` + * sort sat below that early return and never ran. Nothing threw, nothing warned, + * and the two paths disagreed about what "ordered by rank" means. A caller + * paging `orderBy: 'rank', order: 'desc'` over a hybrid find got relevance + * order wearing an ordering request's clothes. + * + * Where `connected` or `fusion` kept the tail alive the defect changed shape + * rather than disappearing: the block had already CUT the page by score, so the + * tail ordered the rows relevance had chosen instead of the rows the ordering + * asks for — a correctly sorted page of the wrong rows. + * + * The early cut fires only once the candidate set reaches `offset + limit` + * rows, which is why small fixtures never saw it: below that threshold the + * block falls through and the tail's sort does apply. That is the whole shape + * of the bug — an ordering that is correct until there is enough data to matter. + * + * THE LAW. An explicit `orderBy` displaces score as the ordering key on every + * path. The candidate set the path produced is ordered IN FULL and the page is + * cut from that ordering — the graph-first law's "page last", applied to + * ordering rather than to filtering. Score-ranked early paging is for the + * default (no `orderBy`) case only, where score IS the requested order. + * + * THE PIN. Differential, against the metadata-only path — the one path that + * always honoured `orderBy`. + * + * WHAT THE DIFFERENTIAL CAN AND CANNOT CLAIM. `orderBy` orders the candidate + * set; it does not enlarge it. The hybrid legs are bounded by construction (the + * text leg and the beam walk each take `limit * 2`), so a differential against + * the metadata-only path — whose universe is every matching row — is only + * meaningful where those bounds provably cover the universe. The fixture is + * sized so they do (12 rows, `limit` 6 → a `limit * 2` = 12-row text leg), and + * the covering is ASSERTED from the leg's own output rather than assumed. This + * pin is about ordering, and it says nothing about recall. + */ +import { describe, it, expect, beforeAll } from 'vitest' +import { Brainy } from '../../src/brainy' +import { NounType, VerbType } from '../../src/types/graphTypes' +import { resolveEntityId } from '../../src/utils/idNormalization' + +/** Embedding width of the default model — the row vectors must match it. */ +const DIM = 384 + +/** A deterministic, per-row-distinct unit vector (no embedder in the fixture). */ +function seededVector(seed: number): number[] { + const v = new Array(DIM) + for (let i = 0; i < DIM; i++) { + v[i] = Math.sin((i + 1) * 0.11 + seed * 0.37) * 0.5 + Math.cos((i + 1) * 0.05 + seed * 0.13) * 0.3 + } + const magnitude = Math.sqrt(v.reduce((sum, x) => sum + x * x, 0)) + return v.map((x) => x / magnitude) +} + +/** + * Ranks, shuffled — so no scoring order can reproduce them by luck, and the + * ordering the pins assert is visibly not the insertion order either. + */ +const RANKS = [7, 3, 11, 1, 9, 5, 12, 2, 10, 4, 8, 6] +const ROWS = RANKS.length +/** The page size every pin uses: `limit * 2` covers the whole universe. */ +const LIMIT = 6 +/** The neighbour subset — the graph-first universe — and its own page size. */ +const NEIGHBOURS = 8 +const GRAPH_LIMIT = 4 + +describe('find(): orderBy is the order on every path', () => { + let brain: Brainy + const QUERY = 'orbital telemetry' + const anchor = 'ordering-anchor' + const neighbourIds: string[] = [] + + beforeAll(async () => { + brain = new Brainy({ requireSubtype: false, storage: { type: 'memory' } }) + await brain.init() + + let seed = 1 + await brain.add({ + id: anchor, + data: 'ground station anchor record', + type: NounType.Thing, + metadata: { lane: 'anchor', rank: 0 }, + vector: seededVector(seed++) + }) + + for (let i = 0; i < ROWS; i++) { + const id = `row-${i}` + await brain.add({ + id, + // EVERY row carries both query words, so the text leg reaches all of + // them and the hybrid candidate set covers the whole universe. + data: `orbital telemetry packet ${i} recorded downlink`, + type: NounType.Document, + metadata: { lane: 'alpha', rank: RANKS[i] }, + vector: seededVector(seed++) + }) + if (i < NEIGHBOURS) { + await brain.relate({ from: anchor, to: id, type: VerbType.RelatedTo }) + neighbourIds.push(resolveEntityId(id)) + } + } + }) + + it('the fixture: the hybrid candidate set covers the whole filter universe', async () => { + const universe: string[] = await (brain as any).filterIdsBelted({ lane: 'alpha' }) + expect(universe).toHaveLength(ROWS) + + // The text leg is bounded at `limit * 2`; the fixture is sized so that + // bound reaches every row in the universe. This is the precondition the + // differential below rests on — asserted from the leg itself. + const textScored = await (brain as any).executeTextSearchScored(QUERY, LIMIT * 2, universe) + expect(textScored).toHaveLength(ROWS) + + // And the candidate set is large enough to trigger the score-ranked early + // cut this pin exists to keep out of an ordered query's way. + expect(ROWS).toBeGreaterThanOrEqual(LIMIT) + }) + + it('metadata-only + orderBy: the reference ordering', async () => { + const rows = await brain.find({ + where: { lane: 'alpha' }, + orderBy: 'rank', + order: 'desc', + limit: LIMIT + } as any) + expect(rows.map((r: any) => r.metadata.rank)).toEqual([12, 11, 10, 9, 8, 7]) + }) + + it('hybrid (query + where) + orderBy: the same page as the metadata-only path', async () => { + const params = { where: { lane: 'alpha' }, orderBy: 'rank', order: 'desc' as const, limit: LIMIT } + const expected = await brain.find(params as any) + const actual = await brain.find({ ...params, query: QUERY } as any) + + expect(actual).toHaveLength(expected.length) + expect(actual.map((r: any) => r.id)).toEqual(expected.map((r: any) => r.id)) + expect(actual.map((r: any) => r.metadata.rank)).toEqual([12, 11, 10, 9, 8, 7]) + }) + + it('hybrid + orderBy asc: the ordering key is honoured in both directions', async () => { + const params = { where: { lane: 'alpha' }, orderBy: 'rank', order: 'asc' as const, limit: LIMIT } + const expected = await brain.find(params as any) + const actual = await brain.find({ ...params, query: QUERY } as any) + + expect(actual.map((r: any) => r.id)).toEqual(expected.map((r: any) => r.id)) + expect(actual.map((r: any) => r.metadata.rank)).toEqual([1, 2, 3, 4, 5, 6]) + }) + + it('hybrid + orderBy + offset: page two is page two of the ORDERING', async () => { + const params = { + where: { lane: 'alpha' }, + orderBy: 'rank', + order: 'desc' as const, + limit: LIMIT, + offset: LIMIT + } + const expected = await brain.find(params as any) + const actual = await brain.find({ ...params, query: QUERY } as any) + + expect(actual).toHaveLength(LIMIT) + expect(actual.map((r: any) => r.id)).toEqual(expected.map((r: any) => r.id)) + expect(actual.map((r: any) => r.metadata.rank)).toEqual([6, 5, 4, 3, 2, 1]) + }) + + it('hybrid + orderBy: paging walks the ordering monotonically, no row twice', async () => { + const seen: number[] = [] + for (let offset = 0; offset < ROWS; offset += LIMIT) { + const page = await brain.find({ + query: QUERY, + where: { lane: 'alpha' }, + orderBy: 'rank', + order: 'desc', + limit: LIMIT, + offset + } as any) + seen.push(...page.map((r: any) => r.metadata.rank)) + } + expect(seen).toHaveLength(ROWS) + expect(new Set(seen).size).toBe(ROWS) + // Strictly descending across every page boundary. + for (let i = 1; i < seen.length; i++) expect(seen[i]).toBeLessThan(seen[i - 1]) + }) + + it('vector + where + orderBy: field order, not distance order', async () => { + // The beam walk takes `limit * 2` = the whole universe here, so the page is + // the true top of the ordering — which distance order cannot produce. + const rows = await brain.find({ + vector: seededVector(1000), + where: { lane: 'alpha' }, + orderBy: 'rank', + order: 'desc', + limit: LIMIT + } as any) + expect(rows.map((r: any) => r.metadata.rank)).toEqual([12, 11, 10, 9, 8, 7]) + }) + + it('graph-first (query + connected + where) + orderBy: the neighbour set, ordered', async () => { + const actual = await brain.find({ + query: QUERY, + connected: { from: anchor, direction: 'out' as const }, + where: { lane: 'alpha' }, + orderBy: 'rank', + order: 'desc', + limit: GRAPH_LIMIT + } as any) + + expect(actual).toHaveLength(GRAPH_LIMIT) + const neighbours = new Set(neighbourIds) + for (const r of actual) expect(neighbours.has(r.id)).toBe(true) + + // The ordering covers the whole neighbour set, so the page holds the + // highest ranks AMONG THE NEIGHBOURS — not the ones the score ranking + // happened to surface first and the tail then sorted among themselves. + const expectedRanks = RANKS.slice(0, NEIGHBOURS) + .sort((a, b) => b - a) + .slice(0, GRAPH_LIMIT) + expect(expectedRanks).toEqual([12, 11, 9, 7]) + expect(actual.map((r: any) => r.metadata.rank)).toEqual(expectedRanks) + }) + + it('fusion + orderBy: the ordering survives the fusion rescore', async () => { + const actual = await brain.find({ + query: QUERY, + where: { lane: 'alpha' }, + fusion: 'weighted', + orderBy: 'rank', + order: 'desc', + limit: LIMIT + } as any) + expect(actual.map((r: any) => r.metadata.rank)).toEqual([12, 11, 10, 9, 8, 7]) + }) + + it('no orderBy: score order still stands (the default is untouched)', async () => { + const rows = await brain.find({ query: QUERY, where: { lane: 'alpha' }, limit: LIMIT } as any) + expect(rows).toHaveLength(LIMIT) + const scores = rows.map((r: any) => r.score) + for (let i = 1; i < scores.length; i++) expect(scores[i]).toBeLessThanOrEqual(scores[i - 1]) + }) +}) diff --git a/tests/integration/metadata-vector-exclusion.test.ts b/tests/integration/metadata-vector-exclusion.test.ts index 9e11f9dc..0ca25388 100644 --- a/tests/integration/metadata-vector-exclusion.test.ts +++ b/tests/integration/metadata-vector-exclusion.test.ts @@ -26,6 +26,7 @@ import { describe, it, expect, beforeEach, afterEach } from 'vitest' import { Brainy } from '../../src/brainy.js' import { NounType } from '../../src/types/graphTypes.js' import { existsSync, rmSync } from 'fs' +import { MetadataArrayTooLargeError, MAX_INDEXED_ARRAY_LENGTH } from '../../src/errors/brainyError.js' describe('Metadata Vector Exclusion Fix', () => { let brainy: Brainy @@ -155,29 +156,56 @@ describe('Metadata Vector Exclusion Fix', () => { expect(results[0].entity.metadata?.name).toBe('Bob') }) - it('should skip indexing large arrays (>10 elements)', async () => { - // Add entity with a large array (not a vector, just bulk data). + it('should REFUSE an array over the indexing bound, by name', async () => { + // A large array (not a vector, just bulk data). This used to be SKIPPED in + // silence at a bound of 10 — the field simply vanished from the index and + // the row dropped out of every `where` on it, indistinguishably from "no + // row matches". The bound is now MAX_INDEXED_ARRAY_LENGTH and it REFUSES. const largeArray = Array.from({ length: 100 }, (_, i) => `item${i}`) - await brainy.add({ - type: NounType.Document, - data: 'Doc with large array', - metadata: { - name: 'Doc with large array', - items: largeArray - } - }) + const err = await brainy + .add({ + type: NounType.Document, + data: 'Doc with large array', + metadata: { + name: 'Doc with large array', + items: largeArray + } + }) + .catch((e: any) => e) - // Large arrays (> 10 elements) are deliberately skipped to avoid indexing - // bulk/vector-like payloads: 'items' must NOT appear, and the 100 elements - // must NOT have produced 100 indexed fields. + expect(err).toBeInstanceOf(MetadataArrayTooLargeError) + expect(err.field).toBe('items') + expect(err.length).toBe(100) + expect(err.limit).toBe(MAX_INDEXED_ARRAY_LENGTH) + + // Nothing was indexed from the refused write — no 'items' field, and above + // all no per-element numeric fields (the original explosion class). const fields = await brainy.getAvailableFields() expect(fields).not.toContain('items') const numericFields = fields.filter(f => /(^|\.)\d+$/.test(f)) expect(numericFields).toEqual([]) + }) - // The scalar 'name' field IS indexed. - expect(fields).toContain('name') + it('should index an array UP TO the bound — the old limit of 10 was the bug', async () => { + await brainy.add({ + type: NounType.Document, + data: 'Doc with a long-but-legitimate tag list', + metadata: { + name: 'Doc with many tags', + items: Array.from({ length: MAX_INDEXED_ARRAY_LENGTH }, (_, i) => `item${i}`) + } + }) + + const fields = await brainy.getAvailableFields() + // The field IS indexed now, and still without per-element numeric fields. + expect(fields).toContain('items') + expect(fields.filter(f => /(^|\.)\d+$/.test(f))).toEqual([]) + + // And the eleventh element — the one the old bound silently dropped the + // whole field for — really is searchable. + const hits = await brainy.find({ where: { items: 'item10' } }) + expect(hits.length).toBeGreaterThan(0) }) it('should preserve HNSW vector search functionality', async () => { diff --git a/tests/integration/readonly-close-no-marker.test.ts b/tests/integration/readonly-close-no-marker.test.ts index 7bcf99df..ad9357db 100644 --- a/tests/integration/readonly-close-no-marker.test.ts +++ b/tests/integration/readonly-close-no-marker.test.ts @@ -149,12 +149,12 @@ describe('a read-only brain writes no clean-shutdown evidence', () => { brain = null // The FILE SET under `_system/` is unchanged — a reader creates and - // removes nothing. (Other files under `_system/` — e.g. the metadata - // field registry, which stamps its own `lastUpdated` on every persist — - // are a pre-existing, separate concern outside this fix's scope: this - // pin is specifically about the generation store's clean-shutdown - // evidence, not about every subsystem's close() being a true no-op for - // a reader.) + // removes nothing. This pin is specifically about the generation store's + // clean-shutdown evidence. The wider law — that a reader leaves EVERY + // file under `_system/` byte-identical, which this fix left open as a + // known residual (the metadata field registry and the three statistics + // files were still re-stamped by a reader's close) — is closed and pinned + // in `readonly-close-writes-nothing.test.ts`. const after = snapshotDir(systemDir()) expect([...after.keys()].sort()).toEqual([...before.keys()].sort()) diff --git a/tests/integration/readonly-close-writes-nothing.test.ts b/tests/integration/readonly-close-writes-nothing.test.ts new file mode 100644 index 00000000..701a1974 --- /dev/null +++ b/tests/integration/readonly-close-writes-nothing.test.ts @@ -0,0 +1,261 @@ +/** + * @module tests/integration/readonly-close-writes-nothing + * @description A READ-ONLY BRAIN LEAVES `_system/` BYTE-IDENTICAL — the WHOLE + * directory, not just the clean-shutdown marker. + * + * `readonly-close-no-marker` closed the marker half of this law and named the + * rest as a known, out-of-scope residual: + * + * "Other files under `_system/` — e.g. the metadata field registry, which + * stamps its own `lastUpdated` on every persist — are a pre-existing, + * separate concern outside this fix's scope." + * + * This is that residual, closed. MEASURED on the base before the fix, a + * read-only open → read → close rewrote FOUR files: + * + * _system/__metadata_field_registry__.json.gz + * _system/type-statistics.json.gz + * _system/subtype-statistics.json.gz + * _system/verb-subtype-statistics.json.gz + * + * THE CAUSE was not the closes the marker fix guarded — it was Phase 1 of + * `closeDurableSteps`, where every component flush ran unconditionally. A flush + * is a write by definition: `MetadataIndexManager#flush()` saves the field + * registry "even with no dirty fields" (its own comment), and the storage + * adapter's count flush re-stamps the three statistics files. A session that + * committed nothing re-stamped all four. Phase 2's closes were ungated too — + * the graph index's close drains both LSM MemTables and stamps a watermark, + * and the optional vector/metadata `close` hooks (unimplemented in the + * reference engine, filled in by a native provider) persist buffered state. + * + * THE LAW. A reader writes nothing, anywhere under `_system/`, at open or at + * close. It still RELEASES what it holds: the graph index's auto-flush interval + * is cleared through `stopBackgroundFlush()`, the non-writing half of its + * close, so nothing outlives the session. + * + * WHY IT MATTERS beyond tidiness: `_system/` is where a store keeps its + * evidence about itself — what the writer committed, what the projections have + * seen. A reader that rewrites any of it is vouching for a state it only + * observed, and on shared or snapshot storage it mutates bytes another process + * owns. + */ + +import { describe, it, expect, beforeEach, afterEach } from 'vitest' +import { mkdtempSync, rmSync, readdirSync, readFileSync, statSync } from 'node:fs' +import { createHash } from 'node:crypto' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { Brainy } from '../../src/brainy.js' +import { NounType, VerbType } from '../../src/types/graphTypes.js' + +/** Recursively hash every regular file under `dir`, keyed by its path relative to `dir`. */ +function snapshotDir(dir: string): Map { + const out = new Map() + const walk = (rel: string): void => { + const abs = rel ? join(dir, rel) : dir + let entries: string[] + try { + entries = readdirSync(abs) + } catch { + return + } + for (const name of entries) { + const childRel = rel ? join(rel, name) : name + const childAbs = join(dir, childRel) + const st = statSync(childAbs) + if (st.isDirectory()) { + walk(childRel) + } else if (st.isFile()) { + out.set(childRel, createHash('sha256').update(readFileSync(childAbs)).digest('hex')) + } + } + } + walk('') + return out +} + +/** Every path where `after` differs from `before`, labelled — the failure message. */ +function diff(before: Map, after: Map): string[] { + const lines: string[] = [] + for (const [path, hash] of after) { + if (!before.has(path)) lines.push(`ADDED ${path}`) + else if (before.get(path) !== hash) lines.push(`CHANGED ${path}`) + } + for (const path of before.keys()) if (!after.has(path)) lines.push(`REMOVED ${path}`) + return lines.sort() +} + +describe('a read-only brain writes nothing under `_system/`', () => { + let dir: string + let brain: Brainy | null = null + + beforeEach(() => { + dir = mkdtempSync(join(tmpdir(), 'brainy-readonly-writes-')) + }) + + afterEach(async () => { + if (brain) { + try { + await brain.close() + } catch { + /* already closed */ + } + brain = null + } + try { + rmSync(dir, { recursive: true, force: true }) + } catch { + /* ignore */ + } + }) + + const systemDir = () => join(dir, '_system') + + /** + * A writer seeds a store with nouns, verbs and queryable metadata — enough + * that the field registry, the statistics files and the graph index all hold + * real content — then closes cleanly. + */ + async function seedStore(): Promise { + const writer = new Brainy({ + requireSubtype: false, + storage: { type: 'filesystem', path: dir } + }) + await writer.init() + for (let i = 0; i < 6; i++) { + await writer.add({ + id: `seed-${i}`, + data: `seed entity ${i}`, + type: i % 2 === 0 ? NounType.Concept : NounType.Document, + metadata: { lane: i % 2 === 0 ? 'alpha' : 'beta', rank: i, tags: [`t${i}`, 'shared'] }, + vector: [] + }) + } + for (let i = 1; i < 6; i++) { + await writer.relate({ from: 'seed-0', to: `seed-${i}`, type: VerbType.RelatedTo }) + } + await writer.flush() + await writer.close() + } + + it('open → read → close leaves every file under `_system/` byte-identical', async () => { + await seedStore() + + const before = snapshotDir(systemDir()) + expect(before.size, 'the writer left a populated `_system/`').toBeGreaterThan(0) + + brain = await Brainy.openReadOnly({ storage: { type: 'filesystem', path: dir } }) + expect(brain.isReadOnly).toBe(true) + + // Exercise the read surface that drives each subsystem: statistics (counts), + // a metadata filter (field index + registry), a graph walk (adjacency), a + // vector search, and a direct get. + await brain.stats() + await brain.find({ where: { lane: 'alpha' }, limit: 10 } as any) + await brain.find({ where: { tags: 'shared' }, limit: 10 } as any) + await brain.find({ connected: { from: 'seed-0', direction: 'out' }, limit: 10 } as any) + await brain.get('seed-1') + + await brain.close() + brain = null + + const after = snapshotDir(systemDir()) + const changes = diff(before, after) + expect(changes, `a reader modified \`_system/\`:\n${changes.join('\n')}`).toEqual([]) + }, 120_000) + + it('names the four files that used to change — the measured shape of the defect', async () => { + await seedStore() + const before = snapshotDir(systemDir()) + + // These are the exact paths the base rewrote. Naming them keeps the pin + // honest about what it caught: if a future change reintroduces the write, + // the test above fails and this one says which subsystem did it. + const previouslyRewritten = [ + '__metadata_field_registry__.json.gz', + 'type-statistics.json.gz', + 'subtype-statistics.json.gz', + 'verb-subtype-statistics.json.gz' + ] + for (const name of previouslyRewritten) { + expect(before.has(name), `fixture must contain ${name}`).toBe(true) + } + + brain = await Brainy.openReadOnly({ storage: { type: 'filesystem', path: dir } }) + await brain.stats() + await brain.find({ where: { lane: 'alpha' }, limit: 10 } as any) + await brain.close() + brain = null + + const after = snapshotDir(systemDir()) + for (const name of previouslyRewritten) { + expect(after.get(name), `${name} was rewritten by a reader`).toBe(before.get(name)) + } + }, 120_000) + + it('a reader that only opens and closes — touching nothing — writes nothing', async () => { + await seedStore() + const before = snapshotDir(systemDir()) + + brain = await Brainy.openReadOnly({ storage: { type: 'filesystem', path: dir } }) + await brain.close() + brain = null + + const changes = diff(before, snapshotDir(systemDir())) + expect(changes, `an idle reader modified \`_system/\`:\n${changes.join('\n')}`).toEqual([]) + }, 120_000) + + it('two readers in sequence each leave the store exactly as they found it', async () => { + await seedStore() + const before = snapshotDir(systemDir()) + + for (let i = 0; i < 2; i++) { + const reader = await Brainy.openReadOnly({ storage: { type: 'filesystem', path: dir } }) + await reader.find({ where: { lane: 'beta' }, limit: 10 } as any) + await reader.close() + const changes = diff(before, snapshotDir(systemDir())) + expect(changes, `reader ${i + 1} modified \`_system/\`:\n${changes.join('\n')}`).toEqual([]) + } + }, 120_000) + + it('the store outside `_system/` is untouched too — a reader writes nowhere', async () => { + await seedStore() + const before = snapshotDir(dir) + + brain = await Brainy.openReadOnly({ storage: { type: 'filesystem', path: dir } }) + await brain.stats() + await brain.find({ where: { lane: 'alpha' }, limit: 10 } as any) + await brain.close() + brain = null + + const changes = diff(before, snapshotDir(dir)) + expect(changes, `a reader modified the store:\n${changes.join('\n')}`).toEqual([]) + }, 120_000) + + it('a WRITER still persists on close — the guard did not disarm the write path', async () => { + await seedStore() + const before = snapshotDir(systemDir()) + + const writer = new Brainy({ requireSubtype: false, storage: { type: 'filesystem', path: dir } }) + await writer.init() + await writer.add({ + id: 'after-reader', + data: 'a new row', + type: NounType.Concept, + metadata: { lane: 'gamma', rank: 99 }, + vector: [] + }) + await writer.close() + + // The writer's close DID move `_system/` — that is the whole point of the + // asymmetry, and the guard must not have flattened it. + expect(diff(before, snapshotDir(systemDir())).length).toBeGreaterThan(0) + + // And the row is really there on the next open. + const reopened = new Brainy({ requireSubtype: false, storage: { type: 'filesystem', path: dir } }) + await reopened.init() + brain = reopened + const hits = await reopened.find({ where: { lane: 'gamma' }, limit: 10 } as any) + expect(hits.length).toBe(1) + }, 120_000) +}) diff --git a/tests/unit/utils/metadataIndex-array-bound.test.ts b/tests/unit/utils/metadataIndex-array-bound.test.ts new file mode 100644 index 00000000..cbbf6b63 --- /dev/null +++ b/tests/unit/utils/metadataIndex-array-bound.test.ts @@ -0,0 +1,242 @@ +/** + * @module tests/unit/utils/metadataIndex-array-bound + * @description THE INDEXABLE-ARRAY BOUND — a law with a name and a refusal, + * not a `continue`. + * + * THE DEFECT. An array-valued metadata field indexes one posting per element, + * so the index has always carried a ceiling. It was 10, and it was applied by a + * bare `continue` deep inside field extraction: + * + * if (Array.isArray(value) && value.length > 10) continue + * + * A row whose `tags` array held ELEVEN entries therefore had that field skipped + * entirely — no posting, no error, no warning. The row then failed to match + * every filtered search on `tags`, including `{ tags: 'a-tag-it-really-has' }`, + * and the caller had no way to tell that from "no row matches". Eleven tags is + * not an exotic shape; the eleventh tag made the row invisible. + * + * THE LAW. Arrays of scalars index up to {@link MAX_INDEXED_ARRAY_LENGTH} = 64, + * hardcoded (the zero-config law: no knob), which clears every legitimate + * multi-value field and stays far below any embedding width. Above it the WRITE + * IS REFUSED by name — `MetadataArrayTooLargeError`, carrying the field, the + * length and the bound — at `add`, `update`, `relate` and `updateRelation` + * alike. Nothing is skipped in silence. + * + * THE ONE PLACE THE BOUND STILL SKIPS is a row already on disk, written by an + * older engine under the old rule and read back by a rebuild, a catch-up fold + * or a remove. Refusing there would make an existing store un-rebuildable — so + * the row is admitted and the skipped field is NARRATED. Both sides are pinned. + */ +import { describe, it, expect, beforeEach, vi, afterEach } from 'vitest' +import { Brainy } from '../../../src/brainy' +import { NounType, VerbType } from '../../../src/types/graphTypes' +import { MetadataArrayTooLargeError, MAX_INDEXED_ARRAY_LENGTH } from '../../../src/errors/brainyError' +import { resolveEntityId } from '../../../src/utils/idNormalization' +import { prodLog } from '../../../src/utils/logger' + +/** `n` distinct scalar tags. */ +function tags(n: number, prefix = 't'): string[] { + return Array.from({ length: n }, (_, i) => `${prefix}${i}`) +} + +describe('the indexable-array bound', () => { + let brain: Brainy + + beforeEach(async () => { + brain = new Brainy({ requireSubtype: false, storage: { type: 'memory' } }) + await brain.init() + }) + + describe('BELOW the bound: the array indexes, every element of it', () => { + it('the eleven-element array that used to vanish is searchable', async () => { + // ELEVEN — one over the old silent limit, the whole shape of the defect. + await brain.add({ + id: 'eleven', + data: 'a row with eleven tags', + type: NounType.Document, + metadata: { tags: tags(11) }, + vector: [] + }) + + // Every element is a posting, including the eleventh. + for (const tag of tags(11)) { + const hits = await brain.find({ where: { tags: tag }, limit: 10 } as any) + expect(hits.map((r: any) => r.id)).toContain(resolveEntityId('eleven')) + } + }) + + it('indexes right up to the bound — all 64 elements', async () => { + await brain.add({ + id: 'at-bound', + data: 'a row at the bound', + type: NounType.Document, + metadata: { tags: tags(MAX_INDEXED_ARRAY_LENGTH) }, + vector: [] + }) + + // The first, the last, and one in the middle. + for (const tag of ['t0', `t${MAX_INDEXED_ARRAY_LENGTH - 1}`, 't31']) { + const hits = await brain.find({ where: { tags: tag }, limit: 10 } as any) + expect(hits.map((r: any) => r.id)).toContain(resolveEntityId('at-bound')) + } + }) + + it('a nested bag\'s array indexes under its dotted address', async () => { + await brain.add({ + id: 'nested', + data: 'a row with a nested tag list', + type: NounType.Document, + metadata: { facets: { labels: tags(20, 'l') } }, + vector: [] + }) + const hits = await brain.find({ where: { 'facets.labels': 'l19' }, limit: 10 } as any) + expect(hits.map((r: any) => r.id)).toContain(resolveEntityId('nested')) + }) + }) + + describe('ABOVE the bound: the write is refused, by name', () => { + const OVER = MAX_INDEXED_ARRAY_LENGTH + 1 + + it('add() throws a typed error naming the field, the length and the bound', async () => { + const err = await brain + .add({ + id: 'too-many', + data: 'a row with too many tags', + type: NounType.Document, + metadata: { tags: tags(OVER) }, + vector: [] + } as any) + .catch((e: any) => e) + + expect(err).toBeInstanceOf(MetadataArrayTooLargeError) + expect(err.field).toBe('tags') + expect(err.length).toBe(OVER) + expect(err.limit).toBe(MAX_INDEXED_ARRAY_LENGTH) + expect(err.type).toBe('VALIDATION') + // The message carries all three, and names the cures. + expect(err.message).toContain('tags') + expect(err.message).toContain(String(OVER)) + expect(err.message).toContain(String(MAX_INDEXED_ARRAY_LENGTH)) + expect(err.message).toContain('vector') + }) + + it('the refused row is not written at all — no half-indexed ghost', async () => { + await expect( + brain.add({ + id: 'refused', + data: 'refused', + type: NounType.Document, + metadata: { tags: tags(OVER) }, + vector: [] + } as any) + ).rejects.toBeInstanceOf(MetadataArrayTooLargeError) + + expect(await brain.get('refused')).toBeNull() + const hits = await brain.find({ where: { tags: 't0' }, limit: 10 } as any) + expect(hits.map((r: any) => r.id)).not.toContain(resolveEntityId('refused')) + }) + + it('a 384-float embedding parked in the metadata bag is refused, not swallowed', async () => { + const err = await brain + .add({ + id: 'bag-vector', + data: 'an embedding in the wrong place', + type: NounType.Document, + metadata: { embedding: Array.from({ length: 384 }, (_, i) => i / 384) }, + vector: [] + } as any) + .catch((e: any) => e) + + expect(err).toBeInstanceOf(MetadataArrayTooLargeError) + expect(err.field).toBe('embedding') + expect(err.length).toBe(384) + }) + + it('update() refuses it too', async () => { + await brain.add({ + id: 'grow', + data: 'starts small', + type: NounType.Document, + metadata: { tags: tags(3) }, + vector: [] + }) + await expect( + brain.update({ id: 'grow', metadata: { tags: tags(OVER) } } as any) + ).rejects.toBeInstanceOf(MetadataArrayTooLargeError) + + // And the row keeps the values it had. + const hits = await brain.find({ where: { tags: 't1' }, limit: 10 } as any) + expect(hits.map((r: any) => r.id)).toContain(resolveEntityId('grow')) + }) + + it('relate() refuses it on a verb\'s metadata', async () => { + await brain.add({ id: 'a', data: 'a', type: NounType.Thing, vector: [] }) + await brain.add({ id: 'b', data: 'b', type: NounType.Thing, vector: [] }) + await expect( + brain.relate({ + from: 'a', + to: 'b', + type: VerbType.RelatedTo, + metadata: { tags: tags(OVER) } + } as any) + ).rejects.toBeInstanceOf(MetadataArrayTooLargeError) + }) + + it('a nested oversize array is refused under its dotted address', async () => { + const err = await brain + .add({ + id: 'nested-over', + data: 'nested and too long', + type: NounType.Document, + metadata: { facets: { labels: tags(OVER, 'l') } }, + vector: [] + } as any) + .catch((e: any) => e) + expect(err).toBeInstanceOf(MetadataArrayTooLargeError) + expect(err.field).toBe('facets.labels') + }) + }) + + describe('a row already on disk is admitted, and the skip is NARRATED', () => { + afterEach(() => { + vi.restoreAllMocks() + }) + + it('extraction over an old oversize row warns by field, length and bound', async () => { + const warn = vi.spyOn(prodLog, 'warn').mockImplementation(() => {}) + const index = (brain as any).metadataIndex + + // The shape an older engine persisted: the write door never saw it, so + // this reaches extraction directly — exactly as a rebuild or a remove + // reading the row back would. + const fields = index.extractIndexableFields({ + metadata: { tags: tags(MAX_INDEXED_ARRAY_LENGTH + 5), keep: 'me' } + }) + + // The oversize field contributes nothing... + expect(fields.filter((f: any) => f.field === 'tags')).toHaveLength(0) + // ...the rest of the row indexes normally — the row is not rejected... + expect(fields.some((f: any) => f.field === 'keep' && f.value === 'me')).toBe(true) + // ...and the skip is said out loud, with everything needed to act on it. + expect(warn).toHaveBeenCalled() + const said = warn.mock.calls.map((c: any[]) => String(c[0])).join('\n') + expect(said).toContain('tags') + expect(said).toContain(String(MAX_INDEXED_ARRAY_LENGTH + 5)) + expect(said).toContain(String(MAX_INDEXED_ARRAY_LENGTH)) + expect(said).toContain('NOT indexed') + }) + + it('an at-bound row on disk is indexed in full and says nothing', async () => { + const warn = vi.spyOn(prodLog, 'warn').mockImplementation(() => {}) + const index = (brain as any).metadataIndex + + const fields = index.extractIndexableFields({ + metadata: { tags: tags(MAX_INDEXED_ARRAY_LENGTH) } + }) + expect(fields.filter((f: any) => f.field === 'tags')).toHaveLength(MAX_INDEXED_ARRAY_LENGTH) + + const said = warn.mock.calls.map((c: any[]) => String(c[0])).join('\n') + expect(said).not.toContain('indexing bound') + }) + }) +}) diff --git a/tests/unit/utils/metadataIndex-sparse-range-collation.test.ts b/tests/unit/utils/metadataIndex-sparse-range-collation.test.ts new file mode 100644 index 00000000..d6d00568 --- /dev/null +++ b/tests/unit/utils/metadataIndex-sparse-range-collation.test.ts @@ -0,0 +1,258 @@ +/** + * @module tests/unit/utils/metadataIndex-sparse-range-collation + * @description RANGE QUERIES ON THE LEGACY SPARSE INDEX — order, or a refusal. + * Never a confidently ordered wrong answer. + * + * THE TWO RANGE PATHS. `getIdsForRange` routes a `gte` / `lt` / `between` two + * ways. The column store compares RAW values and is correct. The legacy sparse + * chunk index — the pre-7.20.0 fallback, still read for workspaces that have + * not been rebuilt — compared `normalizeValue()` output, and `normalizeValue` + * carries an escape hatch that destroys order on purpose: a string over 100 + * characters is replaced by a short hash so it can serve as a filesystem-safe + * key. Ordering hashes ranks rows by digest. + * + * THE DEFECT, IN TWO SHAPES. + * + * (a) A LONG BOUND against ordinary values. `where: { title: { gte: } }` collapsed the BOUND to `__HASH_…`, whose + * leading underscores sort below every letter — so a bound that should + * have excluded everything matched the entire field instead. This is the + * shape that reaches a caller who never stored a long value at all. + * + * (b) LONG VALUES in the index. A field whose values ran long was persisted + * hashed, so its order is not recoverable from this index at all. The old + * code compared the digests anyway and returned a subset chosen by hash. + * + * THE LAW. Bounds are normalized WITHOUT the hash escape hatch, so a long + * bound stays comparable — (a) is simply fixed. Where the persisted KEY is a + * hash, the order does not exist to be computed, and the query throws a typed + * `BrainyError('INVALID_QUERY')` naming the field and the cure — (b) is + * refused by name. Loud beats wrong. + * + * THE FIXTURE is a genuine legacy index: it is written through the same + * `ChunkManager` / `SparseIndex` doors a pre-7.20.0 engine wrote through, with + * keys normalized exactly as that engine normalized them, into a field the + * column store does not serve. The chunk WRITE path was removed in 11be039, so + * this is the only way the shape the read path exists for can be built. + * + * NOT CLAIMED HERE. The persisted keys are also lower-cased and trimmed by + * `normalizeValue`, so this path's string ranges are case-INSENSITIVE where + * the column store's are not. The raw values are not in the index to compare — + * that divergence is a property of the bytes on disk and it ends when the + * column store adopts the field. It is named in `getIdsFromChunksForRange`'s + * doc comment rather than papered over. + */ +import { describe, it, expect, beforeEach } from 'vitest' +import { Brainy } from '../../../src/brainy' +import { NounType } from '../../../src/types/graphTypes' +import { SparseIndex, ChunkManager } from '../../../src/utils/metadataIndexChunking' +import { BrainyError } from '../../../src/errors/brainyError' + +/** The field the legacy index covers — deliberately never given to a row, so + * the column store never learns it and the sparse fallback is the only path. */ +const FIELD = 'legacyTitle' + +/** + * Write a legacy sparse index for `field` exactly as a pre-7.20.0 engine did: + * one chunk, keys normalized through the index's own `normalizeValue`, ids as + * roaring bitmaps, a zone map and a bloom filter over the chunk. + * + * @param brain - The live brain whose metadata index gains the legacy field. + * @param field - Field name to index. + * @param valueToIds - Raw value → the entity ids that carried it. + */ +async function writeLegacySparseIndex( + brain: any, + field: string, + valueToIds: Array<[string, string[]]> +): Promise { + const index = brain.metadataIndex + const chunkManager: ChunkManager = index.chunkManager + const sparseIndex = new SparseIndex(field) + + // The keys a pre-7.20.0 writer persisted: normalizeValue output, hash escape + // hatch and all. This is what makes the fixture the real shape. + const chunk = await chunkManager.createChunk(field) + for (const [value, ids] of valueToIds) { + const key = index.normalizeValue(value, field) + for (const id of ids) await chunkManager.addToChunk(chunk, key, id) + } + await chunkManager.saveChunk(chunk) + + sparseIndex.registerChunk( + { + chunkId: chunk.chunkId, + field, + valueCount: chunk.entries.size, + idCount: Array.from(chunk.entries.values()).reduce((s: number, b: any) => s + b.size, 0), + zoneMap: (chunkManager as any).calculateZoneMap(chunk), + lastUpdated: Date.now(), + splitThreshold: 80, + mergeThreshold: 20 + }, + chunkManager.createBloomFilter(chunk) + ) + + await index.saveSparseIndex(field, sparseIndex) +} + +/** A deterministic string of `n` characters starting with `lead`. */ +function longString(lead: string, n: number): string { + return lead + 'x'.repeat(n - lead.length) +} + +describe('legacy sparse index: range queries order values, or refuse', () => { + let brain: Brainy + let index: any + let ids: string[] + + beforeEach(async () => { + brain = new Brainy({ requireSubtype: false, storage: { type: 'memory' } }) + await brain.init() + index = (brain as any).metadataIndex + + // Rows exist (so the id mapper can resolve them) but carry NO `legacyTitle` + // — the column store must not serve the field the pins query. + ids = [] + for (let i = 0; i < 3; i++) { + const id = `row-${i}` + await brain.add({ id, data: `row ${i}`, type: NounType.Thing, metadata: { lane: 'a' }, vector: [] }) + ids.push(id) + } + expect(index.columnStore.hasField(FIELD)).toBe(false) + }) + + describe('(a) a long BOUND against ordinary short values', () => { + // 'apple' < 'mango' < 'zebra', and every bound below is compared against + // these three raw keys. + beforeEach(async () => { + await writeLegacySparseIndex(brain, FIELD, [ + ['apple', [ids[0]]], + ['mango', [ids[1]]], + ['zebra', [ids[2]]] + ]) + }) + + it('the fixture: the values are stored raw, the long bound is what hashes', () => { + expect(index.normalizeValue('apple', FIELD)).toBe('apple') + // The bound is what the old code collapsed — and a digest sorts below + // every letter, which is exactly why `gte` matched everything. + const bound = longString('zzz', 120) + expect(index.normalizeValue(bound, FIELD)).toMatch(/^__HASH_/) + expect(index.normalizeValue(bound, FIELD) < 'apple').toBe(true) + }) + + it('gte a bound above every value matches NOTHING (it used to match all)', async () => { + const bound = longString('zzz', 120) + const matched = await index.getIdsForRange(FIELD, bound, undefined, true, true) + expect(matched).toEqual([]) + }) + + it('lte a bound above every value matches EVERY value', async () => { + const bound = longString('zzz', 120) + const matched = await index.getIdsForRange(FIELD, undefined, bound, true, true) + expect(matched).toHaveLength(3) + }) + + it('gte a long bound below every value matches every value', async () => { + const bound = longString('aaa', 120) + const matched = await index.getIdsForRange(FIELD, bound, undefined, true, true) + expect(matched).toHaveLength(3) + }) + + it('a long bound orders BETWEEN the values, not below all of them', async () => { + // 'mmm…' sits between 'mango' and 'zebra'. + const bound = longString('mmm', 120) + const matched = await index.getIdsForRange(FIELD, bound, undefined, true, true) + expect(matched).toHaveLength(1) + }) + + it('short bounds are unchanged — the ordinary case still orders correctly', async () => { + expect(await index.getIdsForRange(FIELD, 'b', undefined, true, true)).toHaveLength(2) + expect(await index.getIdsForRange(FIELD, undefined, 'n', true, true)).toHaveLength(2) + expect(await index.getIdsForRange(FIELD, 'b', 'n', true, true)).toHaveLength(1) + // Strict bounds stay strict. + expect(await index.getIdsForRange(FIELD, 'mango', undefined, false, true)).toHaveLength(1) + expect(await index.getIdsForRange(FIELD, 'mango', undefined, true, true)).toHaveLength(2) + }) + }) + + describe('(b) long VALUES — the index holds hashes, so the range is refused', () => { + beforeEach(async () => { + await writeLegacySparseIndex(brain, FIELD, [ + [longString('alpha', 140), [ids[0]]], + [longString('mike', 140), [ids[1]]], + [longString('zulu', 140), [ids[2]]] + ]) + }) + + it('the fixture: the persisted keys really are hashes', async () => { + const chunk = await index.chunkManager.loadChunk(FIELD, 0) + const keys = Array.from(chunk.entries.keys()) as string[] + expect(keys).toHaveLength(3) + for (const k of keys) expect(k).toMatch(/^__HASH_/) + // And their digest order is NOT their value order — the wrong answer the + // old code returned was wrong, not merely arbitrary. + const digestOrder = [...keys].sort() + const valueOrder = [ + index.normalizeValue(longString('alpha', 140), FIELD), + index.normalizeValue(longString('mike', 140), FIELD), + index.normalizeValue(longString('zulu', 140), FIELD) + ] + expect(digestOrder).not.toEqual(valueOrder) + }) + + it('a range over the hashed field throws a typed refusal naming the field', async () => { + await expect( + index.getIdsForRange(FIELD, longString('mike', 140), undefined, true, true) + ).rejects.toThrow(BrainyError) + + const err = await index + .getIdsForRange(FIELD, longString('mike', 140), undefined, true, true) + .catch((e: any) => e) + expect(err).toBeInstanceOf(BrainyError) + expect(err.type).toBe('INVALID_QUERY') + expect(err.message).toContain(FIELD) + expect(err.message).toContain('hash') + // The cure is named, not left to the caller to guess. + expect(err.message).toContain('repairIndex') + }) + + it('every range shape refuses — gte, lte and between alike', async () => { + const lo = longString('alpha', 140) + const hi = longString('zulu', 140) + for (const [min, max] of [ + [lo, undefined], + [undefined, hi], + [lo, hi] + ] as Array<[any, any]>) { + const err = await index.getIdsForRange(FIELD, min, max, true, true).catch((e: any) => e) + expect(err).toBeInstanceOf(BrainyError) + expect(err.type).toBe('INVALID_QUERY') + } + }) + + it('EQUALITY still works on the hashed field — only ordering is refused', async () => { + const matched = await index.getIds(FIELD, longString('mike', 140)) + expect(matched).toHaveLength(1) + }) + }) + + describe('numeric ranges on the legacy path are untouched', () => { + beforeEach(async () => { + await writeLegacySparseIndex(brain, FIELD, [ + ['5', [ids[0]]], + ['50', [ids[1]]], + ['500', [ids[2]]] + ]) + }) + + it('numbers still compare numerically, not lexicographically', async () => { + // The whole point of compareNormalizedValues: "50" < "500" numerically + // even though "500" < "50" would hold as strings by prefix. + expect(await index.getIdsForRange(FIELD, 10, undefined, true, true)).toHaveLength(2) + expect(await index.getIdsForRange(FIELD, undefined, 100, true, true)).toHaveLength(2) + expect(await index.getIdsForRange(FIELD, 10, 100, true, true)).toHaveLength(1) + }) + }) +})