Compare commits
6 commits
main
...
fix/10412-
| Author | SHA1 | Date | |
|---|---|---|---|
| b8a80b9900 | |||
| 3a339ce4af | |||
| 47cfaa7669 | |||
| 59003fd8bc | |||
| 913b4ffc6b | |||
| 819a5c5a8a |
13 changed files with 1401 additions and 57 deletions
|
|
@ -1507,6 +1507,7 @@
|
|||
"BrainyError",
|
||||
"DerivedArtifactMissingError",
|
||||
"GraphIndexNotReadyError",
|
||||
"MetadataArrayTooLargeError",
|
||||
"MetadataIndexNotReadyError",
|
||||
"MigrationInProgressError",
|
||||
"ProtectedArtifactError",
|
||||
|
|
|
|||
|
|
@ -8195,7 +8195,23 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
|||
// Rank by score (top offset+limit), then drop the offset — identical ordering
|
||||
// to a full `sort((a, b) => b.score - a.score)` + slice, but the native
|
||||
// `sort:topK` provider can compute only the page instead of the full sort.
|
||||
if (results.length >= offset + limit) {
|
||||
//
|
||||
// ONLY when score IS the requested order. An explicit `orderBy` names a
|
||||
// different ordering key, and this block cannot serve it: it ranks by
|
||||
// score and CUTS the page, so the tail's `orderBy` sort below either
|
||||
// never runs at all (the early return, when there is no `connected` /
|
||||
// `fusion` work left) or runs over a page that score already chose —
|
||||
// ordering eight rows relevance picked instead of the eight the field
|
||||
// ordering asks for. Both readings were silent: `find({ query, where,
|
||||
// orderBy })` answered in score order while `find({ where, orderBy })`
|
||||
// answered in field order, and nothing said the request had been dropped.
|
||||
//
|
||||
// With `orderBy` present the candidate set falls through UNCUT to the
|
||||
// tail, which orders it in full and pages that ordering — "page last",
|
||||
// the graph-first law applied to ordering rather than to filtering. The
|
||||
// set is bounded by the legs (the text matches inside the universe plus
|
||||
// the beam walk's `limit * 2`), not by the store.
|
||||
if (!params.orderBy && results.length >= offset + limit) {
|
||||
const k = offset + limit
|
||||
const order = rankIndicesByScore(results.map(r => r.score), k, true)
|
||||
results = reorderByIndices(results, order).slice(offset, k)
|
||||
|
|
@ -13013,7 +13029,7 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
|||
}
|
||||
return this._flushQueued
|
||||
}
|
||||
return this.startFlushLeader()
|
||||
return this.#startFlushLeader()
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -13022,16 +13038,23 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
|||
* the ONE queued waiter (if any) is promoted. The `finally` callback returns
|
||||
* nothing on purpose: a callback that returned the promoted run's promise
|
||||
* would make the leader await its own follower.
|
||||
*
|
||||
* ECMAScript-private (`#`), not TypeScript `private`: `private` is erased at
|
||||
* compile time, so the method would still land on the prototype — and the
|
||||
* contract manifest reads the surface the BUILD exposes, so it would emit
|
||||
* this as a contract door. A door is a promise every engine implementing the
|
||||
* contract must keep; this is the flush gate's own bookkeeping. `#` keeps it
|
||||
* off the prototype, where the emitter cannot see it.
|
||||
* @returns The leader's own promise, settling on its own body alone.
|
||||
*/
|
||||
private startFlushLeader(): Promise<void> {
|
||||
#startFlushLeader(): Promise<void> {
|
||||
const run = this._runFlush()
|
||||
// `finally` and not `then`: a failed flush must still open the gate, or
|
||||
// one rejection would wedge every later flush behind a promise nobody
|
||||
// will ever settle.
|
||||
const gated: Promise<void> = run.finally(() => {
|
||||
if (this._flushInFlight === gated) this._flushInFlight = null
|
||||
this.promoteQueuedFlush()
|
||||
this.#promoteQueuedFlush()
|
||||
})
|
||||
this._flushInFlight = gated
|
||||
return gated
|
||||
|
|
@ -13042,9 +13065,12 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
|||
* leader and settle its deferred from that run. Never throws into the
|
||||
* leader's `finally`: a synchronous failure starting the promoted run is
|
||||
* reported to the waiter, which must be settled on every path.
|
||||
*
|
||||
* ECMAScript-private for the same reason as the leader starter above:
|
||||
* internals are not doors.
|
||||
* @returns Nothing.
|
||||
*/
|
||||
private promoteQueuedFlush(): void {
|
||||
#promoteQueuedFlush(): void {
|
||||
const settle = this._flushQueuedSettle
|
||||
if (!settle) return
|
||||
// Clear BEFORE starting, so the promoted run's own joiners queue afresh
|
||||
|
|
@ -13052,7 +13078,7 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
|||
this._flushQueued = null
|
||||
this._flushQueuedSettle = null
|
||||
try {
|
||||
this.startFlushLeader().then(settle.resolve, settle.reject)
|
||||
this.#startFlushLeader().then(settle.resolve, settle.reject)
|
||||
} catch (error) {
|
||||
settle.reject(error)
|
||||
}
|
||||
|
|
@ -20505,34 +20531,45 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
|||
|
||||
// Phase 1: Flush ALL components in parallel to persist buffered data
|
||||
// This is critical when cor native providers buffer data in Rust memory
|
||||
//
|
||||
// READ-ONLY GUARD, applied to EVERY flush here. A flush is a write by
|
||||
// definition, and a reader has nothing of its own to persist — but these
|
||||
// calls were not conditional, so a read-only open → read → close REWROTE
|
||||
// four files under `_system/`: the metadata field registry (whose flush()
|
||||
// saves it unconditionally, "even with no dirty fields"), and the three
|
||||
// type/subtype statistics files the storage adapter's count flush stamps.
|
||||
// Every one of them was re-stamped on a session that committed nothing.
|
||||
// A reader must leave `_system/` exactly as it found it — the same law the
|
||||
// clean-shutdown marker already lives under (see the generation-store
|
||||
// guard below and `Brainy.openReadOnly`).
|
||||
await Promise.all([
|
||||
// Flush HNSW dirty nodes (deferred persistence mode)
|
||||
(async () => {
|
||||
if (this.index && typeof this.index.flush === 'function') {
|
||||
if (this.index && !this.isReadOnly && typeof this.index.flush === 'function') {
|
||||
await this.index.flush()
|
||||
}
|
||||
})(),
|
||||
// Flush metadata index (field indexes + EntityIdMapper)
|
||||
(async () => {
|
||||
if (this.metadataIndex && typeof this.metadataIndex.flush === 'function') {
|
||||
if (this.metadataIndex && !this.isReadOnly && typeof this.metadataIndex.flush === 'function') {
|
||||
await this.metadataIndex.flush()
|
||||
}
|
||||
})(),
|
||||
// Flush graph adjacency index (LSM trees)
|
||||
(async () => {
|
||||
if (this.graphIndex && typeof this.graphIndex.flush === 'function') {
|
||||
if (this.graphIndex && !this.isReadOnly && typeof this.graphIndex.flush === 'function') {
|
||||
await this.graphIndex.flush()
|
||||
}
|
||||
})(),
|
||||
// Flush storage adapter counts
|
||||
(async () => {
|
||||
if (this.storage && typeof this.storage.flushCounts === 'function') {
|
||||
if (this.storage && !this.isReadOnly && typeof this.storage.flushCounts === 'function') {
|
||||
await this.storage.flushCounts()
|
||||
}
|
||||
})(),
|
||||
// Flush aggregation index state
|
||||
(async () => {
|
||||
if (this._aggregationIndex) {
|
||||
if (this._aggregationIndex && !this.isReadOnly) {
|
||||
await this._aggregationIndex.flush()
|
||||
}
|
||||
})(),
|
||||
|
|
@ -20581,21 +20618,37 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
|||
|
||||
// Phase 2: Close components to release resources (timers, file handles)
|
||||
// Data is already safe on disk from Phase 1
|
||||
//
|
||||
// READ-ONLY GUARD, same law as Phase 1. Each of these closes is a WRITER:
|
||||
// the graph index drains both LSM MemTables to SSTables and stamps its
|
||||
// watermark, and the vector/metadata `close` hooks — optional doors the
|
||||
// reference engine leaves unimplemented, but which a native provider fills
|
||||
// in — persist their buffered state. None of that is a reader's to write.
|
||||
//
|
||||
// A reader still has to RELEASE what it holds, which is why this is a
|
||||
// branch rather than a skip: `stopBackgroundFlush()` is the non-writing
|
||||
// half of the graph index's close, clearing the auto-flush interval that
|
||||
// would otherwise outlive the session. The optional hooks have no
|
||||
// non-writing counterpart to call, and a provider that buffers nothing on
|
||||
// a read-only open has nothing to release.
|
||||
await Promise.all([
|
||||
(async () => {
|
||||
if (this.graphIndex && typeof this.graphIndex.close === 'function') {
|
||||
if (!this.graphIndex) return
|
||||
if (this.isReadOnly) {
|
||||
this.graphIndex.stopBackgroundFlush()
|
||||
} else if (typeof this.graphIndex.close === 'function') {
|
||||
await this.graphIndex.close()
|
||||
}
|
||||
})(),
|
||||
(async () => {
|
||||
const index = this.index as JsHnswVectorIndex & VectorIndexOptionalHooks
|
||||
if (index && typeof index.close === 'function') {
|
||||
if (index && !this.isReadOnly && typeof index.close === 'function') {
|
||||
await index.close()
|
||||
}
|
||||
})(),
|
||||
(async () => {
|
||||
const metadataIndex = this.metadataIndex as MetadataIndexManager & MetadataIndexOptionalHooks
|
||||
if (metadataIndex && typeof metadataIndex.close === 'function') {
|
||||
if (metadataIndex && !this.isReadOnly && typeof metadataIndex.close === 'function') {
|
||||
await metadataIndex.close()
|
||||
}
|
||||
})(),
|
||||
|
|
|
|||
|
|
@ -405,3 +405,68 @@ export class MigrationInProgressError extends BrainyError {
|
|||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* THE INDEXABLE-ARRAY BOUND. An array-valued metadata field indexes one posting
|
||||
* per element, so an unbounded array is an unbounded write — a 384-float
|
||||
* embedding parked in the metadata bag would mint 384 postings for one row.
|
||||
* The bound exists to keep that out of the index.
|
||||
*
|
||||
* 64 is hardcoded on purpose (the zero-config law: no knob). It sits far above
|
||||
* every legitimate multi-value field the engine has seen — tags, authors,
|
||||
* categories, labels, participant lists — and far below any real embedding
|
||||
* width, so the two populations do not overlap and no caller has to tune it.
|
||||
*
|
||||
* It replaces a limit of 10 that was applied SILENTLY: a row whose `tags` array
|
||||
* held eleven entries had that field skipped entirely and dropped out of every
|
||||
* filtered search on it, with no error, no warning and no way to tell the
|
||||
* difference from "no row matches". A rule this consequential is a law with a
|
||||
* name and a refusal, not a `continue`.
|
||||
*/
|
||||
export const MAX_INDEXED_ARRAY_LENGTH = 64
|
||||
|
||||
/**
|
||||
* A metadata field carries an array longer than {@link MAX_INDEXED_ARRAY_LENGTH}.
|
||||
*
|
||||
* Thrown at the WRITE door (`add` / `update` / `relate` / `updateRelation`), so
|
||||
* the caller learns at the moment of writing that the field will not be
|
||||
* searchable — rather than discovering it later as rows that quietly fail to
|
||||
* match. Carries the field, its length and the bound so a handler can report
|
||||
* or repair without parsing the message.
|
||||
*
|
||||
* The cure is one of: store the long array outside the indexed bag (`data`
|
||||
* carries arbitrary content and is not indexed element-wise); pass an embedding
|
||||
* as the first-class `vector` parameter, which is where a vector belongs; or
|
||||
* shorten the field to the values that are actually queried.
|
||||
*/
|
||||
export class MetadataArrayTooLargeError extends BrainyError {
|
||||
/** The metadata field whose array is too long (its full dotted address). */
|
||||
public readonly field: string
|
||||
/** How many elements that array holds. */
|
||||
public readonly length: number
|
||||
/** The bound it exceeded — {@link MAX_INDEXED_ARRAY_LENGTH}. */
|
||||
public readonly limit: number
|
||||
|
||||
constructor(site: string, field: string, length: number, limit: number) {
|
||||
super(
|
||||
`${site}: metadata field '${field}' holds ${length} array elements, ` +
|
||||
`over the ${limit}-element indexing bound. An array field indexes one ` +
|
||||
`posting per element, so an unbounded array is an unbounded write. ` +
|
||||
`This write is refused rather than indexed partially or skipped silently ` +
|
||||
`— a skipped field drops the row out of every filtered search on '${field}' ` +
|
||||
`with no way to tell that from "nothing matched". ` +
|
||||
`Cures: put the long array in 'data' (stored, not indexed element-wise); ` +
|
||||
`pass an embedding as the first-class 'vector' parameter; or keep only ` +
|
||||
`the values you actually query in '${field}'.`,
|
||||
'VALIDATION',
|
||||
false
|
||||
)
|
||||
this.name = 'MetadataArrayTooLargeError'
|
||||
this.field = field
|
||||
this.length = length
|
||||
this.limit = limit
|
||||
if (Error.captureStackTrace) {
|
||||
Error.captureStackTrace(this, MetadataArrayTooLargeError)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1105,13 +1105,31 @@ export class GraphAdjacencyIndex implements GraphIndexProvider {
|
|||
}
|
||||
|
||||
/**
|
||||
* Clean shutdown
|
||||
* Stop the auto-flush interval WITHOUT writing anything.
|
||||
*
|
||||
* The non-writing half of {@link close}, for a shutdown that must leave the
|
||||
* store byte-identical — a read-only brain's close. `close()` itself is a
|
||||
* writer: it drains both LSM MemTables to SSTables and stamps the watermark,
|
||||
* which is exactly right for a writer and forbidden for a reader. A reader
|
||||
* still has to release this interval, though: it is the one piece of this
|
||||
* index that outlives the close and could fire against a store the session no
|
||||
* longer owns.
|
||||
*
|
||||
* @returns Nothing.
|
||||
*/
|
||||
async close(): Promise<void> {
|
||||
stopBackgroundFlush(): void {
|
||||
if (this.flushTimer) {
|
||||
clearInterval(this.flushTimer)
|
||||
this.flushTimer = undefined
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Clean shutdown — drains both trees and stamps the watermark. THIS WRITES;
|
||||
* a read-only brain must call {@link stopBackgroundFlush} instead.
|
||||
*/
|
||||
async close(): Promise<void> {
|
||||
this.stopBackgroundFlush()
|
||||
|
||||
// Close both LSM-trees (will flush MemTables to SSTables)
|
||||
if (this.initialized) {
|
||||
|
|
|
|||
|
|
@ -203,7 +203,7 @@ export { EntityNotFoundError, RelationNotFoundError } from './errors/notFound.js
|
|||
|
||||
// Base error + typed migration-lock error — thrown by any data-plane call while a
|
||||
// brain runs its one-time 7.x→8.0 upgrade; catch to answer HTTP 503 + Retry-After.
|
||||
export { BrainyError, MigrationInProgressError, GraphIndexNotReadyError, MetadataIndexNotReadyError, VectorIndexNotReadyError, ProtectedArtifactError, DerivedArtifactMissingError } from './errors/brainyError.js'
|
||||
export { BrainyError, MigrationInProgressError, GraphIndexNotReadyError, MetadataIndexNotReadyError, VectorIndexNotReadyError, ProtectedArtifactError, DerivedArtifactMissingError, MetadataArrayTooLargeError, MAX_INDEXED_ARRAY_LENGTH } from './errors/brainyError.js'
|
||||
export type { BrainyErrorType } from './errors/brainyError.js'
|
||||
|
||||
// ============= 8.0 Db API — generational MVCC =============
|
||||
|
|
|
|||
|
|
@ -40,7 +40,7 @@ import {
|
|||
import { EntityIdMapper } from './entityIdMapper.js'
|
||||
import { RoaringBitmap32, roaringLibraryInitialize } from './roaring/index.js'
|
||||
import { FieldTypeInference, FieldType } from './fieldTypeInference.js'
|
||||
import { BrainyError } from '../errors/brainyError.js'
|
||||
import { BrainyError, MAX_INDEXED_ARRAY_LENGTH } from '../errors/brainyError.js'
|
||||
|
||||
/**
|
||||
* Fields whose values are stored in the sparse index as BUCKETED values
|
||||
|
|
@ -289,8 +289,10 @@ export class MetadataIndexManager implements MetadataIndexProvider {
|
|||
// No name-based exclude/allow lists — the field-addressing law: every
|
||||
// user field indexes, whatever its name ('content', 'data', 'id',
|
||||
// 'vector', … included). Bulk payloads are kept out by uniform value-
|
||||
// SHAPE rules in extractIndexableFields (arrays >10 never become
|
||||
// posting scalars; >100-char values index hashed), never by name.
|
||||
// SHAPE rules in extractIndexableFields (arrays longer than
|
||||
// MAX_INDEXED_ARRAY_LENGTH never become posting scalars, and the write
|
||||
// door refuses them by name; >100-char values index hashed), never by
|
||||
// field name.
|
||||
}
|
||||
|
||||
// Initialize metadata cache with similar config to search cache
|
||||
|
|
@ -961,9 +963,41 @@ export class MetadataIndexManager implements MetadataIndexProvider {
|
|||
}
|
||||
|
||||
/**
|
||||
* Get IDs for a range using chunked sparse index with zone maps and roaring bitmaps
|
||||
* Now fully lazy-loaded via UnifiedCache (no local sparseIndices Map)
|
||||
* Normalize min/max for timestamp bucketing before comparison
|
||||
* Get IDs for a range using the legacy chunked sparse index (zone maps +
|
||||
* roaring bitmaps). Lazy-loaded via UnifiedCache.
|
||||
*
|
||||
* ORDER IS NOT A KEY. This path compares NORMALIZED values, and
|
||||
* {@link normalizeValue} carries an escape hatch that is order-destroying by
|
||||
* design: a string over 100 characters is replaced by {@link hashValue}'s
|
||||
* digest so it can be used as a filesystem-safe key. Feeding that digest to
|
||||
* an ORDERING comparison — which is what a `gte` / `lt` / `between` does —
|
||||
* ranks rows by hash. The result is not empty and not an error: it is a
|
||||
* confidently ordered wrong answer, and it disagrees with the column-store
|
||||
* path (`getIdsForRange` above), which compares raw values and is correct.
|
||||
*
|
||||
* Two changes hold the line here:
|
||||
*
|
||||
* 1. THE BOUNDS ARE NEVER HASHED. They are normalized with `allowHash =
|
||||
* false`, so a long bound stays comparable instead of collapsing to a
|
||||
* digest. This alone fixes the common shape — a long bound queried
|
||||
* against ordinary short values, where the digest sorts below every
|
||||
* letter and `gte` therefore matched the entire store.
|
||||
*
|
||||
* 2. A HASHED KEY IS REFUSED, NEVER GUESSED. The persisted keys are whatever
|
||||
* the pre-7.20.0 writer normalized them to, so a field whose values ran
|
||||
* long is stored hashed and its order is simply not recoverable from this
|
||||
* index. Rather than compare digests, the query throws a typed
|
||||
* `BrainyError('INVALID_QUERY')` naming the field, the bound and the cure.
|
||||
* Loud beats wrong.
|
||||
*
|
||||
* KNOWN, NAMED DIVERGENCE. The persisted keys are also lower-cased and
|
||||
* trimmed by `normalizeValue`, so this path's string ranges are
|
||||
* CASE-INSENSITIVE where the column store's are not. That is a property of
|
||||
* the bytes a pre-7.20.0 engine wrote, not of the comparison: the raw values
|
||||
* are not in the index to compare. The bounds are normalized into the same
|
||||
* case-folded space so the comparison is at least self-consistent, and the
|
||||
* divergence disappears with the field itself once the column store adopts
|
||||
* it. See the module note on `getIdsFromChunks` for the path's lifetime.
|
||||
*/
|
||||
private async getIdsFromChunksForRange(
|
||||
field: string,
|
||||
|
|
@ -979,9 +1013,27 @@ export class MetadataIndexManager implements MetadataIndexProvider {
|
|||
}
|
||||
|
||||
// Normalize min/max for consistent comparison with indexed values
|
||||
// (indexed values are bucketed for timestamps, so we must bucket the query bounds too)
|
||||
const normalizedMin = min !== undefined ? this.normalizeValue(min, field) : undefined
|
||||
const normalizedMax = max !== undefined ? this.normalizeValue(max, field) : undefined
|
||||
// (indexed values are bucketed for timestamps, so we must bucket the query
|
||||
// bounds too) — but NEVER through the hash escape hatch, which would make
|
||||
// the bound incomparable. See the doc comment above.
|
||||
const normalizedMin = min !== undefined ? this.normalizeValue(min, field, false) : undefined
|
||||
const normalizedMax = max !== undefined ? this.normalizeValue(max, field, false) : undefined
|
||||
|
||||
// REFUSE BEFORE SELECTING. Chunk selection itself orders values: it tests
|
||||
// the bounds against each chunk's zone-map min/max. If those are hashes the
|
||||
// selection is already meaningless — and its failure mode is an EMPTY
|
||||
// answer (no chunk appears to overlap), which is the quietest wrong answer
|
||||
// of all. So the key space is checked here, before a single chunk is
|
||||
// chosen, and again per key below for a chunk whose zone map happens to
|
||||
// read clean.
|
||||
for (const chunkId of sparseIndex.getAllChunkIds()) {
|
||||
const zoneMap = sparseIndex.getChunk(chunkId)?.zoneMap
|
||||
for (const bound of [zoneMap?.min, zoneMap?.max]) {
|
||||
if (typeof bound === 'string' && MetadataIndexManager.isHashedValue(bound)) {
|
||||
throw MetadataIndexManager.rangeOverHashedIndex(field)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Find candidate chunks using zone maps
|
||||
const candidateChunkIds = sparseIndex.findChunksForRange(normalizedMin, normalizedMax)
|
||||
|
|
@ -996,6 +1048,13 @@ export class MetadataIndexManager implements MetadataIndexProvider {
|
|||
const chunk = await this.chunkManager.loadChunk(field, chunkId)
|
||||
if (chunk) {
|
||||
for (const [value, bitmap] of chunk.entries) {
|
||||
// A hashed key carries no order. Refuse the range rather than rank by
|
||||
// digest — the whole answer is unsound, so failing on the first one
|
||||
// is the honest outcome.
|
||||
if (MetadataIndexManager.isHashedValue(value)) {
|
||||
throw MetadataIndexManager.rangeOverHashedIndex(field)
|
||||
}
|
||||
|
||||
// Check if value is in range using numeric-aware comparison
|
||||
// (normalizeValue converts numbers to strings, so we must compare numerically)
|
||||
let inRange = true
|
||||
|
|
@ -1024,6 +1083,25 @@ export class MetadataIndexManager implements MetadataIndexProvider {
|
|||
return this.idMapper.intsIterableToUuids(allIntIds)
|
||||
}
|
||||
|
||||
/**
|
||||
* The refusal a range query gets when the legacy sparse index holds hashed
|
||||
* keys for the field. Names the field and the cure; never a wrong answer.
|
||||
*/
|
||||
private static rangeOverHashedIndex(field: string): BrainyError {
|
||||
return new BrainyError(
|
||||
`Range query on field "${field}" cannot be served by the legacy sparse index: ` +
|
||||
`its values were persisted as hashes (values over 100 characters are stored ` +
|
||||
`hashed to stay within filesystem name limits), and a hash carries no order — ` +
|
||||
`comparing them would return a confidently ordered wrong answer. ` +
|
||||
`Equality (\`where: { ${field}: value }\`) still works on this index. ` +
|
||||
`To range over this field, let the column store adopt it: run ` +
|
||||
`brain.repairIndex({ rebuild: ['metadata'] }), which rebuilds the field into ` +
|
||||
`the column store, where ranges compare raw values.`,
|
||||
'INVALID_QUERY',
|
||||
false
|
||||
)
|
||||
}
|
||||
|
||||
/**
|
||||
* Get roaring bitmap for a field-value pair without converting to UUIDs
|
||||
* This is used for fast multi-field intersection queries using hardware-accelerated bitmap AND
|
||||
|
|
@ -1191,8 +1269,17 @@ export class MetadataIndexManager implements MetadataIndexProvider {
|
|||
* value-based detection (DuckDB-inspired). Analyzes actual data values, not names.
|
||||
*
|
||||
* NO FALLBACKS - Pure value-based detection only.
|
||||
*
|
||||
* @param value - The value to normalize.
|
||||
* @param field - Optional field name (drives the per-field statistics strategy).
|
||||
* @param allowHash - Whether the >100-character escape hatch may fire. TRUE
|
||||
* everywhere a normalized value is used as a KEY (equality postings, chunk
|
||||
* entries, filenames) — that is what the hash exists for. FALSE on the
|
||||
* ORDER-comparing path: a hash is deliberately order-destroying, so a
|
||||
* bound that hashes can only be compared as nonsense. See
|
||||
* {@link isHashedValue} and `getIdsFromChunksForRange`.
|
||||
*/
|
||||
private normalizeValue(value: any, field?: string): string {
|
||||
private normalizeValue(value: any, field?: string, allowHash: boolean = true): string {
|
||||
if (value === null || value === undefined) return '__NULL__'
|
||||
if (typeof value === 'boolean') return value ? '__TRUE__' : '__FALSE__'
|
||||
|
||||
|
|
@ -1250,21 +1337,34 @@ export class MetadataIndexManager implements MetadataIndexProvider {
|
|||
// Default normalization
|
||||
if (typeof value === 'number') return value.toString()
|
||||
if (Array.isArray(value)) {
|
||||
const joined = value.map(v => this.normalizeValue(v, field)).join(',')
|
||||
const joined = value.map(v => this.normalizeValue(v, field, allowHash)).join(',')
|
||||
// Hash very long array values to avoid filesystem limits
|
||||
if (joined.length > 100) {
|
||||
if (allowHash && joined.length > 100) {
|
||||
return this.hashValue(joined)
|
||||
}
|
||||
return joined
|
||||
}
|
||||
const stringValue = String(value).toLowerCase().trim()
|
||||
// Hash very long string values to avoid filesystem limits
|
||||
if (stringValue.length > 100) {
|
||||
if (allowHash && stringValue.length > 100) {
|
||||
return this.hashValue(stringValue)
|
||||
}
|
||||
return stringValue
|
||||
}
|
||||
|
||||
/**
|
||||
* Is this normalized value a HASH rather than the value itself?
|
||||
*
|
||||
* {@link hashValue} is an escape hatch for filesystem name limits, and it is
|
||||
* deliberately order-destroying: two values whose hashes compare one way
|
||||
* routinely compare the other way themselves. Anything that ORDERS normalized
|
||||
* values has to know when it is holding one, because comparing hashes yields
|
||||
* a confident, wrong answer rather than an error.
|
||||
*/
|
||||
private static isHashedValue(normalized: string): boolean {
|
||||
return normalized.startsWith('__HASH_')
|
||||
}
|
||||
|
||||
/**
|
||||
* Create a short hash for long values to avoid filesystem filename limits
|
||||
*/
|
||||
|
|
@ -1289,9 +1389,10 @@ export class MetadataIndexManager implements MetadataIndexProvider {
|
|||
* 'content', 'vector' in a bag are ordinary user fields)
|
||||
* - Record-frame plumbing (vector, connections, level, data, _rev, id)
|
||||
* never indexes — that is namespace routing, not a name carve-out
|
||||
* - Value-SHAPE rules apply uniformly to all names: arrays >10 never
|
||||
* become posting scalars; purely numeric key names (array indices)
|
||||
* skip; >100-char values index hashed (normalizeValue)
|
||||
* - Value-SHAPE rules apply uniformly to all names: arrays longer than
|
||||
* MAX_INDEXED_ARRAY_LENGTH never become posting scalars (and say so —
|
||||
* the write door refuses them outright); purely numeric key names
|
||||
* (array indices) skip; >100-char values index hashed (normalizeValue)
|
||||
*/
|
||||
private extractIndexableFields(data: any): Array<{ field: string, value: any }> {
|
||||
const fields: Array<{ field: string, value: any }> = []
|
||||
|
|
@ -1353,13 +1454,37 @@ export class MetadataIndexManager implements MetadataIndexProvider {
|
|||
// This catches vectors stored as objects: {0: 0.1, 1: 0.2, ...}
|
||||
if (/^\d+$/.test(key)) continue
|
||||
|
||||
// Skip large arrays (> 10 elements) - likely vectors or bulk data
|
||||
if (Array.isArray(value) && value.length > 10) continue
|
||||
// THE INDEXABLE-ARRAY BOUND ({@link MAX_INDEXED_ARRAY_LENGTH}). An
|
||||
// array field mints one posting per element, so the index has always
|
||||
// carried a ceiling — it was 10, and it was applied by this bare
|
||||
// `continue`: an eleven-element `tags` array had its whole field
|
||||
// skipped and the row dropped out of every filtered search on it, with
|
||||
// no error, no warning, and nothing to distinguish that from "no row
|
||||
// matches". The ceiling is not the defect; the silence was.
|
||||
//
|
||||
// The write door refuses this shape by name now
|
||||
// (`MetadataArrayTooLargeError`, thrown from paramValidation's
|
||||
// `rejectOversizeIndexArrays`), so a live add/update never reaches
|
||||
// here over the bound. Reaching it means the row is ALREADY on disk —
|
||||
// written by an older engine under the old rule — and this is a
|
||||
// rebuild, a catch-up fold or a remove reading it back. Refusing there
|
||||
// would make an existing store un-rebuildable, so the row is admitted
|
||||
// and the skipped field is NARRATED instead. Never silent, either way.
|
||||
if (Array.isArray(value) && value.length > MAX_INDEXED_ARRAY_LENGTH) {
|
||||
prodLog.warn(
|
||||
`[brainy] metadata field '${fullKey}' holds ${value.length} array elements, ` +
|
||||
`over the ${MAX_INDEXED_ARRAY_LENGTH}-element indexing bound — the field is ` +
|
||||
`NOT indexed for this row, so it will not match a where-clause on '${fullKey}'. ` +
|
||||
`This row predates the bound (the write door refuses this shape now). ` +
|
||||
`Move the long array into 'data', or pass an embedding as the 'vector' parameter.`
|
||||
)
|
||||
continue
|
||||
}
|
||||
|
||||
if (value && typeof value === 'object' && !Array.isArray(value)) {
|
||||
// Recurse into nested objects (but not arrays), keeping the frame
|
||||
extract(value, fullKey, frame)
|
||||
} else if (Array.isArray(value) && value.length <= 10) {
|
||||
} else if (Array.isArray(value)) {
|
||||
// Small arrays: index as multi-value field (all with same field name)
|
||||
// Example: tags: ["javascript", "node"] → field="tags", value="javascript" + field="tags", value="node"
|
||||
for (const item of value) {
|
||||
|
|
|
|||
|
|
@ -18,6 +18,7 @@ import { findCallerLocation } from './callerLocation.js'
|
|||
import * as os from 'node:os'
|
||||
import * as fs from 'node:fs'
|
||||
import { parseFieldAddress, UnsupportedFindOptionError } from '../db/fieldAddressing.js'
|
||||
import { MAX_INDEXED_ARRAY_LENGTH, MetadataArrayTooLargeError } from '../errors/brainyError.js'
|
||||
|
||||
const getSystemMemory = (): number => {
|
||||
if (os) {
|
||||
|
|
@ -538,8 +539,53 @@ function rejectForgedSystemKeys(metadata: Record<string, unknown> | undefined, s
|
|||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* THE INDEXABLE-ARRAY BOUND, enforced at the write door.
|
||||
*
|
||||
* An array-valued metadata field indexes one posting per element, so the index
|
||||
* has always carried a ceiling. It used to be 10, and it was applied by a bare
|
||||
* `continue` deep inside field extraction: a row whose `tags` array held eleven
|
||||
* entries had that field skipped entirely and dropped out of every filtered
|
||||
* search on it — no error, no warning, and no way for the caller to tell the
|
||||
* difference from "no row matches". Silence is the defect; the ceiling is not.
|
||||
*
|
||||
* The bound is now {@link MAX_INDEXED_ARRAY_LENGTH}, high enough that every
|
||||
* legitimate multi-value field clears it, and it REFUSES here instead of
|
||||
* dropping data downstream. Refusing at the write door is what makes it
|
||||
* actionable: the caller learns at the moment of writing, with the field, the
|
||||
* length and the bound in hand.
|
||||
*
|
||||
* Scope is the caller's own metadata bag — the values that become postings.
|
||||
* Nested bags are walked, because a nested field indexes under its dotted
|
||||
* address exactly like a top-level one. Arrays of OBJECTS are not walked: the
|
||||
* index only ever makes postings from an array's scalar elements.
|
||||
*
|
||||
* @param metadata - The caller's metadata bag (undefined is fine).
|
||||
* @param site - The write door's name, for the message ('add()', 'update()', …).
|
||||
* @throws {MetadataArrayTooLargeError} Naming the field, its length and the bound.
|
||||
*/
|
||||
function rejectOversizeIndexArrays(metadata: Record<string, unknown> | undefined, site: string): void {
|
||||
if (!metadata) return
|
||||
|
||||
const walk = (bag: Record<string, unknown>, prefix: string): void => {
|
||||
for (const [key, value] of Object.entries(bag)) {
|
||||
const address = prefix ? `${prefix}.${key}` : key
|
||||
if (Array.isArray(value)) {
|
||||
if (value.length > MAX_INDEXED_ARRAY_LENGTH) {
|
||||
throw new MetadataArrayTooLargeError(site, address, value.length, MAX_INDEXED_ARRAY_LENGTH)
|
||||
}
|
||||
} else if (value && typeof value === 'object') {
|
||||
walk(value as Record<string, unknown>, address)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
walk(metadata, '')
|
||||
}
|
||||
|
||||
export function validateAddParams(params: AddParams): void {
|
||||
rejectForgedSystemKeys(params.metadata as Record<string, unknown> | undefined, 'add()')
|
||||
rejectOversizeIndexArrays(params.metadata as Record<string, unknown> | undefined, 'add()')
|
||||
// 'data' is ABSENT only when null/undefined — an empty string ('') is real
|
||||
// content (a legitimate empty file's first write) and must not be treated
|
||||
// as missing. Falsy-but-present values (0, false, '') all count as present;
|
||||
|
|
@ -608,6 +654,7 @@ export function validateAddParams(params: AddParams): void {
|
|||
*/
|
||||
export function validateUpdateParams(params: UpdateParams): void {
|
||||
rejectForgedSystemKeys(params.metadata as Record<string, unknown> | undefined, 'update()')
|
||||
rejectOversizeIndexArrays(params.metadata as Record<string, unknown> | undefined, 'update()')
|
||||
// Same absent-vs-empty distinction as validateAddParams: '' is a real new
|
||||
// value (e.g. truncating a file to empty content via overwrite), only
|
||||
// null/undefined means "no new data was given".
|
||||
|
|
@ -682,6 +729,7 @@ export function validateUpdateParams(params: UpdateParams): void {
|
|||
*/
|
||||
export function validateRelateParams(params: RelateParams): void {
|
||||
rejectForgedSystemKeys(params.metadata as Record<string, unknown> | undefined, 'relate()')
|
||||
rejectOversizeIndexArrays(params.metadata as Record<string, unknown> | undefined, 'relate()')
|
||||
// 8.0 verb-id contract (L.7): verb ids are UUIDs, generated by brainy.
|
||||
// RelateParams has no `id` field — an untyped caller passing one would
|
||||
// previously have it silently ignored (a generated UUID was used instead).
|
||||
|
|
@ -731,6 +779,7 @@ export function validateRelateParams(params: RelateParams): void {
|
|||
*/
|
||||
export function validateUpdateRelationParams(params: UpdateRelationParams): void {
|
||||
rejectForgedSystemKeys(params.metadata as Record<string, unknown> | undefined, 'updateRelation()')
|
||||
rejectOversizeIndexArrays(params.metadata as Record<string, unknown> | undefined, 'updateRelation()')
|
||||
if (!params.id) {
|
||||
throw new Error('id is required for updateRelation')
|
||||
}
|
||||
|
|
|
|||
244
tests/integration/find-orderby-every-path.test.ts
Normal file
244
tests/integration/find-orderby-every-path.test.ts
Normal file
|
|
@ -0,0 +1,244 @@
|
|||
/**
|
||||
* @module tests/integration/find-orderby-every-path
|
||||
* @description `orderBy` IS THE ORDER — on every find() path, not just the
|
||||
* metadata-only one.
|
||||
*
|
||||
* THE DEFECT. `find({ where, orderBy })` (metadata only) answered in field
|
||||
* order. `find({ query, where, orderBy })` and `find({ vector, where, orderBy })`
|
||||
* answered in SCORE order, silently: the vector/filter block ranked the fused
|
||||
* candidates by score, cut the page, and returned early — the tail's `orderBy`
|
||||
* sort sat below that early return and never ran. Nothing threw, nothing warned,
|
||||
* and the two paths disagreed about what "ordered by rank" means. A caller
|
||||
* paging `orderBy: 'rank', order: 'desc'` over a hybrid find got relevance
|
||||
* order wearing an ordering request's clothes.
|
||||
*
|
||||
* Where `connected` or `fusion` kept the tail alive the defect changed shape
|
||||
* rather than disappearing: the block had already CUT the page by score, so the
|
||||
* tail ordered the rows relevance had chosen instead of the rows the ordering
|
||||
* asks for — a correctly sorted page of the wrong rows.
|
||||
*
|
||||
* The early cut fires only once the candidate set reaches `offset + limit`
|
||||
* rows, which is why small fixtures never saw it: below that threshold the
|
||||
* block falls through and the tail's sort does apply. That is the whole shape
|
||||
* of the bug — an ordering that is correct until there is enough data to matter.
|
||||
*
|
||||
* THE LAW. An explicit `orderBy` displaces score as the ordering key on every
|
||||
* path. The candidate set the path produced is ordered IN FULL and the page is
|
||||
* cut from that ordering — the graph-first law's "page last", applied to
|
||||
* ordering rather than to filtering. Score-ranked early paging is for the
|
||||
* default (no `orderBy`) case only, where score IS the requested order.
|
||||
*
|
||||
* THE PIN. Differential, against the metadata-only path — the one path that
|
||||
* always honoured `orderBy`.
|
||||
*
|
||||
* WHAT THE DIFFERENTIAL CAN AND CANNOT CLAIM. `orderBy` orders the candidate
|
||||
* set; it does not enlarge it. The hybrid legs are bounded by construction (the
|
||||
* text leg and the beam walk each take `limit * 2`), so a differential against
|
||||
* the metadata-only path — whose universe is every matching row — is only
|
||||
* meaningful where those bounds provably cover the universe. The fixture is
|
||||
* sized so they do (12 rows, `limit` 6 → a `limit * 2` = 12-row text leg), and
|
||||
* the covering is ASSERTED from the leg's own output rather than assumed. This
|
||||
* pin is about ordering, and it says nothing about recall.
|
||||
*/
|
||||
import { describe, it, expect, beforeAll } from 'vitest'
|
||||
import { Brainy } from '../../src/brainy'
|
||||
import { NounType, VerbType } from '../../src/types/graphTypes'
|
||||
import { resolveEntityId } from '../../src/utils/idNormalization'
|
||||
|
||||
/** Embedding width of the default model — the row vectors must match it. */
|
||||
const DIM = 384
|
||||
|
||||
/** A deterministic, per-row-distinct unit vector (no embedder in the fixture). */
|
||||
function seededVector(seed: number): number[] {
|
||||
const v = new Array<number>(DIM)
|
||||
for (let i = 0; i < DIM; i++) {
|
||||
v[i] = Math.sin((i + 1) * 0.11 + seed * 0.37) * 0.5 + Math.cos((i + 1) * 0.05 + seed * 0.13) * 0.3
|
||||
}
|
||||
const magnitude = Math.sqrt(v.reduce((sum, x) => sum + x * x, 0))
|
||||
return v.map((x) => x / magnitude)
|
||||
}
|
||||
|
||||
/**
|
||||
* Ranks, shuffled — so no scoring order can reproduce them by luck, and the
|
||||
* ordering the pins assert is visibly not the insertion order either.
|
||||
*/
|
||||
const RANKS = [7, 3, 11, 1, 9, 5, 12, 2, 10, 4, 8, 6]
|
||||
const ROWS = RANKS.length
|
||||
/** The page size every pin uses: `limit * 2` covers the whole universe. */
|
||||
const LIMIT = 6
|
||||
/** The neighbour subset — the graph-first universe — and its own page size. */
|
||||
const NEIGHBOURS = 8
|
||||
const GRAPH_LIMIT = 4
|
||||
|
||||
describe('find(): orderBy is the order on every path', () => {
|
||||
let brain: Brainy<any>
|
||||
const QUERY = 'orbital telemetry'
|
||||
const anchor = 'ordering-anchor'
|
||||
const neighbourIds: string[] = []
|
||||
|
||||
beforeAll(async () => {
|
||||
brain = new Brainy({ requireSubtype: false, storage: { type: 'memory' } })
|
||||
await brain.init()
|
||||
|
||||
let seed = 1
|
||||
await brain.add({
|
||||
id: anchor,
|
||||
data: 'ground station anchor record',
|
||||
type: NounType.Thing,
|
||||
metadata: { lane: 'anchor', rank: 0 },
|
||||
vector: seededVector(seed++)
|
||||
})
|
||||
|
||||
for (let i = 0; i < ROWS; i++) {
|
||||
const id = `row-${i}`
|
||||
await brain.add({
|
||||
id,
|
||||
// EVERY row carries both query words, so the text leg reaches all of
|
||||
// them and the hybrid candidate set covers the whole universe.
|
||||
data: `orbital telemetry packet ${i} recorded downlink`,
|
||||
type: NounType.Document,
|
||||
metadata: { lane: 'alpha', rank: RANKS[i] },
|
||||
vector: seededVector(seed++)
|
||||
})
|
||||
if (i < NEIGHBOURS) {
|
||||
await brain.relate({ from: anchor, to: id, type: VerbType.RelatedTo })
|
||||
neighbourIds.push(resolveEntityId(id))
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
it('the fixture: the hybrid candidate set covers the whole filter universe', async () => {
|
||||
const universe: string[] = await (brain as any).filterIdsBelted({ lane: 'alpha' })
|
||||
expect(universe).toHaveLength(ROWS)
|
||||
|
||||
// The text leg is bounded at `limit * 2`; the fixture is sized so that
|
||||
// bound reaches every row in the universe. This is the precondition the
|
||||
// differential below rests on — asserted from the leg itself.
|
||||
const textScored = await (brain as any).executeTextSearchScored(QUERY, LIMIT * 2, universe)
|
||||
expect(textScored).toHaveLength(ROWS)
|
||||
|
||||
// And the candidate set is large enough to trigger the score-ranked early
|
||||
// cut this pin exists to keep out of an ordered query's way.
|
||||
expect(ROWS).toBeGreaterThanOrEqual(LIMIT)
|
||||
})
|
||||
|
||||
it('metadata-only + orderBy: the reference ordering', async () => {
|
||||
const rows = await brain.find({
|
||||
where: { lane: 'alpha' },
|
||||
orderBy: 'rank',
|
||||
order: 'desc',
|
||||
limit: LIMIT
|
||||
} as any)
|
||||
expect(rows.map((r: any) => r.metadata.rank)).toEqual([12, 11, 10, 9, 8, 7])
|
||||
})
|
||||
|
||||
it('hybrid (query + where) + orderBy: the same page as the metadata-only path', async () => {
|
||||
const params = { where: { lane: 'alpha' }, orderBy: 'rank', order: 'desc' as const, limit: LIMIT }
|
||||
const expected = await brain.find(params as any)
|
||||
const actual = await brain.find({ ...params, query: QUERY } as any)
|
||||
|
||||
expect(actual).toHaveLength(expected.length)
|
||||
expect(actual.map((r: any) => r.id)).toEqual(expected.map((r: any) => r.id))
|
||||
expect(actual.map((r: any) => r.metadata.rank)).toEqual([12, 11, 10, 9, 8, 7])
|
||||
})
|
||||
|
||||
it('hybrid + orderBy asc: the ordering key is honoured in both directions', async () => {
|
||||
const params = { where: { lane: 'alpha' }, orderBy: 'rank', order: 'asc' as const, limit: LIMIT }
|
||||
const expected = await brain.find(params as any)
|
||||
const actual = await brain.find({ ...params, query: QUERY } as any)
|
||||
|
||||
expect(actual.map((r: any) => r.id)).toEqual(expected.map((r: any) => r.id))
|
||||
expect(actual.map((r: any) => r.metadata.rank)).toEqual([1, 2, 3, 4, 5, 6])
|
||||
})
|
||||
|
||||
it('hybrid + orderBy + offset: page two is page two of the ORDERING', async () => {
|
||||
const params = {
|
||||
where: { lane: 'alpha' },
|
||||
orderBy: 'rank',
|
||||
order: 'desc' as const,
|
||||
limit: LIMIT,
|
||||
offset: LIMIT
|
||||
}
|
||||
const expected = await brain.find(params as any)
|
||||
const actual = await brain.find({ ...params, query: QUERY } as any)
|
||||
|
||||
expect(actual).toHaveLength(LIMIT)
|
||||
expect(actual.map((r: any) => r.id)).toEqual(expected.map((r: any) => r.id))
|
||||
expect(actual.map((r: any) => r.metadata.rank)).toEqual([6, 5, 4, 3, 2, 1])
|
||||
})
|
||||
|
||||
it('hybrid + orderBy: paging walks the ordering monotonically, no row twice', async () => {
|
||||
const seen: number[] = []
|
||||
for (let offset = 0; offset < ROWS; offset += LIMIT) {
|
||||
const page = await brain.find({
|
||||
query: QUERY,
|
||||
where: { lane: 'alpha' },
|
||||
orderBy: 'rank',
|
||||
order: 'desc',
|
||||
limit: LIMIT,
|
||||
offset
|
||||
} as any)
|
||||
seen.push(...page.map((r: any) => r.metadata.rank))
|
||||
}
|
||||
expect(seen).toHaveLength(ROWS)
|
||||
expect(new Set(seen).size).toBe(ROWS)
|
||||
// Strictly descending across every page boundary.
|
||||
for (let i = 1; i < seen.length; i++) expect(seen[i]).toBeLessThan(seen[i - 1])
|
||||
})
|
||||
|
||||
it('vector + where + orderBy: field order, not distance order', async () => {
|
||||
// The beam walk takes `limit * 2` = the whole universe here, so the page is
|
||||
// the true top of the ordering — which distance order cannot produce.
|
||||
const rows = await brain.find({
|
||||
vector: seededVector(1000),
|
||||
where: { lane: 'alpha' },
|
||||
orderBy: 'rank',
|
||||
order: 'desc',
|
||||
limit: LIMIT
|
||||
} as any)
|
||||
expect(rows.map((r: any) => r.metadata.rank)).toEqual([12, 11, 10, 9, 8, 7])
|
||||
})
|
||||
|
||||
it('graph-first (query + connected + where) + orderBy: the neighbour set, ordered', async () => {
|
||||
const actual = await brain.find({
|
||||
query: QUERY,
|
||||
connected: { from: anchor, direction: 'out' as const },
|
||||
where: { lane: 'alpha' },
|
||||
orderBy: 'rank',
|
||||
order: 'desc',
|
||||
limit: GRAPH_LIMIT
|
||||
} as any)
|
||||
|
||||
expect(actual).toHaveLength(GRAPH_LIMIT)
|
||||
const neighbours = new Set(neighbourIds)
|
||||
for (const r of actual) expect(neighbours.has(r.id)).toBe(true)
|
||||
|
||||
// The ordering covers the whole neighbour set, so the page holds the
|
||||
// highest ranks AMONG THE NEIGHBOURS — not the ones the score ranking
|
||||
// happened to surface first and the tail then sorted among themselves.
|
||||
const expectedRanks = RANKS.slice(0, NEIGHBOURS)
|
||||
.sort((a, b) => b - a)
|
||||
.slice(0, GRAPH_LIMIT)
|
||||
expect(expectedRanks).toEqual([12, 11, 9, 7])
|
||||
expect(actual.map((r: any) => r.metadata.rank)).toEqual(expectedRanks)
|
||||
})
|
||||
|
||||
it('fusion + orderBy: the ordering survives the fusion rescore', async () => {
|
||||
const actual = await brain.find({
|
||||
query: QUERY,
|
||||
where: { lane: 'alpha' },
|
||||
fusion: 'weighted',
|
||||
orderBy: 'rank',
|
||||
order: 'desc',
|
||||
limit: LIMIT
|
||||
} as any)
|
||||
expect(actual.map((r: any) => r.metadata.rank)).toEqual([12, 11, 10, 9, 8, 7])
|
||||
})
|
||||
|
||||
it('no orderBy: score order still stands (the default is untouched)', async () => {
|
||||
const rows = await brain.find({ query: QUERY, where: { lane: 'alpha' }, limit: LIMIT } as any)
|
||||
expect(rows).toHaveLength(LIMIT)
|
||||
const scores = rows.map((r: any) => r.score)
|
||||
for (let i = 1; i < scores.length; i++) expect(scores[i]).toBeLessThanOrEqual(scores[i - 1])
|
||||
})
|
||||
})
|
||||
|
|
@ -26,6 +26,7 @@ import { describe, it, expect, beforeEach, afterEach } from 'vitest'
|
|||
import { Brainy } from '../../src/brainy.js'
|
||||
import { NounType } from '../../src/types/graphTypes.js'
|
||||
import { existsSync, rmSync } from 'fs'
|
||||
import { MetadataArrayTooLargeError, MAX_INDEXED_ARRAY_LENGTH } from '../../src/errors/brainyError.js'
|
||||
|
||||
describe('Metadata Vector Exclusion Fix', () => {
|
||||
let brainy: Brainy
|
||||
|
|
@ -155,29 +156,56 @@ describe('Metadata Vector Exclusion Fix', () => {
|
|||
expect(results[0].entity.metadata?.name).toBe('Bob')
|
||||
})
|
||||
|
||||
it('should skip indexing large arrays (>10 elements)', async () => {
|
||||
// Add entity with a large array (not a vector, just bulk data).
|
||||
it('should REFUSE an array over the indexing bound, by name', async () => {
|
||||
// A large array (not a vector, just bulk data). This used to be SKIPPED in
|
||||
// silence at a bound of 10 — the field simply vanished from the index and
|
||||
// the row dropped out of every `where` on it, indistinguishably from "no
|
||||
// row matches". The bound is now MAX_INDEXED_ARRAY_LENGTH and it REFUSES.
|
||||
const largeArray = Array.from({ length: 100 }, (_, i) => `item${i}`)
|
||||
|
||||
await brainy.add({
|
||||
type: NounType.Document,
|
||||
data: 'Doc with large array',
|
||||
metadata: {
|
||||
name: 'Doc with large array',
|
||||
items: largeArray
|
||||
}
|
||||
})
|
||||
const err = await brainy
|
||||
.add({
|
||||
type: NounType.Document,
|
||||
data: 'Doc with large array',
|
||||
metadata: {
|
||||
name: 'Doc with large array',
|
||||
items: largeArray
|
||||
}
|
||||
})
|
||||
.catch((e: any) => e)
|
||||
|
||||
// Large arrays (> 10 elements) are deliberately skipped to avoid indexing
|
||||
// bulk/vector-like payloads: 'items' must NOT appear, and the 100 elements
|
||||
// must NOT have produced 100 indexed fields.
|
||||
expect(err).toBeInstanceOf(MetadataArrayTooLargeError)
|
||||
expect(err.field).toBe('items')
|
||||
expect(err.length).toBe(100)
|
||||
expect(err.limit).toBe(MAX_INDEXED_ARRAY_LENGTH)
|
||||
|
||||
// Nothing was indexed from the refused write — no 'items' field, and above
|
||||
// all no per-element numeric fields (the original explosion class).
|
||||
const fields = await brainy.getAvailableFields()
|
||||
expect(fields).not.toContain('items')
|
||||
const numericFields = fields.filter(f => /(^|\.)\d+$/.test(f))
|
||||
expect(numericFields).toEqual([])
|
||||
})
|
||||
|
||||
// The scalar 'name' field IS indexed.
|
||||
expect(fields).toContain('name')
|
||||
it('should index an array UP TO the bound — the old limit of 10 was the bug', async () => {
|
||||
await brainy.add({
|
||||
type: NounType.Document,
|
||||
data: 'Doc with a long-but-legitimate tag list',
|
||||
metadata: {
|
||||
name: 'Doc with many tags',
|
||||
items: Array.from({ length: MAX_INDEXED_ARRAY_LENGTH }, (_, i) => `item${i}`)
|
||||
}
|
||||
})
|
||||
|
||||
const fields = await brainy.getAvailableFields()
|
||||
// The field IS indexed now, and still without per-element numeric fields.
|
||||
expect(fields).toContain('items')
|
||||
expect(fields.filter(f => /(^|\.)\d+$/.test(f))).toEqual([])
|
||||
|
||||
// And the eleventh element — the one the old bound silently dropped the
|
||||
// whole field for — really is searchable.
|
||||
const hits = await brainy.find({ where: { items: 'item10' } })
|
||||
expect(hits.length).toBeGreaterThan(0)
|
||||
})
|
||||
|
||||
it('should preserve HNSW vector search functionality', async () => {
|
||||
|
|
|
|||
|
|
@ -149,12 +149,12 @@ describe('a read-only brain writes no clean-shutdown evidence', () => {
|
|||
brain = null
|
||||
|
||||
// The FILE SET under `_system/` is unchanged — a reader creates and
|
||||
// removes nothing. (Other files under `_system/` — e.g. the metadata
|
||||
// field registry, which stamps its own `lastUpdated` on every persist —
|
||||
// are a pre-existing, separate concern outside this fix's scope: this
|
||||
// pin is specifically about the generation store's clean-shutdown
|
||||
// evidence, not about every subsystem's close() being a true no-op for
|
||||
// a reader.)
|
||||
// removes nothing. This pin is specifically about the generation store's
|
||||
// clean-shutdown evidence. The wider law — that a reader leaves EVERY
|
||||
// file under `_system/` byte-identical, which this fix left open as a
|
||||
// known residual (the metadata field registry and the three statistics
|
||||
// files were still re-stamped by a reader's close) — is closed and pinned
|
||||
// in `readonly-close-writes-nothing.test.ts`.
|
||||
const after = snapshotDir(systemDir())
|
||||
expect([...after.keys()].sort()).toEqual([...before.keys()].sort())
|
||||
|
||||
|
|
|
|||
261
tests/integration/readonly-close-writes-nothing.test.ts
Normal file
261
tests/integration/readonly-close-writes-nothing.test.ts
Normal file
|
|
@ -0,0 +1,261 @@
|
|||
/**
|
||||
* @module tests/integration/readonly-close-writes-nothing
|
||||
* @description A READ-ONLY BRAIN LEAVES `_system/` BYTE-IDENTICAL — the WHOLE
|
||||
* directory, not just the clean-shutdown marker.
|
||||
*
|
||||
* `readonly-close-no-marker` closed the marker half of this law and named the
|
||||
* rest as a known, out-of-scope residual:
|
||||
*
|
||||
* "Other files under `_system/` — e.g. the metadata field registry, which
|
||||
* stamps its own `lastUpdated` on every persist — are a pre-existing,
|
||||
* separate concern outside this fix's scope."
|
||||
*
|
||||
* This is that residual, closed. MEASURED on the base before the fix, a
|
||||
* read-only open → read → close rewrote FOUR files:
|
||||
*
|
||||
* _system/__metadata_field_registry__.json.gz
|
||||
* _system/type-statistics.json.gz
|
||||
* _system/subtype-statistics.json.gz
|
||||
* _system/verb-subtype-statistics.json.gz
|
||||
*
|
||||
* THE CAUSE was not the closes the marker fix guarded — it was Phase 1 of
|
||||
* `closeDurableSteps`, where every component flush ran unconditionally. A flush
|
||||
* is a write by definition: `MetadataIndexManager#flush()` saves the field
|
||||
* registry "even with no dirty fields" (its own comment), and the storage
|
||||
* adapter's count flush re-stamps the three statistics files. A session that
|
||||
* committed nothing re-stamped all four. Phase 2's closes were ungated too —
|
||||
* the graph index's close drains both LSM MemTables and stamps a watermark,
|
||||
* and the optional vector/metadata `close` hooks (unimplemented in the
|
||||
* reference engine, filled in by a native provider) persist buffered state.
|
||||
*
|
||||
* THE LAW. A reader writes nothing, anywhere under `_system/`, at open or at
|
||||
* close. It still RELEASES what it holds: the graph index's auto-flush interval
|
||||
* is cleared through `stopBackgroundFlush()`, the non-writing half of its
|
||||
* close, so nothing outlives the session.
|
||||
*
|
||||
* WHY IT MATTERS beyond tidiness: `_system/` is where a store keeps its
|
||||
* evidence about itself — what the writer committed, what the projections have
|
||||
* seen. A reader that rewrites any of it is vouching for a state it only
|
||||
* observed, and on shared or snapshot storage it mutates bytes another process
|
||||
* owns.
|
||||
*/
|
||||
|
||||
import { describe, it, expect, beforeEach, afterEach } from 'vitest'
|
||||
import { mkdtempSync, rmSync, readdirSync, readFileSync, statSync } from 'node:fs'
|
||||
import { createHash } from 'node:crypto'
|
||||
import { tmpdir } from 'node:os'
|
||||
import { join } from 'node:path'
|
||||
import { Brainy } from '../../src/brainy.js'
|
||||
import { NounType, VerbType } from '../../src/types/graphTypes.js'
|
||||
|
||||
/** Recursively hash every regular file under `dir`, keyed by its path relative to `dir`. */
|
||||
function snapshotDir(dir: string): Map<string, string> {
|
||||
const out = new Map<string, string>()
|
||||
const walk = (rel: string): void => {
|
||||
const abs = rel ? join(dir, rel) : dir
|
||||
let entries: string[]
|
||||
try {
|
||||
entries = readdirSync(abs)
|
||||
} catch {
|
||||
return
|
||||
}
|
||||
for (const name of entries) {
|
||||
const childRel = rel ? join(rel, name) : name
|
||||
const childAbs = join(dir, childRel)
|
||||
const st = statSync(childAbs)
|
||||
if (st.isDirectory()) {
|
||||
walk(childRel)
|
||||
} else if (st.isFile()) {
|
||||
out.set(childRel, createHash('sha256').update(readFileSync(childAbs)).digest('hex'))
|
||||
}
|
||||
}
|
||||
}
|
||||
walk('')
|
||||
return out
|
||||
}
|
||||
|
||||
/** Every path where `after` differs from `before`, labelled — the failure message. */
|
||||
function diff(before: Map<string, string>, after: Map<string, string>): string[] {
|
||||
const lines: string[] = []
|
||||
for (const [path, hash] of after) {
|
||||
if (!before.has(path)) lines.push(`ADDED ${path}`)
|
||||
else if (before.get(path) !== hash) lines.push(`CHANGED ${path}`)
|
||||
}
|
||||
for (const path of before.keys()) if (!after.has(path)) lines.push(`REMOVED ${path}`)
|
||||
return lines.sort()
|
||||
}
|
||||
|
||||
describe('a read-only brain writes nothing under `_system/`', () => {
|
||||
let dir: string
|
||||
let brain: Brainy | null = null
|
||||
|
||||
beforeEach(() => {
|
||||
dir = mkdtempSync(join(tmpdir(), 'brainy-readonly-writes-'))
|
||||
})
|
||||
|
||||
afterEach(async () => {
|
||||
if (brain) {
|
||||
try {
|
||||
await brain.close()
|
||||
} catch {
|
||||
/* already closed */
|
||||
}
|
||||
brain = null
|
||||
}
|
||||
try {
|
||||
rmSync(dir, { recursive: true, force: true })
|
||||
} catch {
|
||||
/* ignore */
|
||||
}
|
||||
})
|
||||
|
||||
const systemDir = () => join(dir, '_system')
|
||||
|
||||
/**
|
||||
* A writer seeds a store with nouns, verbs and queryable metadata — enough
|
||||
* that the field registry, the statistics files and the graph index all hold
|
||||
* real content — then closes cleanly.
|
||||
*/
|
||||
async function seedStore(): Promise<void> {
|
||||
const writer = new Brainy({
|
||||
requireSubtype: false,
|
||||
storage: { type: 'filesystem', path: dir }
|
||||
})
|
||||
await writer.init()
|
||||
for (let i = 0; i < 6; i++) {
|
||||
await writer.add({
|
||||
id: `seed-${i}`,
|
||||
data: `seed entity ${i}`,
|
||||
type: i % 2 === 0 ? NounType.Concept : NounType.Document,
|
||||
metadata: { lane: i % 2 === 0 ? 'alpha' : 'beta', rank: i, tags: [`t${i}`, 'shared'] },
|
||||
vector: []
|
||||
})
|
||||
}
|
||||
for (let i = 1; i < 6; i++) {
|
||||
await writer.relate({ from: 'seed-0', to: `seed-${i}`, type: VerbType.RelatedTo })
|
||||
}
|
||||
await writer.flush()
|
||||
await writer.close()
|
||||
}
|
||||
|
||||
it('open → read → close leaves every file under `_system/` byte-identical', async () => {
|
||||
await seedStore()
|
||||
|
||||
const before = snapshotDir(systemDir())
|
||||
expect(before.size, 'the writer left a populated `_system/`').toBeGreaterThan(0)
|
||||
|
||||
brain = await Brainy.openReadOnly({ storage: { type: 'filesystem', path: dir } })
|
||||
expect(brain.isReadOnly).toBe(true)
|
||||
|
||||
// Exercise the read surface that drives each subsystem: statistics (counts),
|
||||
// a metadata filter (field index + registry), a graph walk (adjacency), a
|
||||
// vector search, and a direct get.
|
||||
await brain.stats()
|
||||
await brain.find({ where: { lane: 'alpha' }, limit: 10 } as any)
|
||||
await brain.find({ where: { tags: 'shared' }, limit: 10 } as any)
|
||||
await brain.find({ connected: { from: 'seed-0', direction: 'out' }, limit: 10 } as any)
|
||||
await brain.get('seed-1')
|
||||
|
||||
await brain.close()
|
||||
brain = null
|
||||
|
||||
const after = snapshotDir(systemDir())
|
||||
const changes = diff(before, after)
|
||||
expect(changes, `a reader modified \`_system/\`:\n${changes.join('\n')}`).toEqual([])
|
||||
}, 120_000)
|
||||
|
||||
it('names the four files that used to change — the measured shape of the defect', async () => {
|
||||
await seedStore()
|
||||
const before = snapshotDir(systemDir())
|
||||
|
||||
// These are the exact paths the base rewrote. Naming them keeps the pin
|
||||
// honest about what it caught: if a future change reintroduces the write,
|
||||
// the test above fails and this one says which subsystem did it.
|
||||
const previouslyRewritten = [
|
||||
'__metadata_field_registry__.json.gz',
|
||||
'type-statistics.json.gz',
|
||||
'subtype-statistics.json.gz',
|
||||
'verb-subtype-statistics.json.gz'
|
||||
]
|
||||
for (const name of previouslyRewritten) {
|
||||
expect(before.has(name), `fixture must contain ${name}`).toBe(true)
|
||||
}
|
||||
|
||||
brain = await Brainy.openReadOnly({ storage: { type: 'filesystem', path: dir } })
|
||||
await brain.stats()
|
||||
await brain.find({ where: { lane: 'alpha' }, limit: 10 } as any)
|
||||
await brain.close()
|
||||
brain = null
|
||||
|
||||
const after = snapshotDir(systemDir())
|
||||
for (const name of previouslyRewritten) {
|
||||
expect(after.get(name), `${name} was rewritten by a reader`).toBe(before.get(name))
|
||||
}
|
||||
}, 120_000)
|
||||
|
||||
it('a reader that only opens and closes — touching nothing — writes nothing', async () => {
|
||||
await seedStore()
|
||||
const before = snapshotDir(systemDir())
|
||||
|
||||
brain = await Brainy.openReadOnly({ storage: { type: 'filesystem', path: dir } })
|
||||
await brain.close()
|
||||
brain = null
|
||||
|
||||
const changes = diff(before, snapshotDir(systemDir()))
|
||||
expect(changes, `an idle reader modified \`_system/\`:\n${changes.join('\n')}`).toEqual([])
|
||||
}, 120_000)
|
||||
|
||||
it('two readers in sequence each leave the store exactly as they found it', async () => {
|
||||
await seedStore()
|
||||
const before = snapshotDir(systemDir())
|
||||
|
||||
for (let i = 0; i < 2; i++) {
|
||||
const reader = await Brainy.openReadOnly({ storage: { type: 'filesystem', path: dir } })
|
||||
await reader.find({ where: { lane: 'beta' }, limit: 10 } as any)
|
||||
await reader.close()
|
||||
const changes = diff(before, snapshotDir(systemDir()))
|
||||
expect(changes, `reader ${i + 1} modified \`_system/\`:\n${changes.join('\n')}`).toEqual([])
|
||||
}
|
||||
}, 120_000)
|
||||
|
||||
it('the store outside `_system/` is untouched too — a reader writes nowhere', async () => {
|
||||
await seedStore()
|
||||
const before = snapshotDir(dir)
|
||||
|
||||
brain = await Brainy.openReadOnly({ storage: { type: 'filesystem', path: dir } })
|
||||
await brain.stats()
|
||||
await brain.find({ where: { lane: 'alpha' }, limit: 10 } as any)
|
||||
await brain.close()
|
||||
brain = null
|
||||
|
||||
const changes = diff(before, snapshotDir(dir))
|
||||
expect(changes, `a reader modified the store:\n${changes.join('\n')}`).toEqual([])
|
||||
}, 120_000)
|
||||
|
||||
it('a WRITER still persists on close — the guard did not disarm the write path', async () => {
|
||||
await seedStore()
|
||||
const before = snapshotDir(systemDir())
|
||||
|
||||
const writer = new Brainy({ requireSubtype: false, storage: { type: 'filesystem', path: dir } })
|
||||
await writer.init()
|
||||
await writer.add({
|
||||
id: 'after-reader',
|
||||
data: 'a new row',
|
||||
type: NounType.Concept,
|
||||
metadata: { lane: 'gamma', rank: 99 },
|
||||
vector: []
|
||||
})
|
||||
await writer.close()
|
||||
|
||||
// The writer's close DID move `_system/` — that is the whole point of the
|
||||
// asymmetry, and the guard must not have flattened it.
|
||||
expect(diff(before, snapshotDir(systemDir())).length).toBeGreaterThan(0)
|
||||
|
||||
// And the row is really there on the next open.
|
||||
const reopened = new Brainy({ requireSubtype: false, storage: { type: 'filesystem', path: dir } })
|
||||
await reopened.init()
|
||||
brain = reopened
|
||||
const hits = await reopened.find({ where: { lane: 'gamma' }, limit: 10 } as any)
|
||||
expect(hits.length).toBe(1)
|
||||
}, 120_000)
|
||||
})
|
||||
242
tests/unit/utils/metadataIndex-array-bound.test.ts
Normal file
242
tests/unit/utils/metadataIndex-array-bound.test.ts
Normal file
|
|
@ -0,0 +1,242 @@
|
|||
/**
|
||||
* @module tests/unit/utils/metadataIndex-array-bound
|
||||
* @description THE INDEXABLE-ARRAY BOUND — a law with a name and a refusal,
|
||||
* not a `continue`.
|
||||
*
|
||||
* THE DEFECT. An array-valued metadata field indexes one posting per element,
|
||||
* so the index has always carried a ceiling. It was 10, and it was applied by a
|
||||
* bare `continue` deep inside field extraction:
|
||||
*
|
||||
* if (Array.isArray(value) && value.length > 10) continue
|
||||
*
|
||||
* A row whose `tags` array held ELEVEN entries therefore had that field skipped
|
||||
* entirely — no posting, no error, no warning. The row then failed to match
|
||||
* every filtered search on `tags`, including `{ tags: 'a-tag-it-really-has' }`,
|
||||
* and the caller had no way to tell that from "no row matches". Eleven tags is
|
||||
* not an exotic shape; the eleventh tag made the row invisible.
|
||||
*
|
||||
* THE LAW. Arrays of scalars index up to {@link MAX_INDEXED_ARRAY_LENGTH} = 64,
|
||||
* hardcoded (the zero-config law: no knob), which clears every legitimate
|
||||
* multi-value field and stays far below any embedding width. Above it the WRITE
|
||||
* IS REFUSED by name — `MetadataArrayTooLargeError`, carrying the field, the
|
||||
* length and the bound — at `add`, `update`, `relate` and `updateRelation`
|
||||
* alike. Nothing is skipped in silence.
|
||||
*
|
||||
* THE ONE PLACE THE BOUND STILL SKIPS is a row already on disk, written by an
|
||||
* older engine under the old rule and read back by a rebuild, a catch-up fold
|
||||
* or a remove. Refusing there would make an existing store un-rebuildable — so
|
||||
* the row is admitted and the skipped field is NARRATED. Both sides are pinned.
|
||||
*/
|
||||
import { describe, it, expect, beforeEach, vi, afterEach } from 'vitest'
|
||||
import { Brainy } from '../../../src/brainy'
|
||||
import { NounType, VerbType } from '../../../src/types/graphTypes'
|
||||
import { MetadataArrayTooLargeError, MAX_INDEXED_ARRAY_LENGTH } from '../../../src/errors/brainyError'
|
||||
import { resolveEntityId } from '../../../src/utils/idNormalization'
|
||||
import { prodLog } from '../../../src/utils/logger'
|
||||
|
||||
/** `n` distinct scalar tags. */
|
||||
function tags(n: number, prefix = 't'): string[] {
|
||||
return Array.from({ length: n }, (_, i) => `${prefix}${i}`)
|
||||
}
|
||||
|
||||
describe('the indexable-array bound', () => {
|
||||
let brain: Brainy<any>
|
||||
|
||||
beforeEach(async () => {
|
||||
brain = new Brainy({ requireSubtype: false, storage: { type: 'memory' } })
|
||||
await brain.init()
|
||||
})
|
||||
|
||||
describe('BELOW the bound: the array indexes, every element of it', () => {
|
||||
it('the eleven-element array that used to vanish is searchable', async () => {
|
||||
// ELEVEN — one over the old silent limit, the whole shape of the defect.
|
||||
await brain.add({
|
||||
id: 'eleven',
|
||||
data: 'a row with eleven tags',
|
||||
type: NounType.Document,
|
||||
metadata: { tags: tags(11) },
|
||||
vector: []
|
||||
})
|
||||
|
||||
// Every element is a posting, including the eleventh.
|
||||
for (const tag of tags(11)) {
|
||||
const hits = await brain.find({ where: { tags: tag }, limit: 10 } as any)
|
||||
expect(hits.map((r: any) => r.id)).toContain(resolveEntityId('eleven'))
|
||||
}
|
||||
})
|
||||
|
||||
it('indexes right up to the bound — all 64 elements', async () => {
|
||||
await brain.add({
|
||||
id: 'at-bound',
|
||||
data: 'a row at the bound',
|
||||
type: NounType.Document,
|
||||
metadata: { tags: tags(MAX_INDEXED_ARRAY_LENGTH) },
|
||||
vector: []
|
||||
})
|
||||
|
||||
// The first, the last, and one in the middle.
|
||||
for (const tag of ['t0', `t${MAX_INDEXED_ARRAY_LENGTH - 1}`, 't31']) {
|
||||
const hits = await brain.find({ where: { tags: tag }, limit: 10 } as any)
|
||||
expect(hits.map((r: any) => r.id)).toContain(resolveEntityId('at-bound'))
|
||||
}
|
||||
})
|
||||
|
||||
it('a nested bag\'s array indexes under its dotted address', async () => {
|
||||
await brain.add({
|
||||
id: 'nested',
|
||||
data: 'a row with a nested tag list',
|
||||
type: NounType.Document,
|
||||
metadata: { facets: { labels: tags(20, 'l') } },
|
||||
vector: []
|
||||
})
|
||||
const hits = await brain.find({ where: { 'facets.labels': 'l19' }, limit: 10 } as any)
|
||||
expect(hits.map((r: any) => r.id)).toContain(resolveEntityId('nested'))
|
||||
})
|
||||
})
|
||||
|
||||
describe('ABOVE the bound: the write is refused, by name', () => {
|
||||
const OVER = MAX_INDEXED_ARRAY_LENGTH + 1
|
||||
|
||||
it('add() throws a typed error naming the field, the length and the bound', async () => {
|
||||
const err = await brain
|
||||
.add({
|
||||
id: 'too-many',
|
||||
data: 'a row with too many tags',
|
||||
type: NounType.Document,
|
||||
metadata: { tags: tags(OVER) },
|
||||
vector: []
|
||||
} as any)
|
||||
.catch((e: any) => e)
|
||||
|
||||
expect(err).toBeInstanceOf(MetadataArrayTooLargeError)
|
||||
expect(err.field).toBe('tags')
|
||||
expect(err.length).toBe(OVER)
|
||||
expect(err.limit).toBe(MAX_INDEXED_ARRAY_LENGTH)
|
||||
expect(err.type).toBe('VALIDATION')
|
||||
// The message carries all three, and names the cures.
|
||||
expect(err.message).toContain('tags')
|
||||
expect(err.message).toContain(String(OVER))
|
||||
expect(err.message).toContain(String(MAX_INDEXED_ARRAY_LENGTH))
|
||||
expect(err.message).toContain('vector')
|
||||
})
|
||||
|
||||
it('the refused row is not written at all — no half-indexed ghost', async () => {
|
||||
await expect(
|
||||
brain.add({
|
||||
id: 'refused',
|
||||
data: 'refused',
|
||||
type: NounType.Document,
|
||||
metadata: { tags: tags(OVER) },
|
||||
vector: []
|
||||
} as any)
|
||||
).rejects.toBeInstanceOf(MetadataArrayTooLargeError)
|
||||
|
||||
expect(await brain.get('refused')).toBeNull()
|
||||
const hits = await brain.find({ where: { tags: 't0' }, limit: 10 } as any)
|
||||
expect(hits.map((r: any) => r.id)).not.toContain(resolveEntityId('refused'))
|
||||
})
|
||||
|
||||
it('a 384-float embedding parked in the metadata bag is refused, not swallowed', async () => {
|
||||
const err = await brain
|
||||
.add({
|
||||
id: 'bag-vector',
|
||||
data: 'an embedding in the wrong place',
|
||||
type: NounType.Document,
|
||||
metadata: { embedding: Array.from({ length: 384 }, (_, i) => i / 384) },
|
||||
vector: []
|
||||
} as any)
|
||||
.catch((e: any) => e)
|
||||
|
||||
expect(err).toBeInstanceOf(MetadataArrayTooLargeError)
|
||||
expect(err.field).toBe('embedding')
|
||||
expect(err.length).toBe(384)
|
||||
})
|
||||
|
||||
it('update() refuses it too', async () => {
|
||||
await brain.add({
|
||||
id: 'grow',
|
||||
data: 'starts small',
|
||||
type: NounType.Document,
|
||||
metadata: { tags: tags(3) },
|
||||
vector: []
|
||||
})
|
||||
await expect(
|
||||
brain.update({ id: 'grow', metadata: { tags: tags(OVER) } } as any)
|
||||
).rejects.toBeInstanceOf(MetadataArrayTooLargeError)
|
||||
|
||||
// And the row keeps the values it had.
|
||||
const hits = await brain.find({ where: { tags: 't1' }, limit: 10 } as any)
|
||||
expect(hits.map((r: any) => r.id)).toContain(resolveEntityId('grow'))
|
||||
})
|
||||
|
||||
it('relate() refuses it on a verb\'s metadata', async () => {
|
||||
await brain.add({ id: 'a', data: 'a', type: NounType.Thing, vector: [] })
|
||||
await brain.add({ id: 'b', data: 'b', type: NounType.Thing, vector: [] })
|
||||
await expect(
|
||||
brain.relate({
|
||||
from: 'a',
|
||||
to: 'b',
|
||||
type: VerbType.RelatedTo,
|
||||
metadata: { tags: tags(OVER) }
|
||||
} as any)
|
||||
).rejects.toBeInstanceOf(MetadataArrayTooLargeError)
|
||||
})
|
||||
|
||||
it('a nested oversize array is refused under its dotted address', async () => {
|
||||
const err = await brain
|
||||
.add({
|
||||
id: 'nested-over',
|
||||
data: 'nested and too long',
|
||||
type: NounType.Document,
|
||||
metadata: { facets: { labels: tags(OVER, 'l') } },
|
||||
vector: []
|
||||
} as any)
|
||||
.catch((e: any) => e)
|
||||
expect(err).toBeInstanceOf(MetadataArrayTooLargeError)
|
||||
expect(err.field).toBe('facets.labels')
|
||||
})
|
||||
})
|
||||
|
||||
describe('a row already on disk is admitted, and the skip is NARRATED', () => {
|
||||
afterEach(() => {
|
||||
vi.restoreAllMocks()
|
||||
})
|
||||
|
||||
it('extraction over an old oversize row warns by field, length and bound', async () => {
|
||||
const warn = vi.spyOn(prodLog, 'warn').mockImplementation(() => {})
|
||||
const index = (brain as any).metadataIndex
|
||||
|
||||
// The shape an older engine persisted: the write door never saw it, so
|
||||
// this reaches extraction directly — exactly as a rebuild or a remove
|
||||
// reading the row back would.
|
||||
const fields = index.extractIndexableFields({
|
||||
metadata: { tags: tags(MAX_INDEXED_ARRAY_LENGTH + 5), keep: 'me' }
|
||||
})
|
||||
|
||||
// The oversize field contributes nothing...
|
||||
expect(fields.filter((f: any) => f.field === 'tags')).toHaveLength(0)
|
||||
// ...the rest of the row indexes normally — the row is not rejected...
|
||||
expect(fields.some((f: any) => f.field === 'keep' && f.value === 'me')).toBe(true)
|
||||
// ...and the skip is said out loud, with everything needed to act on it.
|
||||
expect(warn).toHaveBeenCalled()
|
||||
const said = warn.mock.calls.map((c: any[]) => String(c[0])).join('\n')
|
||||
expect(said).toContain('tags')
|
||||
expect(said).toContain(String(MAX_INDEXED_ARRAY_LENGTH + 5))
|
||||
expect(said).toContain(String(MAX_INDEXED_ARRAY_LENGTH))
|
||||
expect(said).toContain('NOT indexed')
|
||||
})
|
||||
|
||||
it('an at-bound row on disk is indexed in full and says nothing', async () => {
|
||||
const warn = vi.spyOn(prodLog, 'warn').mockImplementation(() => {})
|
||||
const index = (brain as any).metadataIndex
|
||||
|
||||
const fields = index.extractIndexableFields({
|
||||
metadata: { tags: tags(MAX_INDEXED_ARRAY_LENGTH) }
|
||||
})
|
||||
expect(fields.filter((f: any) => f.field === 'tags')).toHaveLength(MAX_INDEXED_ARRAY_LENGTH)
|
||||
|
||||
const said = warn.mock.calls.map((c: any[]) => String(c[0])).join('\n')
|
||||
expect(said).not.toContain('indexing bound')
|
||||
})
|
||||
})
|
||||
})
|
||||
258
tests/unit/utils/metadataIndex-sparse-range-collation.test.ts
Normal file
258
tests/unit/utils/metadataIndex-sparse-range-collation.test.ts
Normal file
|
|
@ -0,0 +1,258 @@
|
|||
/**
|
||||
* @module tests/unit/utils/metadataIndex-sparse-range-collation
|
||||
* @description RANGE QUERIES ON THE LEGACY SPARSE INDEX — order, or a refusal.
|
||||
* Never a confidently ordered wrong answer.
|
||||
*
|
||||
* THE TWO RANGE PATHS. `getIdsForRange` routes a `gte` / `lt` / `between` two
|
||||
* ways. The column store compares RAW values and is correct. The legacy sparse
|
||||
* chunk index — the pre-7.20.0 fallback, still read for workspaces that have
|
||||
* not been rebuilt — compared `normalizeValue()` output, and `normalizeValue`
|
||||
* carries an escape hatch that destroys order on purpose: a string over 100
|
||||
* characters is replaced by a short hash so it can serve as a filesystem-safe
|
||||
* key. Ordering hashes ranks rows by digest.
|
||||
*
|
||||
* THE DEFECT, IN TWO SHAPES.
|
||||
*
|
||||
* (a) A LONG BOUND against ordinary values. `where: { title: { gte: <a
|
||||
* 120-character string> } }` collapsed the BOUND to `__HASH_…`, whose
|
||||
* leading underscores sort below every letter — so a bound that should
|
||||
* have excluded everything matched the entire field instead. This is the
|
||||
* shape that reaches a caller who never stored a long value at all.
|
||||
*
|
||||
* (b) LONG VALUES in the index. A field whose values ran long was persisted
|
||||
* hashed, so its order is not recoverable from this index at all. The old
|
||||
* code compared the digests anyway and returned a subset chosen by hash.
|
||||
*
|
||||
* THE LAW. Bounds are normalized WITHOUT the hash escape hatch, so a long
|
||||
* bound stays comparable — (a) is simply fixed. Where the persisted KEY is a
|
||||
* hash, the order does not exist to be computed, and the query throws a typed
|
||||
* `BrainyError('INVALID_QUERY')` naming the field and the cure — (b) is
|
||||
* refused by name. Loud beats wrong.
|
||||
*
|
||||
* THE FIXTURE is a genuine legacy index: it is written through the same
|
||||
* `ChunkManager` / `SparseIndex` doors a pre-7.20.0 engine wrote through, with
|
||||
* keys normalized exactly as that engine normalized them, into a field the
|
||||
* column store does not serve. The chunk WRITE path was removed in 11be039, so
|
||||
* this is the only way the shape the read path exists for can be built.
|
||||
*
|
||||
* NOT CLAIMED HERE. The persisted keys are also lower-cased and trimmed by
|
||||
* `normalizeValue`, so this path's string ranges are case-INSENSITIVE where
|
||||
* the column store's are not. The raw values are not in the index to compare —
|
||||
* that divergence is a property of the bytes on disk and it ends when the
|
||||
* column store adopts the field. It is named in `getIdsFromChunksForRange`'s
|
||||
* doc comment rather than papered over.
|
||||
*/
|
||||
import { describe, it, expect, beforeEach } from 'vitest'
|
||||
import { Brainy } from '../../../src/brainy'
|
||||
import { NounType } from '../../../src/types/graphTypes'
|
||||
import { SparseIndex, ChunkManager } from '../../../src/utils/metadataIndexChunking'
|
||||
import { BrainyError } from '../../../src/errors/brainyError'
|
||||
|
||||
/** The field the legacy index covers — deliberately never given to a row, so
|
||||
* the column store never learns it and the sparse fallback is the only path. */
|
||||
const FIELD = 'legacyTitle'
|
||||
|
||||
/**
|
||||
* Write a legacy sparse index for `field` exactly as a pre-7.20.0 engine did:
|
||||
* one chunk, keys normalized through the index's own `normalizeValue`, ids as
|
||||
* roaring bitmaps, a zone map and a bloom filter over the chunk.
|
||||
*
|
||||
* @param brain - The live brain whose metadata index gains the legacy field.
|
||||
* @param field - Field name to index.
|
||||
* @param valueToIds - Raw value → the entity ids that carried it.
|
||||
*/
|
||||
async function writeLegacySparseIndex(
|
||||
brain: any,
|
||||
field: string,
|
||||
valueToIds: Array<[string, string[]]>
|
||||
): Promise<void> {
|
||||
const index = brain.metadataIndex
|
||||
const chunkManager: ChunkManager = index.chunkManager
|
||||
const sparseIndex = new SparseIndex(field)
|
||||
|
||||
// The keys a pre-7.20.0 writer persisted: normalizeValue output, hash escape
|
||||
// hatch and all. This is what makes the fixture the real shape.
|
||||
const chunk = await chunkManager.createChunk(field)
|
||||
for (const [value, ids] of valueToIds) {
|
||||
const key = index.normalizeValue(value, field)
|
||||
for (const id of ids) await chunkManager.addToChunk(chunk, key, id)
|
||||
}
|
||||
await chunkManager.saveChunk(chunk)
|
||||
|
||||
sparseIndex.registerChunk(
|
||||
{
|
||||
chunkId: chunk.chunkId,
|
||||
field,
|
||||
valueCount: chunk.entries.size,
|
||||
idCount: Array.from(chunk.entries.values()).reduce((s: number, b: any) => s + b.size, 0),
|
||||
zoneMap: (chunkManager as any).calculateZoneMap(chunk),
|
||||
lastUpdated: Date.now(),
|
||||
splitThreshold: 80,
|
||||
mergeThreshold: 20
|
||||
},
|
||||
chunkManager.createBloomFilter(chunk)
|
||||
)
|
||||
|
||||
await index.saveSparseIndex(field, sparseIndex)
|
||||
}
|
||||
|
||||
/** A deterministic string of `n` characters starting with `lead`. */
|
||||
function longString(lead: string, n: number): string {
|
||||
return lead + 'x'.repeat(n - lead.length)
|
||||
}
|
||||
|
||||
describe('legacy sparse index: range queries order values, or refuse', () => {
|
||||
let brain: Brainy<any>
|
||||
let index: any
|
||||
let ids: string[]
|
||||
|
||||
beforeEach(async () => {
|
||||
brain = new Brainy({ requireSubtype: false, storage: { type: 'memory' } })
|
||||
await brain.init()
|
||||
index = (brain as any).metadataIndex
|
||||
|
||||
// Rows exist (so the id mapper can resolve them) but carry NO `legacyTitle`
|
||||
// — the column store must not serve the field the pins query.
|
||||
ids = []
|
||||
for (let i = 0; i < 3; i++) {
|
||||
const id = `row-${i}`
|
||||
await brain.add({ id, data: `row ${i}`, type: NounType.Thing, metadata: { lane: 'a' }, vector: [] })
|
||||
ids.push(id)
|
||||
}
|
||||
expect(index.columnStore.hasField(FIELD)).toBe(false)
|
||||
})
|
||||
|
||||
describe('(a) a long BOUND against ordinary short values', () => {
|
||||
// 'apple' < 'mango' < 'zebra', and every bound below is compared against
|
||||
// these three raw keys.
|
||||
beforeEach(async () => {
|
||||
await writeLegacySparseIndex(brain, FIELD, [
|
||||
['apple', [ids[0]]],
|
||||
['mango', [ids[1]]],
|
||||
['zebra', [ids[2]]]
|
||||
])
|
||||
})
|
||||
|
||||
it('the fixture: the values are stored raw, the long bound is what hashes', () => {
|
||||
expect(index.normalizeValue('apple', FIELD)).toBe('apple')
|
||||
// The bound is what the old code collapsed — and a digest sorts below
|
||||
// every letter, which is exactly why `gte` matched everything.
|
||||
const bound = longString('zzz', 120)
|
||||
expect(index.normalizeValue(bound, FIELD)).toMatch(/^__HASH_/)
|
||||
expect(index.normalizeValue(bound, FIELD) < 'apple').toBe(true)
|
||||
})
|
||||
|
||||
it('gte a bound above every value matches NOTHING (it used to match all)', async () => {
|
||||
const bound = longString('zzz', 120)
|
||||
const matched = await index.getIdsForRange(FIELD, bound, undefined, true, true)
|
||||
expect(matched).toEqual([])
|
||||
})
|
||||
|
||||
it('lte a bound above every value matches EVERY value', async () => {
|
||||
const bound = longString('zzz', 120)
|
||||
const matched = await index.getIdsForRange(FIELD, undefined, bound, true, true)
|
||||
expect(matched).toHaveLength(3)
|
||||
})
|
||||
|
||||
it('gte a long bound below every value matches every value', async () => {
|
||||
const bound = longString('aaa', 120)
|
||||
const matched = await index.getIdsForRange(FIELD, bound, undefined, true, true)
|
||||
expect(matched).toHaveLength(3)
|
||||
})
|
||||
|
||||
it('a long bound orders BETWEEN the values, not below all of them', async () => {
|
||||
// 'mmm…' sits between 'mango' and 'zebra'.
|
||||
const bound = longString('mmm', 120)
|
||||
const matched = await index.getIdsForRange(FIELD, bound, undefined, true, true)
|
||||
expect(matched).toHaveLength(1)
|
||||
})
|
||||
|
||||
it('short bounds are unchanged — the ordinary case still orders correctly', async () => {
|
||||
expect(await index.getIdsForRange(FIELD, 'b', undefined, true, true)).toHaveLength(2)
|
||||
expect(await index.getIdsForRange(FIELD, undefined, 'n', true, true)).toHaveLength(2)
|
||||
expect(await index.getIdsForRange(FIELD, 'b', 'n', true, true)).toHaveLength(1)
|
||||
// Strict bounds stay strict.
|
||||
expect(await index.getIdsForRange(FIELD, 'mango', undefined, false, true)).toHaveLength(1)
|
||||
expect(await index.getIdsForRange(FIELD, 'mango', undefined, true, true)).toHaveLength(2)
|
||||
})
|
||||
})
|
||||
|
||||
describe('(b) long VALUES — the index holds hashes, so the range is refused', () => {
|
||||
beforeEach(async () => {
|
||||
await writeLegacySparseIndex(brain, FIELD, [
|
||||
[longString('alpha', 140), [ids[0]]],
|
||||
[longString('mike', 140), [ids[1]]],
|
||||
[longString('zulu', 140), [ids[2]]]
|
||||
])
|
||||
})
|
||||
|
||||
it('the fixture: the persisted keys really are hashes', async () => {
|
||||
const chunk = await index.chunkManager.loadChunk(FIELD, 0)
|
||||
const keys = Array.from(chunk.entries.keys()) as string[]
|
||||
expect(keys).toHaveLength(3)
|
||||
for (const k of keys) expect(k).toMatch(/^__HASH_/)
|
||||
// And their digest order is NOT their value order — the wrong answer the
|
||||
// old code returned was wrong, not merely arbitrary.
|
||||
const digestOrder = [...keys].sort()
|
||||
const valueOrder = [
|
||||
index.normalizeValue(longString('alpha', 140), FIELD),
|
||||
index.normalizeValue(longString('mike', 140), FIELD),
|
||||
index.normalizeValue(longString('zulu', 140), FIELD)
|
||||
]
|
||||
expect(digestOrder).not.toEqual(valueOrder)
|
||||
})
|
||||
|
||||
it('a range over the hashed field throws a typed refusal naming the field', async () => {
|
||||
await expect(
|
||||
index.getIdsForRange(FIELD, longString('mike', 140), undefined, true, true)
|
||||
).rejects.toThrow(BrainyError)
|
||||
|
||||
const err = await index
|
||||
.getIdsForRange(FIELD, longString('mike', 140), undefined, true, true)
|
||||
.catch((e: any) => e)
|
||||
expect(err).toBeInstanceOf(BrainyError)
|
||||
expect(err.type).toBe('INVALID_QUERY')
|
||||
expect(err.message).toContain(FIELD)
|
||||
expect(err.message).toContain('hash')
|
||||
// The cure is named, not left to the caller to guess.
|
||||
expect(err.message).toContain('repairIndex')
|
||||
})
|
||||
|
||||
it('every range shape refuses — gte, lte and between alike', async () => {
|
||||
const lo = longString('alpha', 140)
|
||||
const hi = longString('zulu', 140)
|
||||
for (const [min, max] of [
|
||||
[lo, undefined],
|
||||
[undefined, hi],
|
||||
[lo, hi]
|
||||
] as Array<[any, any]>) {
|
||||
const err = await index.getIdsForRange(FIELD, min, max, true, true).catch((e: any) => e)
|
||||
expect(err).toBeInstanceOf(BrainyError)
|
||||
expect(err.type).toBe('INVALID_QUERY')
|
||||
}
|
||||
})
|
||||
|
||||
it('EQUALITY still works on the hashed field — only ordering is refused', async () => {
|
||||
const matched = await index.getIds(FIELD, longString('mike', 140))
|
||||
expect(matched).toHaveLength(1)
|
||||
})
|
||||
})
|
||||
|
||||
describe('numeric ranges on the legacy path are untouched', () => {
|
||||
beforeEach(async () => {
|
||||
await writeLegacySparseIndex(brain, FIELD, [
|
||||
['5', [ids[0]]],
|
||||
['50', [ids[1]]],
|
||||
['500', [ids[2]]]
|
||||
])
|
||||
})
|
||||
|
||||
it('numbers still compare numerically, not lexicographically', async () => {
|
||||
// The whole point of compareNormalizedValues: "50" < "500" numerically
|
||||
// even though "500" < "50" would hold as strings by prefix.
|
||||
expect(await index.getIdsForRange(FIELD, 10, undefined, true, true)).toHaveLength(2)
|
||||
expect(await index.getIdsForRange(FIELD, undefined, 100, true, true)).toHaveLength(2)
|
||||
expect(await index.getIdsForRange(FIELD, 10, 100, true, true)).toHaveLength(1)
|
||||
})
|
||||
})
|
||||
})
|
||||
Reference in a new issue