diff --git a/docs/FIND_SYSTEM.md b/docs/FIND_SYSTEM.md index 6aa33515..77fbbd79 100644 --- a/docs/FIND_SYSTEM.md +++ b/docs/FIND_SYSTEM.md @@ -369,71 +369,6 @@ return results.slice(offset, offset + limit) // → Auto-correction: Use most likely alternative based on affinity data ``` -## Field Projection (`fields`) - -`find()` and `get()` accept a `fields` list. Without it they return the whole -record; with it they return only the fields you name — and, where the index can -supply them, without opening the canonical record at all. - -```ts -// A list page: two user fields and one engine scalar. No document bodies. -await brain.find({ - where: { kind: 'post' }, - fields: ['title', 'slug', 'system.createdAt'], - limit: 50 -}) - -await brain.get(id, { fields: ['title'] }) -``` - -### Why it exists - -A list view that renders a title and a date does not need the body, but without -a projection every row hydrates its full record and throws almost all of it -away. On a posts list that is the dominant cost of the query. - -### The rules - -| | | -|---|---| -| **`fields` absent** | The full record, byte-identical to before. Nothing changes. | -| **Field names** | The one addressing law: a bare name is user metadata (`'title'`), `system.*` is an engine scalar (`'system.createdAt'`). | -| **A field the row lacks** | Simply **absent** from the result. Never an error. | -| **Identity** | Every row keeps its `id` (and `score` on `find`) regardless — a row you cannot identify is not a row. | -| **Where values come from** | The **column store**, which holds raw values. Never the sparse index, which buckets timestamps for range queries. | -| **A field the column cannot serve** | The canonical record is read for that field only. Correct, just not free. | - -### Missing fields are absent, not errors - -This is deliberate and differs from `orderBy`, which throws -`UnresolvableFieldError` for an unknown field. A typo in `orderBy` silently -changes the ordering, so it must be loud. A projection asks "give me these if -you have them", and an optional field must not turn a list into a failure — so -`fields` uses the permissive path. - -### Cost - -When every named field is column-served, a projected page performs **zero** -canonical reads. When one is not, only that read happens and the rest still come -from the index. Both are pinned by counting reads rather than timing them, in -`tests/integration/find-fields-projection.test.ts`. - -### `related()` takes no `fields` - -A `Relation` carries `from` and `to` as **ids** and hydrates no entity record, -so there is nothing for a projection to trim. Projecting the endpoints would be -a new capability rather than a projection of an existing one. - -### For engine implementers - -Projection is served through an optional provider door, -`getScalarsForIds(ids, fields)` on `MetadataIndexProvider`. The contract is in -`src/plugin.ts`; the short version is **return only what you can serve exactly, -and say what you served**. The caller diffs the answer against the request and -reads records for the remainder, so omission costs a read while a wrong value is -a wrong answer nobody can see. An engine without the door still works — every -field falls back to the record. - ## Performance Characteristics ### Query Performance by Type diff --git a/docs/api-contract.json b/docs/api-contract.json index c4f4e056..12cb37c8 100644 --- a/docs/api-contract.json +++ b/docs/api-contract.json @@ -1507,7 +1507,6 @@ "BrainyError", "DerivedArtifactMissingError", "GraphIndexNotReadyError", - "MetadataArrayTooLargeError", "MetadataIndexNotReadyError", "MigrationInProgressError", "ProtectedArtifactError", diff --git a/src/brainy.ts b/src/brainy.ts index b8eb7f56..3fe57053 100644 --- a/src/brainy.ts +++ b/src/brainy.ts @@ -4193,16 +4193,6 @@ export class Brainy implements BrainyInterface { } // Route to metadata-only or full entity based on options - // A PROJECTED get goes through the same seam every list page uses, so a - // detail read of two scalars costs an index read rather than a record read. - // It is checked before `includeVectors` because the two are incompatible by - // construction: a projection returns the named fields, and a vector is not - // one of them unless it was named. - if (options?.fields !== undefined && options.fields.length > 0) { - const page = await this.#hydratePage([id], options.fields) - return page.get(id) ?? null - } - const includeVectors = options?.includeVectors ?? false // Default: metadata-only (fast) if (includeVectors) { @@ -4249,170 +4239,6 @@ export class Brainy implements BrainyInterface { * const children = childIds.map(id => childrenMap.get(id)).filter(Boolean) * ``` */ - /** - * **The projection seam** — hydrate a page of ids under an optional `fields` - * projection, opening the canonical record only when the index cannot serve - * what was asked for. - * - * Without a projection this is exactly `batchGet`, byte for byte: the whole - * point is that `fields` absent changes nothing. - * - * With one, the order is: ask the index for the named scalars in a single - * batched door; see which requested fields it actually served; and read - * records ONLY if something is still missing — and only to fill those fields. - * A page whose every requested field is index-served performs zero canonical - * reads, which is the whole reason the door exists. - * - * `guardFields` are fetched ALONGSIDE the projection and trimmed off before - * the caller sees them. find()'s index-integrity guard re-validates every row - * against its own predicate, and it reads the entity to do so — so a row - * projected down to `title` would fail a `where: { kind }` it genuinely - * matches, and the whole page would vanish. The fields a filter names are - * fields the index can serve by definition, so carrying them costs nothing - * and keeps the guard honest. - * - * A field nothing can supply is simply absent from the row. That is the - * permissive law: a projection asks "these, if you have them", and an - * optional field must not turn a list into an exception. It deliberately does - * NOT route through the strict address resolver, which throws - * `UnresolvableFieldError` for an unknown key — that strictness is right for - * `orderBy`, where a typo silently changes the order, and wrong here, where - * the honest answer is "this row does not have that". - * - * @param ids - Canonical ids for the page. - * @param fields - The projection, or undefined for the full record. - * @returns `id → entity`, projected when `fields` was given. - */ - /** - * The index keys find()'s integrity guard reads when it re-validates a row. - * - * The guard calls `entityMatchesFind(entity, params)`, so a projected entity - * must still carry whatever the params constrain — otherwise a row that - * genuinely matches is dropped for lacking the evidence. These are fetched - * with the projection and trimmed off before the caller sees them. - * - * @param params - The find params. - * @returns Index keys to carry through hydration. - */ - #guardFieldsFor(params: FindParams): string[] { - const keys: string[] = [] - if (params.where && typeof params.where === 'object') { - // Top-level where keys only: nested `anyOf`/`allOf` branches are carried - // by their own keys when the guard walks them, and a filter whose - // evidence is missing keeps the row (the guard's own catch) rather than - // dropping it. - for (const key of Object.keys(params.where as Record)) { - if (key === 'anyOf' || key === 'allOf' || key === 'not') continue - keys.push(key) - } - } - if (params.type !== undefined) keys.push('system.type') - if (params.subtype !== undefined) keys.push('system.subtype') - if (params.service !== undefined) keys.push('system.service') - if (params.excludeVFS === true) keys.push('vfsType', 'isVFSEntity') - return keys - } - - async #hydratePage( - ids: string[], - fields?: readonly string[], - guardFields: readonly string[] = [] - ): Promise>> { - if (fields === undefined || fields.length === 0) return this.batchGet(ids) - - const wanted = [...new Set([...fields, ...guardFields])] - const provider = this.metadataIndex as unknown as MetadataIndexProvider - let served = new Map>() - if (typeof provider.getScalarsForIds === 'function') { - served = await provider.getScalarsForIds(ids, wanted) - } - - // Which ids still owe a field? Only those cost a record read, and a page - // that owes nothing costs none at all. - const owing: string[] = [] - for (const id of ids) { - const row = served.get(id) - if (row === undefined || wanted.some((f) => !(f in row))) owing.push(id) - } - - // The records are read for the OWED fields only; everything the index - // already served is used as-is, so a body field pulls its own record and - // no more than that. - const records = owing.length > 0 ? await this.batchGet(owing) : new Map>() - - const out = new Map>() - for (const id of ids) { - const fromIndex = served.get(id) - const record = records.get(id) - // An id neither the index nor storage knows is not a row. - if (fromIndex === undefined && record === undefined) continue - out.set(id, this.#projectEntity(id, wanted, fromIndex, record)) - } - return out - } - - /** - * Build one projected entity: `id`, plus exactly the requested fields that - * something could supply. - * - * Values come from the index first and the record second, and they must agree - * — the index only reports what it can serve exactly, so a field it served is - * the record's value. A field neither has is omitted rather than set to - * `undefined`: absent and present-and-undefined are different answers, and a - * caller checking `'slug' in row.metadata` deserves the true one. - * - * @param id - The entity id, always present on the result. - * @param fields - The requested index keys. - * @param fromIndex - What the index served for this id, if anything. - * @param record - The canonical entity, if one had to be read. - * @returns The projected entity. - */ - #projectEntity( - id: string, - fields: readonly string[], - fromIndex: Record | undefined, - record: Entity | undefined - ): Entity { - const projected: Record = { id } - const metadata: Record = {} - let sawMetadata = false - - for (const field of fields) { - let value: unknown - let found = false - if (fromIndex !== undefined && field in fromIndex) { - value = fromIndex[field] - found = true - } else if (record !== undefined) { - if (field.startsWith('system.')) { - const inner = field.slice('system.'.length) - const bag = record as unknown as Record - if (inner in bag && bag[inner] !== undefined) { - value = bag[inner] - found = true - } - } else { - const bag = (record.metadata ?? {}) as Record - if (field in bag) { - value = bag[field] - found = true - } - } - } - if (!found) continue - - if (field.startsWith('system.')) { - projected[field.slice('system.'.length)] = value - } else { - metadata[field] = value - sawMetadata = true - } - } - - if (sawMetadata) projected.metadata = metadata - return projected as unknown as Entity - } - async batchGet(ids: string[], options?: GetOptions): Promise>> { // Canonical read (see get): resolves by id from storage, no derived index. await this.ensureInitialized({ needs: [] }) @@ -8211,7 +8037,7 @@ export class Brainy implements BrainyInterface { // Batch-load entities for 10x faster cloud storage performance // GCS: 10 entities = 1×50ms vs 10×50ms = 500ms (10x faster) - const entitiesMap = await this.#hydratePage(pageIds, params.fields, this.#guardFieldsFor(params)) + const entitiesMap = await this.batchGet(pageIds) for (const id of pageIds) { const entity = entitiesMap.get(id) if (entity) { @@ -8248,7 +8074,7 @@ export class Brainy implements BrainyInterface { if (hiddenIds.size > 0) allUuids = allUuids.filter((id) => !hiddenIds.has(id)) const pageIds = allUuids.slice(offset, offset + limit) - const entitiesMap = await this.#hydratePage(pageIds, params.fields, this.#guardFieldsFor(params)) + const entitiesMap = await this.batchGet(pageIds) for (const id of pageIds) { const entity = entitiesMap.get(id) if (entity) { @@ -8276,7 +8102,7 @@ export class Brainy implements BrainyInterface { const pageIds = filteredIds.slice(offset, offset + limit) // Batch-load entities for 10x faster cloud storage performance - const entitiesMap = await this.#hydratePage(pageIds, params.fields, this.#guardFieldsFor(params)) + const entitiesMap = await this.batchGet(pageIds) for (const id of pageIds) { const entity = entitiesMap.get(id) if (entity) { @@ -8486,23 +8312,7 @@ export class Brainy implements BrainyInterface { // Rank by score (top offset+limit), then drop the offset — identical ordering // to a full `sort((a, b) => b.score - a.score)` + slice, but the native // `sort:topK` provider can compute only the page instead of the full sort. - // - // ONLY when score IS the requested order. An explicit `orderBy` names a - // different ordering key, and this block cannot serve it: it ranks by - // score and CUTS the page, so the tail's `orderBy` sort below either - // never runs at all (the early return, when there is no `connected` / - // `fusion` work left) or runs over a page that score already chose — - // ordering eight rows relevance picked instead of the eight the field - // ordering asks for. Both readings were silent: `find({ query, where, - // orderBy })` answered in score order while `find({ where, orderBy })` - // answered in field order, and nothing said the request had been dropped. - // - // With `orderBy` present the candidate set falls through UNCUT to the - // tail, which orders it in full and pages that ordering — "page last", - // the graph-first law applied to ordering rather than to filtering. The - // set is bounded by the legs (the text matches inside the universe plus - // the beam walk's `limit * 2`), not by the store. - if (!params.orderBy && results.length >= offset + limit) { + if (results.length >= offset + limit) { const k = offset + limit const order = rankIndicesByScore(results.map(r => r.score), k, true) results = reorderByIndices(results, order).slice(offset, k) @@ -8527,7 +8337,7 @@ export class Brainy implements BrainyInterface { // Batch-load entities for current page - O(page_size) instead of O(total_results) // GCS: 10 entities = 1×50ms vs 10×50ms = 500ms (10x faster) - const entitiesMap = await this.#hydratePage(pageIds, params.fields, this.#guardFieldsFor(params)) + const entitiesMap = await this.batchGet(pageIds) for (const id of pageIds) { const entity = entitiesMap.get(id) if (entity) { @@ -8555,7 +8365,7 @@ export class Brainy implements BrainyInterface { // Batch-load entities for paginated results (10x faster on GCS) const sortedResults: Result[] = [] - const entitiesMap = await this.#hydratePage(pageIds, params.fields, this.#guardFieldsFor(params)) + const entitiesMap = await this.batchGet(pageIds) for (const id of pageIds) { const entity = entitiesMap.get(id) if (entity) { @@ -8660,28 +8470,6 @@ export class Brainy implements BrainyInterface { }) } - // PROJECTION TRIM — applied once, here, AFTER the integrity guard, so every - // find() path is trimmed uniformly and the guard still saw the evidence it - // needs. Hydration carried the guard's fields alongside the projection; - // this removes them, leaving exactly what the caller named. - // - // Rows that reached here from a path the seam does not hydrate (a vector or - // text leg builds its own entities) are trimmed from what they already - // hold, so the ANSWER is the same everywhere — only the cost differs, and - // only on the paths that still read a record. - if (params.fields !== undefined && params.fields.length > 0 && result.length > 0) { - const named = [...new Set(params.fields)] - result = result.map((r) => { - const projected = this.#projectEntity( - r.id, - named, - undefined, - r.entity as unknown as Entity - ) - return { ...r, entity: projected } as typeof r - }) - } - // includeVectors — opt-in vector hydration. Default (false) keeps the perf // contract: every result path above builds entities via the metadata-only // fast path, so `entity.vector` is the empty stub. When requested, fetch the @@ -13342,7 +13130,7 @@ export class Brainy implements BrainyInterface { } return this._flushQueued } - return this.#startFlushLeader() + return this.startFlushLeader() } /** @@ -13351,23 +13139,16 @@ export class Brainy implements BrainyInterface { * the ONE queued waiter (if any) is promoted. The `finally` callback returns * nothing on purpose: a callback that returned the promoted run's promise * would make the leader await its own follower. - * - * ECMAScript-private (`#`), not TypeScript `private`: `private` is erased at - * compile time, so the method would still land on the prototype — and the - * contract manifest reads the surface the BUILD exposes, so it would emit - * this as a contract door. A door is a promise every engine implementing the - * contract must keep; this is the flush gate's own bookkeeping. `#` keeps it - * off the prototype, where the emitter cannot see it. * @returns The leader's own promise, settling on its own body alone. */ - #startFlushLeader(): Promise { + private startFlushLeader(): Promise { const run = this._runFlush() // `finally` and not `then`: a failed flush must still open the gate, or // one rejection would wedge every later flush behind a promise nobody // will ever settle. const gated: Promise = run.finally(() => { if (this._flushInFlight === gated) this._flushInFlight = null - this.#promoteQueuedFlush() + this.promoteQueuedFlush() }) this._flushInFlight = gated return gated @@ -13378,12 +13159,9 @@ export class Brainy implements BrainyInterface { * leader and settle its deferred from that run. Never throws into the * leader's `finally`: a synchronous failure starting the promoted run is * reported to the waiter, which must be settled on every path. - * - * ECMAScript-private for the same reason as the leader starter above: - * internals are not doors. * @returns Nothing. */ - #promoteQueuedFlush(): void { + private promoteQueuedFlush(): void { const settle = this._flushQueuedSettle if (!settle) return // Clear BEFORE starting, so the promoted run's own joiners queue afresh @@ -13391,7 +13169,7 @@ export class Brainy implements BrainyInterface { this._flushQueued = null this._flushQueuedSettle = null try { - this.#startFlushLeader().then(settle.resolve, settle.reject) + this.startFlushLeader().then(settle.resolve, settle.reject) } catch (error) { settle.reject(error) } @@ -17064,7 +16842,7 @@ export class Brainy implements BrainyInterface { ordered = valued.map((v) => v.id) } const pageIds = ordered.slice(offset, offset + limit) - const entitiesMap = await this.#hydratePage(pageIds, params.fields, this.#guardFieldsFor(params)) + const entitiesMap = await this.batchGet(pageIds) const results: Result[] = [] for (const id of pageIds) { const entity = entitiesMap.get(id) @@ -20844,45 +20622,34 @@ export class Brainy implements BrainyInterface { // Phase 1: Flush ALL components in parallel to persist buffered data // This is critical when cor native providers buffer data in Rust memory - // - // READ-ONLY GUARD, applied to EVERY flush here. A flush is a write by - // definition, and a reader has nothing of its own to persist — but these - // calls were not conditional, so a read-only open → read → close REWROTE - // four files under `_system/`: the metadata field registry (whose flush() - // saves it unconditionally, "even with no dirty fields"), and the three - // type/subtype statistics files the storage adapter's count flush stamps. - // Every one of them was re-stamped on a session that committed nothing. - // A reader must leave `_system/` exactly as it found it — the same law the - // clean-shutdown marker already lives under (see the generation-store - // guard below and `Brainy.openReadOnly`). await Promise.all([ // Flush HNSW dirty nodes (deferred persistence mode) (async () => { - if (this.index && !this.isReadOnly && typeof this.index.flush === 'function') { + if (this.index && typeof this.index.flush === 'function') { await this.index.flush() } })(), // Flush metadata index (field indexes + EntityIdMapper) (async () => { - if (this.metadataIndex && !this.isReadOnly && typeof this.metadataIndex.flush === 'function') { + if (this.metadataIndex && typeof this.metadataIndex.flush === 'function') { await this.metadataIndex.flush() } })(), // Flush graph adjacency index (LSM trees) (async () => { - if (this.graphIndex && !this.isReadOnly && typeof this.graphIndex.flush === 'function') { + if (this.graphIndex && typeof this.graphIndex.flush === 'function') { await this.graphIndex.flush() } })(), // Flush storage adapter counts (async () => { - if (this.storage && !this.isReadOnly && typeof this.storage.flushCounts === 'function') { + if (this.storage && typeof this.storage.flushCounts === 'function') { await this.storage.flushCounts() } })(), // Flush aggregation index state (async () => { - if (this._aggregationIndex && !this.isReadOnly) { + if (this._aggregationIndex) { await this._aggregationIndex.flush() } })(), @@ -20931,37 +20698,21 @@ export class Brainy implements BrainyInterface { // Phase 2: Close components to release resources (timers, file handles) // Data is already safe on disk from Phase 1 - // - // READ-ONLY GUARD, same law as Phase 1. Each of these closes is a WRITER: - // the graph index drains both LSM MemTables to SSTables and stamps its - // watermark, and the vector/metadata `close` hooks — optional doors the - // reference engine leaves unimplemented, but which a native provider fills - // in — persist their buffered state. None of that is a reader's to write. - // - // A reader still has to RELEASE what it holds, which is why this is a - // branch rather than a skip: `stopBackgroundFlush()` is the non-writing - // half of the graph index's close, clearing the auto-flush interval that - // would otherwise outlive the session. The optional hooks have no - // non-writing counterpart to call, and a provider that buffers nothing on - // a read-only open has nothing to release. await Promise.all([ (async () => { - if (!this.graphIndex) return - if (this.isReadOnly) { - this.graphIndex.stopBackgroundFlush() - } else if (typeof this.graphIndex.close === 'function') { + if (this.graphIndex && typeof this.graphIndex.close === 'function') { await this.graphIndex.close() } })(), (async () => { const index = this.index as JsHnswVectorIndex & VectorIndexOptionalHooks - if (index && !this.isReadOnly && typeof index.close === 'function') { + if (index && typeof index.close === 'function') { await index.close() } })(), (async () => { const metadataIndex = this.metadataIndex as MetadataIndexManager & MetadataIndexOptionalHooks - if (metadataIndex && !this.isReadOnly && typeof metadataIndex.close === 'function') { + if (metadataIndex && typeof metadataIndex.close === 'function') { await metadataIndex.close() } })(), diff --git a/src/errors/brainyError.ts b/src/errors/brainyError.ts index 4301d3f7..a58236e3 100644 --- a/src/errors/brainyError.ts +++ b/src/errors/brainyError.ts @@ -405,68 +405,3 @@ export class MigrationInProgressError extends BrainyError { } } } - -/** - * THE INDEXABLE-ARRAY BOUND. An array-valued metadata field indexes one posting - * per element, so an unbounded array is an unbounded write — a 384-float - * embedding parked in the metadata bag would mint 384 postings for one row. - * The bound exists to keep that out of the index. - * - * 64 is hardcoded on purpose (the zero-config law: no knob). It sits far above - * every legitimate multi-value field the engine has seen — tags, authors, - * categories, labels, participant lists — and far below any real embedding - * width, so the two populations do not overlap and no caller has to tune it. - * - * It replaces a limit of 10 that was applied SILENTLY: a row whose `tags` array - * held eleven entries had that field skipped entirely and dropped out of every - * filtered search on it, with no error, no warning and no way to tell the - * difference from "no row matches". A rule this consequential is a law with a - * name and a refusal, not a `continue`. - */ -export const MAX_INDEXED_ARRAY_LENGTH = 64 - -/** - * A metadata field carries an array longer than {@link MAX_INDEXED_ARRAY_LENGTH}. - * - * Thrown at the WRITE door (`add` / `update` / `relate` / `updateRelation`), so - * the caller learns at the moment of writing that the field will not be - * searchable — rather than discovering it later as rows that quietly fail to - * match. Carries the field, its length and the bound so a handler can report - * or repair without parsing the message. - * - * The cure is one of: store the long array outside the indexed bag (`data` - * carries arbitrary content and is not indexed element-wise); pass an embedding - * as the first-class `vector` parameter, which is where a vector belongs; or - * shorten the field to the values that are actually queried. - */ -export class MetadataArrayTooLargeError extends BrainyError { - /** The metadata field whose array is too long (its full dotted address). */ - public readonly field: string - /** How many elements that array holds. */ - public readonly length: number - /** The bound it exceeded — {@link MAX_INDEXED_ARRAY_LENGTH}. */ - public readonly limit: number - - constructor(site: string, field: string, length: number, limit: number) { - super( - `${site}: metadata field '${field}' holds ${length} array elements, ` + - `over the ${limit}-element indexing bound. An array field indexes one ` + - `posting per element, so an unbounded array is an unbounded write. ` + - `This write is refused rather than indexed partially or skipped silently ` + - `— a skipped field drops the row out of every filtered search on '${field}' ` + - `with no way to tell that from "nothing matched". ` + - `Cures: put the long array in 'data' (stored, not indexed element-wise); ` + - `pass an embedding as the first-class 'vector' parameter; or keep only ` + - `the values you actually query in '${field}'.`, - 'VALIDATION', - false - ) - this.name = 'MetadataArrayTooLargeError' - this.field = field - this.length = length - this.limit = limit - if (Error.captureStackTrace) { - Error.captureStackTrace(this, MetadataArrayTooLargeError) - } - } -} diff --git a/src/graph/graphAdjacencyIndex.ts b/src/graph/graphAdjacencyIndex.ts index 2c131a30..ebd3b90c 100644 --- a/src/graph/graphAdjacencyIndex.ts +++ b/src/graph/graphAdjacencyIndex.ts @@ -1105,31 +1105,13 @@ export class GraphAdjacencyIndex implements GraphIndexProvider { } /** - * Stop the auto-flush interval WITHOUT writing anything. - * - * The non-writing half of {@link close}, for a shutdown that must leave the - * store byte-identical — a read-only brain's close. `close()` itself is a - * writer: it drains both LSM MemTables to SSTables and stamps the watermark, - * which is exactly right for a writer and forbidden for a reader. A reader - * still has to release this interval, though: it is the one piece of this - * index that outlives the close and could fire against a store the session no - * longer owns. - * - * @returns Nothing. + * Clean shutdown */ - stopBackgroundFlush(): void { + async close(): Promise { if (this.flushTimer) { clearInterval(this.flushTimer) this.flushTimer = undefined } - } - - /** - * Clean shutdown — drains both trees and stamps the watermark. THIS WRITES; - * a read-only brain must call {@link stopBackgroundFlush} instead. - */ - async close(): Promise { - this.stopBackgroundFlush() // Close both LSM-trees (will flush MemTables to SSTables) if (this.initialized) { diff --git a/src/index.ts b/src/index.ts index 673e1e6f..e946f15c 100644 --- a/src/index.ts +++ b/src/index.ts @@ -203,7 +203,7 @@ export { EntityNotFoundError, RelationNotFoundError } from './errors/notFound.js // Base error + typed migration-lock error — thrown by any data-plane call while a // brain runs its one-time 7.x→8.0 upgrade; catch to answer HTTP 503 + Retry-After. -export { BrainyError, MigrationInProgressError, GraphIndexNotReadyError, MetadataIndexNotReadyError, VectorIndexNotReadyError, ProtectedArtifactError, DerivedArtifactMissingError, MetadataArrayTooLargeError, MAX_INDEXED_ARRAY_LENGTH } from './errors/brainyError.js' +export { BrainyError, MigrationInProgressError, GraphIndexNotReadyError, MetadataIndexNotReadyError, VectorIndexNotReadyError, ProtectedArtifactError, DerivedArtifactMissingError } from './errors/brainyError.js' export type { BrainyErrorType } from './errors/brainyError.js' // ============= 8.0 Db API — generational MVCC ============= diff --git a/src/indexes/columnStore/ColumnStore.ts b/src/indexes/columnStore/ColumnStore.ts index 48f4a963..4fe45bff 100644 --- a/src/indexes/columnStore/ColumnStore.ts +++ b/src/indexes/columnStore/ColumnStore.ts @@ -292,58 +292,6 @@ export class ColumnStore implements ColumnStoreProvider { return result } - /** - * Read this column's value for each of `entityIntIds` — the per-id read - * behind `find({ fields })`. - * - * Every other read door here answers "which entities have this value". A - * projection asks the opposite — "what value does this entity have" — and - * without it a projection has to go to the canonical record for a field the - * column is already holding. - * - * The column is walked ONCE and the wanted ids are picked out as they pass, - * so the cost is O(column) per field rather than O(ids x column). Later - * sources win: the tail buffer holds writes newer than any segment, and - * within the segments a later one supersedes an earlier, exactly as `filter` - * treats them. - * - * Values are EXACT — this store keeps raw values, not the bucketed form the - * sparse index uses for range queries — which is what makes it safe to - * project from. Deleted entities are skipped; an id with no value in this - * column is simply absent from the result. - * - * @param field - Field name to read. - * @param entityIntIds - Entity integer ids to read values for. - * @returns `entityIntId -> value` for the ids this column holds. - */ - async valuesForIds( - field: string, - entityIntIds: Iterable - ): Promise> { - const wanted = new Set(entityIntIds) - const out = new Map() - if (wanted.size === 0 || !this.hasField(field)) return out - - const deleted = this.deletedEntities.get(field) - const take = (entry: { value: number | string; entityIntId: number }): void => { - if (!wanted.has(entry.entityIntId)) return - if (deleted && deleted.has(entry.entityIntId)) return - out.set(entry.entityIntId, entry.value) - } - - // Segments oldest -> newest, then the tail: a later write overwrites an - // earlier one for the same id. - const cursors = await this.getSegmentCursors(field) - for (const cursor of cursors) { - for (const entry of cursor.iterateForward()) take(entry) - } - const tailCursor = this.getTailBufferCursor(field) - if (tailCursor) { - for (const entry of tailCursor.iterateForward()) take(entry) - } - return out - } - /** * Range filter: find entities where field is within the bounds. * diff --git a/src/neural/embeddedPatterns.ts b/src/neural/embeddedPatterns.ts index 92e3057a..4f4339f4 100644 --- a/src/neural/embeddedPatterns.ts +++ b/src/neural/embeddedPatterns.ts @@ -2,7 +2,7 @@ * 🧠 BRAINY EMBEDDED PATTERNS * * AUTO-GENERATED - DO NOT EDIT - * Generated: 2026-08-27T09:18:45-07:00 + * Generated: 2025-09-29T10:10:00-07:00 * Patterns: 220 * Coverage: 94-98% of all queries * diff --git a/src/plugin.ts b/src/plugin.ts index 23a8c883..64abfe26 100644 --- a/src/plugin.ts +++ b/src/plugin.ts @@ -495,45 +495,6 @@ export interface MetadataIndexProvider { query: string, ids: readonly string[] ): Promise> - /** - * @description OPTIONAL: read named SCALAR fields for many ids at once, from - * the index's own value storage, WITHOUT touching the canonical record. - * - * This is the door behind `find/get/related({ fields })`. A list view that - * needs a title and a slug currently hydrates the whole record for every row - * — document bodies included — and then discards almost all of it. Serving - * the named scalars from the index turns that into an index read. - * - * ## The contract, and the one rule that makes it safe - * - * **Return only what you can serve EXACTLY, and say what you served.** The - * answer is a per-id map of the fields this index actually resolved; the - * caller diffs it against what was requested and reads the canonical record - * for the remainder. An implementation must therefore OMIT a field rather - * than approximate it — and omission costs only a record read, while a wrong - * value is a wrong answer nobody can see. - * - * That rule is not hypothetical. This engine's own index buckets - * `system.createdAt` and `system.updatedAt` to the minute for range queries, - * so it cannot serve them exactly and omits them. An engine whose column - * store holds raw values can serve the same fields — so the two answer - * differently in COST and identically in CONTENT, which is the only - * difference a projection door is allowed to have. - * - * A field absent from an entity is simply absent from that entity's map. It - * is never an error, and never a `null` standing in for one: absent and - * present-and-null are different answers. - * - * @param ids - Canonical entity ids to read. - * @param fields - Index KEYS (bare = user metadata, `system.*` = engine - * scalar), already address-resolved by the caller. - * @returns `id → { field: value }` for the fields this index served exactly. - * Ids with nothing to serve may be omitted entirely. - */ - getScalarsForIds?( - ids: readonly string[], - fields: readonly string[] - ): Promise>> getSortedIdsForFilter(filter: any, orderBy: string, order?: 'asc' | 'desc', topK?: number): Promise getFilterValues(field: string): Promise getFilterFields(): Promise diff --git a/src/types/brainy.types.ts b/src/types/brainy.types.ts index b99f0261..a0d55c1e 100644 --- a/src/types/brainy.types.ts +++ b/src/types/brainy.types.ts @@ -561,33 +561,6 @@ export interface UpdateRelationParams { * refusal with the fix in hand beats a silent behavior flip. */ export interface FindParams { - /** - * **Field projection** — return only these fields on each row, instead of the - * whole record. - * - * A list view that shows a title and a slug does not need the document body, - * yet without a projection every row hydrates its full record and throws - * almost all of it away. Naming the fields lets them be served from the index - * itself: a scalar the index holds exactly is read from the index, and the - * canonical record is opened ONLY when a requested field cannot be. - * - * Field names follow the one addressing law: a bare name is the user's - * metadata (`'title'`), and `system.*` is an engine scalar - * (`'system.createdAt'`). - * - * - **Absent** ⇒ the full record, exactly as before. - * - A requested field the entity does not carry is simply **absent** from the - * row. It is never an error — a projection asks "give me these if you have - * them", so an optional field must not turn a list into a failure. - * - Every returned row carries `id` (and, on `find`, `score`) regardless: a - * row you cannot identify is not a row. - * - * @example - * // A list page: two user fields and one engine scalar, no document bodies. - * await brain.find({ where: { kind: 'post' }, fields: ['title', 'slug', 'system.createdAt'], limit: 50 }) - */ - fields?: readonly string[] - // Vector Intelligence /** Natural language or semantic search query (embedded and matched via HNSW + text index) */ query?: string @@ -816,12 +789,6 @@ export interface SimilarParams { * Added string ID shorthand syntax */ export interface RelatedParams { - // NOTE: `fields` is deliberately NOT offered here. A Relation carries `from` - // and `to` as IDS and hydrates no entity record, so there is nothing for a - // projection to trim — the param would be decorative. Projecting the - // ENDPOINTS would be a new capability (related() hydrating entities), not a - // projection of an existing one, and it belongs in its own decision. - /** * Filter by source entity ID * @@ -1447,33 +1414,6 @@ export interface ImportResult { * */ export interface GetOptions { - /** - * **Field projection** — return only these fields on each row, instead of the - * whole record. - * - * A list view that shows a title and a slug does not need the document body, - * yet without a projection every row hydrates its full record and throws - * almost all of it away. Naming the fields lets them be served from the index - * itself: a scalar the index holds exactly is read from the index, and the - * canonical record is opened ONLY when a requested field cannot be. - * - * Field names follow the one addressing law: a bare name is the user's - * metadata (`'title'`), and `system.*` is an engine scalar - * (`'system.createdAt'`). - * - * - **Absent** ⇒ the full record, exactly as before. - * - A requested field the entity does not carry is simply **absent** from the - * row. It is never an error — a projection asks "give me these if you have - * them", so an optional field must not turn a list into a failure. - * - Every returned row carries `id` (and, on `find`, `score`) regardless: a - * row you cannot identify is not a row. - * - * @example - * // A list page: two user fields and one engine scalar, no document bodies. - * await brain.find({ where: { kind: 'post' }, fields: ['title', 'slug', 'system.createdAt'], limit: 50 }) - */ - fields?: readonly string[] - /** * Include 384-dimensional vector embeddings in the response * diff --git a/src/utils/metadataIndex.ts b/src/utils/metadataIndex.ts index 1a882945..a3aa7679 100644 --- a/src/utils/metadataIndex.ts +++ b/src/utils/metadataIndex.ts @@ -40,7 +40,7 @@ import { import { EntityIdMapper } from './entityIdMapper.js' import { RoaringBitmap32, roaringLibraryInitialize } from './roaring/index.js' import { FieldTypeInference, FieldType } from './fieldTypeInference.js' -import { BrainyError, MAX_INDEXED_ARRAY_LENGTH } from '../errors/brainyError.js' +import { BrainyError } from '../errors/brainyError.js' /** * Fields whose values are stored in the sparse index as BUCKETED values @@ -289,10 +289,8 @@ export class MetadataIndexManager implements MetadataIndexProvider { // No name-based exclude/allow lists — the field-addressing law: every // user field indexes, whatever its name ('content', 'data', 'id', // 'vector', … included). Bulk payloads are kept out by uniform value- - // SHAPE rules in extractIndexableFields (arrays longer than - // MAX_INDEXED_ARRAY_LENGTH never become posting scalars, and the write - // door refuses them by name; >100-char values index hashed), never by - // field name. + // SHAPE rules in extractIndexableFields (arrays >10 never become + // posting scalars; >100-char values index hashed), never by name. } // Initialize metadata cache with similar config to search cache @@ -963,41 +961,9 @@ export class MetadataIndexManager implements MetadataIndexProvider { } /** - * Get IDs for a range using the legacy chunked sparse index (zone maps + - * roaring bitmaps). Lazy-loaded via UnifiedCache. - * - * ORDER IS NOT A KEY. This path compares NORMALIZED values, and - * {@link normalizeValue} carries an escape hatch that is order-destroying by - * design: a string over 100 characters is replaced by {@link hashValue}'s - * digest so it can be used as a filesystem-safe key. Feeding that digest to - * an ORDERING comparison — which is what a `gte` / `lt` / `between` does — - * ranks rows by hash. The result is not empty and not an error: it is a - * confidently ordered wrong answer, and it disagrees with the column-store - * path (`getIdsForRange` above), which compares raw values and is correct. - * - * Two changes hold the line here: - * - * 1. THE BOUNDS ARE NEVER HASHED. They are normalized with `allowHash = - * false`, so a long bound stays comparable instead of collapsing to a - * digest. This alone fixes the common shape — a long bound queried - * against ordinary short values, where the digest sorts below every - * letter and `gte` therefore matched the entire store. - * - * 2. A HASHED KEY IS REFUSED, NEVER GUESSED. The persisted keys are whatever - * the pre-7.20.0 writer normalized them to, so a field whose values ran - * long is stored hashed and its order is simply not recoverable from this - * index. Rather than compare digests, the query throws a typed - * `BrainyError('INVALID_QUERY')` naming the field, the bound and the cure. - * Loud beats wrong. - * - * KNOWN, NAMED DIVERGENCE. The persisted keys are also lower-cased and - * trimmed by `normalizeValue`, so this path's string ranges are - * CASE-INSENSITIVE where the column store's are not. That is a property of - * the bytes a pre-7.20.0 engine wrote, not of the comparison: the raw values - * are not in the index to compare. The bounds are normalized into the same - * case-folded space so the comparison is at least self-consistent, and the - * divergence disappears with the field itself once the column store adopts - * it. See the module note on `getIdsFromChunks` for the path's lifetime. + * Get IDs for a range using chunked sparse index with zone maps and roaring bitmaps + * Now fully lazy-loaded via UnifiedCache (no local sparseIndices Map) + * Normalize min/max for timestamp bucketing before comparison */ private async getIdsFromChunksForRange( field: string, @@ -1013,27 +979,9 @@ export class MetadataIndexManager implements MetadataIndexProvider { } // Normalize min/max for consistent comparison with indexed values - // (indexed values are bucketed for timestamps, so we must bucket the query - // bounds too) — but NEVER through the hash escape hatch, which would make - // the bound incomparable. See the doc comment above. - const normalizedMin = min !== undefined ? this.normalizeValue(min, field, false) : undefined - const normalizedMax = max !== undefined ? this.normalizeValue(max, field, false) : undefined - - // REFUSE BEFORE SELECTING. Chunk selection itself orders values: it tests - // the bounds against each chunk's zone-map min/max. If those are hashes the - // selection is already meaningless — and its failure mode is an EMPTY - // answer (no chunk appears to overlap), which is the quietest wrong answer - // of all. So the key space is checked here, before a single chunk is - // chosen, and again per key below for a chunk whose zone map happens to - // read clean. - for (const chunkId of sparseIndex.getAllChunkIds()) { - const zoneMap = sparseIndex.getChunk(chunkId)?.zoneMap - for (const bound of [zoneMap?.min, zoneMap?.max]) { - if (typeof bound === 'string' && MetadataIndexManager.isHashedValue(bound)) { - throw MetadataIndexManager.rangeOverHashedIndex(field) - } - } - } + // (indexed values are bucketed for timestamps, so we must bucket the query bounds too) + const normalizedMin = min !== undefined ? this.normalizeValue(min, field) : undefined + const normalizedMax = max !== undefined ? this.normalizeValue(max, field) : undefined // Find candidate chunks using zone maps const candidateChunkIds = sparseIndex.findChunksForRange(normalizedMin, normalizedMax) @@ -1048,13 +996,6 @@ export class MetadataIndexManager implements MetadataIndexProvider { const chunk = await this.chunkManager.loadChunk(field, chunkId) if (chunk) { for (const [value, bitmap] of chunk.entries) { - // A hashed key carries no order. Refuse the range rather than rank by - // digest — the whole answer is unsound, so failing on the first one - // is the honest outcome. - if (MetadataIndexManager.isHashedValue(value)) { - throw MetadataIndexManager.rangeOverHashedIndex(field) - } - // Check if value is in range using numeric-aware comparison // (normalizeValue converts numbers to strings, so we must compare numerically) let inRange = true @@ -1083,25 +1024,6 @@ export class MetadataIndexManager implements MetadataIndexProvider { return this.idMapper.intsIterableToUuids(allIntIds) } - /** - * The refusal a range query gets when the legacy sparse index holds hashed - * keys for the field. Names the field and the cure; never a wrong answer. - */ - private static rangeOverHashedIndex(field: string): BrainyError { - return new BrainyError( - `Range query on field "${field}" cannot be served by the legacy sparse index: ` + - `its values were persisted as hashes (values over 100 characters are stored ` + - `hashed to stay within filesystem name limits), and a hash carries no order — ` + - `comparing them would return a confidently ordered wrong answer. ` + - `Equality (\`where: { ${field}: value }\`) still works on this index. ` + - `To range over this field, let the column store adopt it: run ` + - `brain.repairIndex({ rebuild: ['metadata'] }), which rebuilds the field into ` + - `the column store, where ranges compare raw values.`, - 'INVALID_QUERY', - false - ) - } - /** * Get roaring bitmap for a field-value pair without converting to UUIDs * This is used for fast multi-field intersection queries using hardware-accelerated bitmap AND @@ -1269,17 +1191,8 @@ export class MetadataIndexManager implements MetadataIndexProvider { * value-based detection (DuckDB-inspired). Analyzes actual data values, not names. * * NO FALLBACKS - Pure value-based detection only. - * - * @param value - The value to normalize. - * @param field - Optional field name (drives the per-field statistics strategy). - * @param allowHash - Whether the >100-character escape hatch may fire. TRUE - * everywhere a normalized value is used as a KEY (equality postings, chunk - * entries, filenames) — that is what the hash exists for. FALSE on the - * ORDER-comparing path: a hash is deliberately order-destroying, so a - * bound that hashes can only be compared as nonsense. See - * {@link isHashedValue} and `getIdsFromChunksForRange`. */ - private normalizeValue(value: any, field?: string, allowHash: boolean = true): string { + private normalizeValue(value: any, field?: string): string { if (value === null || value === undefined) return '__NULL__' if (typeof value === 'boolean') return value ? '__TRUE__' : '__FALSE__' @@ -1337,34 +1250,21 @@ export class MetadataIndexManager implements MetadataIndexProvider { // Default normalization if (typeof value === 'number') return value.toString() if (Array.isArray(value)) { - const joined = value.map(v => this.normalizeValue(v, field, allowHash)).join(',') + const joined = value.map(v => this.normalizeValue(v, field)).join(',') // Hash very long array values to avoid filesystem limits - if (allowHash && joined.length > 100) { + if (joined.length > 100) { return this.hashValue(joined) } return joined } const stringValue = String(value).toLowerCase().trim() // Hash very long string values to avoid filesystem limits - if (allowHash && stringValue.length > 100) { + if (stringValue.length > 100) { return this.hashValue(stringValue) } return stringValue } - /** - * Is this normalized value a HASH rather than the value itself? - * - * {@link hashValue} is an escape hatch for filesystem name limits, and it is - * deliberately order-destroying: two values whose hashes compare one way - * routinely compare the other way themselves. Anything that ORDERS normalized - * values has to know when it is holding one, because comparing hashes yields - * a confident, wrong answer rather than an error. - */ - private static isHashedValue(normalized: string): boolean { - return normalized.startsWith('__HASH_') - } - /** * Create a short hash for long values to avoid filesystem filename limits */ @@ -1389,10 +1289,9 @@ export class MetadataIndexManager implements MetadataIndexProvider { * 'content', 'vector' in a bag are ordinary user fields) * - Record-frame plumbing (vector, connections, level, data, _rev, id) * never indexes — that is namespace routing, not a name carve-out - * - Value-SHAPE rules apply uniformly to all names: arrays longer than - * MAX_INDEXED_ARRAY_LENGTH never become posting scalars (and say so — - * the write door refuses them outright); purely numeric key names - * (array indices) skip; >100-char values index hashed (normalizeValue) + * - Value-SHAPE rules apply uniformly to all names: arrays >10 never + * become posting scalars; purely numeric key names (array indices) + * skip; >100-char values index hashed (normalizeValue) */ private extractIndexableFields(data: any): Array<{ field: string, value: any }> { const fields: Array<{ field: string, value: any }> = [] @@ -1454,37 +1353,13 @@ export class MetadataIndexManager implements MetadataIndexProvider { // This catches vectors stored as objects: {0: 0.1, 1: 0.2, ...} if (/^\d+$/.test(key)) continue - // THE INDEXABLE-ARRAY BOUND ({@link MAX_INDEXED_ARRAY_LENGTH}). An - // array field mints one posting per element, so the index has always - // carried a ceiling — it was 10, and it was applied by this bare - // `continue`: an eleven-element `tags` array had its whole field - // skipped and the row dropped out of every filtered search on it, with - // no error, no warning, and nothing to distinguish that from "no row - // matches". The ceiling is not the defect; the silence was. - // - // The write door refuses this shape by name now - // (`MetadataArrayTooLargeError`, thrown from paramValidation's - // `rejectOversizeIndexArrays`), so a live add/update never reaches - // here over the bound. Reaching it means the row is ALREADY on disk — - // written by an older engine under the old rule — and this is a - // rebuild, a catch-up fold or a remove reading it back. Refusing there - // would make an existing store un-rebuildable, so the row is admitted - // and the skipped field is NARRATED instead. Never silent, either way. - if (Array.isArray(value) && value.length > MAX_INDEXED_ARRAY_LENGTH) { - prodLog.warn( - `[brainy] metadata field '${fullKey}' holds ${value.length} array elements, ` + - `over the ${MAX_INDEXED_ARRAY_LENGTH}-element indexing bound — the field is ` + - `NOT indexed for this row, so it will not match a where-clause on '${fullKey}'. ` + - `This row predates the bound (the write door refuses this shape now). ` + - `Move the long array into 'data', or pass an embedding as the 'vector' parameter.` - ) - continue - } + // Skip large arrays (> 10 elements) - likely vectors or bulk data + if (Array.isArray(value) && value.length > 10) continue if (value && typeof value === 'object' && !Array.isArray(value)) { // Recurse into nested objects (but not arrays), keeping the frame extract(value, fullKey, frame) - } else if (Array.isArray(value)) { + } else if (Array.isArray(value) && value.length <= 10) { // Small arrays: index as multi-value field (all with same field name) // Example: tags: ["javascript", "node"] → field="tags", value="javascript" + field="tags", value="node" for (const item of value) { @@ -2930,67 +2805,6 @@ export class MetadataIndexManager implements MetadataIndexProvider { return order === 'asc' ? comparison : -comparison } - /** - * Read named scalar fields for many ids from the COLUMN STORE, without - * touching the canonical record — the `find({ fields })` door. - * - * ## Why the column store and not the sparse index - * - * The column store keeps RAW values; the sparse index keeps a normalized, - * bucketed form built for range queries — `system.createdAt` is indexed at - * minute precision there. A projection served from the sparse index would - * hand back a value that differs from the record's, which is a wrong answer - * nobody can see. So this door reads the column store, and a field the - * column store does not hold is OMITTED rather than approximated. - * - * ## Why batched - * - * `getFieldValueForEntity` answers one (id, field) pair by walking the - * field's storage; called per row it re-walks the same column for every id. - * This walks each column ONCE and picks out every requested id as it passes: - * O(fields x column) instead of O(ids x fields x column). - * - * Omission is always safe — it costs the caller a record read. The caller - * diffs what it asked for against what came back and reads records for the - * remainder, so an index that can serve nothing is slow, never wrong. - * - * @param ids - Canonical entity ids. - * @param fields - Index keys (bare = user metadata, `system.*` = engine scalar). - * @returns `id -> { field: value }` for exactly the pairs this index served. - */ - async getScalarsForIds( - ids: readonly string[], - fields: readonly string[] - ): Promise>> { - const out = new Map>() - if (ids.length === 0 || fields.length === 0) return out - - // int -> id, so a column hit resolves back to the caller's id. An id the - // mapper does not know cannot be in any column, so it is simply absent. - const idByInt = new Map() - for (const id of ids) { - const intId = this.idMapper.getInt(id) - if (intId !== undefined) idByInt.set(intId, id) - } - if (idByInt.size === 0) return out - - for (const field of fields) { - if (!this.columnStore.hasField(field)) continue - const values = await this.columnStore.valuesForIds(field, idByInt.keys()) - for (const [intId, value] of values) { - const id = idByInt.get(intId) - if (id === undefined) continue - let row = out.get(id) - if (row === undefined) { - row = {} - out.set(id, row) - } - row[field] = value - } - } - return out - } - async getFieldValueForEntity(entityId: string, field: string): Promise { // `field` arrives as a FROZEN INDEX KEY (bare = user metadata; // 'system.' = engine scalar). Storage fallbacks read the matching diff --git a/src/utils/paramValidation.ts b/src/utils/paramValidation.ts index f1addb5b..00790a4a 100644 --- a/src/utils/paramValidation.ts +++ b/src/utils/paramValidation.ts @@ -18,7 +18,6 @@ import { findCallerLocation } from './callerLocation.js' import * as os from 'node:os' import * as fs from 'node:fs' import { parseFieldAddress, UnsupportedFindOptionError } from '../db/fieldAddressing.js' -import { MAX_INDEXED_ARRAY_LENGTH, MetadataArrayTooLargeError } from '../errors/brainyError.js' const getSystemMemory = (): number => { if (os) { @@ -539,53 +538,8 @@ function rejectForgedSystemKeys(metadata: Record | undefined, s } } -/** - * THE INDEXABLE-ARRAY BOUND, enforced at the write door. - * - * An array-valued metadata field indexes one posting per element, so the index - * has always carried a ceiling. It used to be 10, and it was applied by a bare - * `continue` deep inside field extraction: a row whose `tags` array held eleven - * entries had that field skipped entirely and dropped out of every filtered - * search on it — no error, no warning, and no way for the caller to tell the - * difference from "no row matches". Silence is the defect; the ceiling is not. - * - * The bound is now {@link MAX_INDEXED_ARRAY_LENGTH}, high enough that every - * legitimate multi-value field clears it, and it REFUSES here instead of - * dropping data downstream. Refusing at the write door is what makes it - * actionable: the caller learns at the moment of writing, with the field, the - * length and the bound in hand. - * - * Scope is the caller's own metadata bag — the values that become postings. - * Nested bags are walked, because a nested field indexes under its dotted - * address exactly like a top-level one. Arrays of OBJECTS are not walked: the - * index only ever makes postings from an array's scalar elements. - * - * @param metadata - The caller's metadata bag (undefined is fine). - * @param site - The write door's name, for the message ('add()', 'update()', …). - * @throws {MetadataArrayTooLargeError} Naming the field, its length and the bound. - */ -function rejectOversizeIndexArrays(metadata: Record | undefined, site: string): void { - if (!metadata) return - - const walk = (bag: Record, prefix: string): void => { - for (const [key, value] of Object.entries(bag)) { - const address = prefix ? `${prefix}.${key}` : key - if (Array.isArray(value)) { - if (value.length > MAX_INDEXED_ARRAY_LENGTH) { - throw new MetadataArrayTooLargeError(site, address, value.length, MAX_INDEXED_ARRAY_LENGTH) - } - } else if (value && typeof value === 'object') { - walk(value as Record, address) - } - } - } - - walk(metadata, '') -} - export function validateAddParams(params: AddParams): void { rejectForgedSystemKeys(params.metadata as Record | undefined, 'add()') - rejectOversizeIndexArrays(params.metadata as Record | undefined, 'add()') // 'data' is ABSENT only when null/undefined — an empty string ('') is real // content (a legitimate empty file's first write) and must not be treated // as missing. Falsy-but-present values (0, false, '') all count as present; @@ -654,7 +608,6 @@ export function validateAddParams(params: AddParams): void { */ export function validateUpdateParams(params: UpdateParams): void { rejectForgedSystemKeys(params.metadata as Record | undefined, 'update()') - rejectOversizeIndexArrays(params.metadata as Record | undefined, 'update()') // Same absent-vs-empty distinction as validateAddParams: '' is a real new // value (e.g. truncating a file to empty content via overwrite), only // null/undefined means "no new data was given". @@ -729,7 +682,6 @@ export function validateUpdateParams(params: UpdateParams): void { */ export function validateRelateParams(params: RelateParams): void { rejectForgedSystemKeys(params.metadata as Record | undefined, 'relate()') - rejectOversizeIndexArrays(params.metadata as Record | undefined, 'relate()') // 8.0 verb-id contract (L.7): verb ids are UUIDs, generated by brainy. // RelateParams has no `id` field — an untyped caller passing one would // previously have it silently ignored (a generated UUID was used instead). @@ -779,7 +731,6 @@ export function validateRelateParams(params: RelateParams): void { */ export function validateUpdateRelationParams(params: UpdateRelationParams): void { rejectForgedSystemKeys(params.metadata as Record | undefined, 'updateRelation()') - rejectOversizeIndexArrays(params.metadata as Record | undefined, 'updateRelation()') if (!params.id) { throw new Error('id is required for updateRelation') } diff --git a/tests/integration/find-fields-projection.test.ts b/tests/integration/find-fields-projection.test.ts deleted file mode 100644 index 5d339f08..00000000 --- a/tests/integration/find-fields-projection.test.ts +++ /dev/null @@ -1,261 +0,0 @@ -/** - * @module tests/integration/find-fields-projection - * @description **Field projection** — `find/get({ fields })` returns only the - * named fields, and serves them from the index when it can. - * - * A list view that shows a title and a slug does not need the document body, - * yet without a projection every row hydrates its whole record and discards - * almost all of it. These pins hold the two halves of the fix: - * - * **The answer.** A projected row is a SUBSET of the full row — for every - * requested field, the projected value equals the value the same query returns - * unprojected. Absent `fields` is byte-identical to today. A requested field the - * entity does not carry is simply absent, never an error. `system.*` resolves to - * the engine scalar, a bare name to the user's metadata. - * - * **The cost.** When every requested field is index-served, the canonical - * record is never opened — asserted by counting reads, not by timing them, so - * it cannot flake into a false green. When one requested field is NOT - * index-served (a body field, or a bucketed timestamp), exactly the owing rows - * are read and the rest are still served from the index. - */ -import { describe, it, expect, beforeAll, vi } from 'vitest' -import { Brainy } from '../../src/brainy' -import { NounType } from '../../src/types/graphTypes' -import { generateTestVector } from '../helpers/test-factory' - -/** Rows carrying a title, a slug, and a large body nobody wants in a list. */ -const ROWS = 12 -const BODY = 'x'.repeat(4096) - -describe('find/get({ fields }) — projection', () => { - let brain: Brainy - const ids: string[] = [] - - beforeAll(async () => { - brain = new Brainy({ requireSubtype: false, storage: { type: 'memory' } }) - await brain.init() - for (let i = 0; i < ROWS; i++) { - ids.push( - await brain.add({ - id: `post-${i}`, - data: `post ${i}`, - type: NounType.Thing, - metadata: { - kind: 'post', - title: `Title ${i}`, - slug: `slug-${i}`, - rank: i, - body: BODY, - // Only some rows carry this, so "missing is absent" is exercised - // by real data rather than by a name nothing ever had. - ...(i % 2 === 0 ? { featured: true } : {}) - }, - vector: generateTestVector() - }) - ) - } - // Persist so the column store holds the values a projection reads from. - await brain.flush() - }) - - /** Count canonical record reads for one call. */ - const countingReads = async (body: () => Promise): Promise<{ out: R; reads: number }> => { - const spy = vi.spyOn(brain as any, 'batchGet') - try { - const out = await body() - const reads = spy.mock.calls.reduce( - (n, call) => n + ((call[0] as string[] | undefined)?.length ?? 0), - 0 - ) - return { out, reads } - } finally { - spy.mockRestore() - } - } - - it('absent fields is byte-identical to today', async () => { - const params = { where: { kind: 'post' }, limit: 5 } - const a = await brain.find({ ...params }) - const b = await brain.find({ ...params, fields: undefined }) - expect(JSON.stringify(b)).toBe(JSON.stringify(a)) - }) - - it('a projected row is a SUBSET of the full row, field for field', async () => { - const shapes: Array> = [ - { where: { kind: 'post' }, limit: 6 }, - { where: { kind: 'post' }, limit: 6, offset: 3 }, - { where: { kind: 'post' }, orderBy: 'rank', order: 'asc', limit: 6 }, - { where: { kind: 'post' }, orderBy: 'rank', order: 'desc', limit: 4 } - ] - for (const shape of shapes) { - const full = await brain.find(shape as never) - const projected = await brain.find({ ...shape, fields: ['title', 'slug'] } as never) - expect(projected.map((r) => r.id), JSON.stringify(shape)).toEqual(full.map((r) => r.id)) - for (let i = 0; i < full.length; i++) { - const fullMeta = (full[i].entity.metadata ?? {}) as Record - const projMeta = (projected[i].entity.metadata ?? {}) as Record - expect(projMeta.title, `${JSON.stringify(shape)} row ${i}`).toEqual(fullMeta.title) - expect(projMeta.slug).toEqual(fullMeta.slug) - } - } - }) - - it('returns ONLY the named fields — the body never rides along', async () => { - const rows = await brain.find({ where: { kind: 'post' }, fields: ['title'], limit: 4 }) - expect(rows).toHaveLength(4) - for (const r of rows) { - const meta = (r.entity.metadata ?? {}) as Record - expect(Object.keys(meta)).toEqual(['title']) - expect(meta.body).toBeUndefined() - // Identity always survives a projection: a row you cannot identify is - // not a row. - expect(typeof r.id).toBe('string') - expect(r.entity.id).toBe(r.id) - } - }) - - it('a missing field is simply ABSENT — never an error', async () => { - // `featured` exists on half the rows; `no-such-field` on none. Neither - // throws, and neither appears as an explicit undefined. - const rows = await brain.find({ - where: { kind: 'post' }, - fields: ['title', 'featured', 'no-such-field'], - limit: ROWS - }) - expect(rows.length).toBeGreaterThan(0) - let withFeatured = 0 - for (const r of rows) { - const meta = (r.entity.metadata ?? {}) as Record - expect('no-such-field' in meta).toBe(false) - if ('featured' in meta) withFeatured += 1 - } - // Real data, not a name nothing ever had: some rows carry it, some do not. - expect(withFeatured).toBeGreaterThan(0) - expect(withFeatured).toBeLessThan(rows.length) - }) - - it('a strict address resolver is NOT on this path', async () => { - // orderBy throws UnresolvableFieldError for an unknown user key, because a - // typo there silently changes the order. A projection must not inherit that - // strictness: the honest answer to "give me this if you have it" is silence. - await expect( - brain.find({ where: { kind: 'post' }, fields: ['definitely-not-a-field'], limit: 2 }) - ).resolves.toBeInstanceOf(Array) - }) - - it('system.* resolves to the engine scalar, a bare name to user metadata', async () => { - const full = await brain.find({ where: { kind: 'post' }, limit: 3 }) - const rows = await brain.find({ - where: { kind: 'post' }, - fields: ['system.createdAt', 'title'], - limit: 3 - }) - for (let i = 0; i < rows.length; i++) { - expect((rows[i].entity as any).createdAt).toEqual((full[i].entity as any).createdAt) - const meta = (rows[i].entity.metadata ?? {}) as Record - expect(meta.title).toEqual((full[i].entity.metadata as any).title) - // The engine scalar lands at the top level, not in the metadata bag — - // the two address spaces never shadow each other. - expect('system.createdAt' in meta).toBe(false) - expect('createdAt' in meta).toBe(false) - } - }) - - it('reads NO canonical record when every requested field is index-served', async () => { - // The cost pin, counted rather than timed. `title` and `slug` are ordinary - // indexed user fields, so the index can serve them exactly. - const { out, reads } = await countingReads(() => - brain.find({ where: { kind: 'post' }, fields: ['title', 'slug'], limit: ROWS }) - ) - expect(out.length).toBeGreaterThan(0) - expect(reads).toBe(0) - }) - - it('reads records only for the fields the column cannot serve', async () => { - // `system.data` is NOT a column the store holds (verified against - // getIndexedFields), so the record must be opened for it — while `title`, - // which the column does hold, still comes from the index. - const { out, reads } = await countingReads(() => - brain.find({ where: { kind: 'post' }, fields: ['title', 'system.data'], limit: 4 }) - ) - expect(out).toHaveLength(4) - expect(reads).toBe(4) - for (const r of out) { - const meta = (r.entity.metadata ?? {}) as Record - expect(Object.keys(meta)).toEqual(['title']) - expect(typeof (r.entity as any).data).toBe('string') - } - }) - - it('a large field the column DOES hold costs no record read', async () => { - // Worth pinning because it is the venue case: the body is column-served on - // this engine, so a list that projects around it pays nothing for it, and - // a list that projects it still pays no record read. - const { reads } = await countingReads(() => - brain.find({ where: { kind: 'post' }, fields: ['body'], limit: 4 }) - ) - expect(reads).toBe(0) - }) - - it('projects a vector-leg find too — the ANSWER is uniform, only the cost is not', async () => { - // The seam hydrates the metadata and graph page paths. A vector or text leg - // builds its own entities, so those rows are trimmed after the integrity - // guard instead. That difference is a COST difference, and this pin exists - // so it can never quietly become an ANSWER difference. - const rows = await brain.find({ query: 'post', fields: ['title'], limit: 3 }) - for (const r of rows) { - const meta = (r.entity.metadata ?? {}) as Record - expect(Object.keys(meta)).toEqual(['title']) - expect(meta.body).toBeUndefined() - expect(r.entity.id).toBe(r.id) - } - }) - - it('get({ fields }) projects a single row through the same seam', async () => { - const full = await brain.get(ids[0]) - const projected = await brain.get(ids[0], { fields: ['title', 'slug'] }) - expect(projected).not.toBeNull() - expect(projected!.id).toBe(full!.id) - const fullMeta = (full!.metadata ?? {}) as Record - const projMeta = (projected!.metadata ?? {}) as Record - expect(projMeta.title).toEqual(fullMeta.title) - expect(projMeta.slug).toEqual(fullMeta.slug) - expect(Object.keys(projMeta).sort()).toEqual(['slug', 'title']) - expect((projected as any).body).toBeUndefined() - }) - - it('get({ fields }) reads no record when the index serves the fields', async () => { - const { reads } = await countingReads(() => brain.get(ids[1], { fields: ['title'] })) - expect(reads).toBe(0) - }) - - it('the door serves EXACT values — the column, never the bucketed index', async () => { - // The sparse index buckets `system.createdAt` to the minute for range - // queries; the column store keeps raw ms. Serving a projection from the - // former would hand back a value that differs from the record's, so the - // door reads the column — and this pin is what proves which one it read. - const index = (brain as any).metadataIndex - const sample = ids.slice(0, 3) - const served = await index.getScalarsForIds(sample, ['title', 'system.createdAt']) - expect(served.size).toBe(sample.length) - for (const id of sample) { - const row = served.get(id)! - const record = await brain.get(id) - expect(row.title).toEqual((record!.metadata as any).title) - // Exact to the millisecond — a bucketed value would be rounded down to - // the minute and this would fail. - expect(row['system.createdAt']).toEqual((record as any).createdAt) - } - }) - - it('a field the column store does not hold is OMITTED, not approximated', async () => { - const index = (brain as any).metadataIndex - const served = await index.getScalarsForIds(ids.slice(0, 2), ['title', 'system.data']) - for (const [, row] of served) { - expect('title' in row).toBe(true) - // Omission is what makes the caller read the record for it. - expect('system.data' in row).toBe(false) - } - }) -}) diff --git a/tests/integration/find-orderby-every-path.test.ts b/tests/integration/find-orderby-every-path.test.ts deleted file mode 100644 index 7637a79b..00000000 --- a/tests/integration/find-orderby-every-path.test.ts +++ /dev/null @@ -1,244 +0,0 @@ -/** - * @module tests/integration/find-orderby-every-path - * @description `orderBy` IS THE ORDER — on every find() path, not just the - * metadata-only one. - * - * THE DEFECT. `find({ where, orderBy })` (metadata only) answered in field - * order. `find({ query, where, orderBy })` and `find({ vector, where, orderBy })` - * answered in SCORE order, silently: the vector/filter block ranked the fused - * candidates by score, cut the page, and returned early — the tail's `orderBy` - * sort sat below that early return and never ran. Nothing threw, nothing warned, - * and the two paths disagreed about what "ordered by rank" means. A caller - * paging `orderBy: 'rank', order: 'desc'` over a hybrid find got relevance - * order wearing an ordering request's clothes. - * - * Where `connected` or `fusion` kept the tail alive the defect changed shape - * rather than disappearing: the block had already CUT the page by score, so the - * tail ordered the rows relevance had chosen instead of the rows the ordering - * asks for — a correctly sorted page of the wrong rows. - * - * The early cut fires only once the candidate set reaches `offset + limit` - * rows, which is why small fixtures never saw it: below that threshold the - * block falls through and the tail's sort does apply. That is the whole shape - * of the bug — an ordering that is correct until there is enough data to matter. - * - * THE LAW. An explicit `orderBy` displaces score as the ordering key on every - * path. The candidate set the path produced is ordered IN FULL and the page is - * cut from that ordering — the graph-first law's "page last", applied to - * ordering rather than to filtering. Score-ranked early paging is for the - * default (no `orderBy`) case only, where score IS the requested order. - * - * THE PIN. Differential, against the metadata-only path — the one path that - * always honoured `orderBy`. - * - * WHAT THE DIFFERENTIAL CAN AND CANNOT CLAIM. `orderBy` orders the candidate - * set; it does not enlarge it. The hybrid legs are bounded by construction (the - * text leg and the beam walk each take `limit * 2`), so a differential against - * the metadata-only path — whose universe is every matching row — is only - * meaningful where those bounds provably cover the universe. The fixture is - * sized so they do (12 rows, `limit` 6 → a `limit * 2` = 12-row text leg), and - * the covering is ASSERTED from the leg's own output rather than assumed. This - * pin is about ordering, and it says nothing about recall. - */ -import { describe, it, expect, beforeAll } from 'vitest' -import { Brainy } from '../../src/brainy' -import { NounType, VerbType } from '../../src/types/graphTypes' -import { resolveEntityId } from '../../src/utils/idNormalization' - -/** Embedding width of the default model — the row vectors must match it. */ -const DIM = 384 - -/** A deterministic, per-row-distinct unit vector (no embedder in the fixture). */ -function seededVector(seed: number): number[] { - const v = new Array(DIM) - for (let i = 0; i < DIM; i++) { - v[i] = Math.sin((i + 1) * 0.11 + seed * 0.37) * 0.5 + Math.cos((i + 1) * 0.05 + seed * 0.13) * 0.3 - } - const magnitude = Math.sqrt(v.reduce((sum, x) => sum + x * x, 0)) - return v.map((x) => x / magnitude) -} - -/** - * Ranks, shuffled — so no scoring order can reproduce them by luck, and the - * ordering the pins assert is visibly not the insertion order either. - */ -const RANKS = [7, 3, 11, 1, 9, 5, 12, 2, 10, 4, 8, 6] -const ROWS = RANKS.length -/** The page size every pin uses: `limit * 2` covers the whole universe. */ -const LIMIT = 6 -/** The neighbour subset — the graph-first universe — and its own page size. */ -const NEIGHBOURS = 8 -const GRAPH_LIMIT = 4 - -describe('find(): orderBy is the order on every path', () => { - let brain: Brainy - const QUERY = 'orbital telemetry' - const anchor = 'ordering-anchor' - const neighbourIds: string[] = [] - - beforeAll(async () => { - brain = new Brainy({ requireSubtype: false, storage: { type: 'memory' } }) - await brain.init() - - let seed = 1 - await brain.add({ - id: anchor, - data: 'ground station anchor record', - type: NounType.Thing, - metadata: { lane: 'anchor', rank: 0 }, - vector: seededVector(seed++) - }) - - for (let i = 0; i < ROWS; i++) { - const id = `row-${i}` - await brain.add({ - id, - // EVERY row carries both query words, so the text leg reaches all of - // them and the hybrid candidate set covers the whole universe. - data: `orbital telemetry packet ${i} recorded downlink`, - type: NounType.Document, - metadata: { lane: 'alpha', rank: RANKS[i] }, - vector: seededVector(seed++) - }) - if (i < NEIGHBOURS) { - await brain.relate({ from: anchor, to: id, type: VerbType.RelatedTo }) - neighbourIds.push(resolveEntityId(id)) - } - } - }) - - it('the fixture: the hybrid candidate set covers the whole filter universe', async () => { - const universe: string[] = await (brain as any).filterIdsBelted({ lane: 'alpha' }) - expect(universe).toHaveLength(ROWS) - - // The text leg is bounded at `limit * 2`; the fixture is sized so that - // bound reaches every row in the universe. This is the precondition the - // differential below rests on — asserted from the leg itself. - const textScored = await (brain as any).executeTextSearchScored(QUERY, LIMIT * 2, universe) - expect(textScored).toHaveLength(ROWS) - - // And the candidate set is large enough to trigger the score-ranked early - // cut this pin exists to keep out of an ordered query's way. - expect(ROWS).toBeGreaterThanOrEqual(LIMIT) - }) - - it('metadata-only + orderBy: the reference ordering', async () => { - const rows = await brain.find({ - where: { lane: 'alpha' }, - orderBy: 'rank', - order: 'desc', - limit: LIMIT - } as any) - expect(rows.map((r: any) => r.metadata.rank)).toEqual([12, 11, 10, 9, 8, 7]) - }) - - it('hybrid (query + where) + orderBy: the same page as the metadata-only path', async () => { - const params = { where: { lane: 'alpha' }, orderBy: 'rank', order: 'desc' as const, limit: LIMIT } - const expected = await brain.find(params as any) - const actual = await brain.find({ ...params, query: QUERY } as any) - - expect(actual).toHaveLength(expected.length) - expect(actual.map((r: any) => r.id)).toEqual(expected.map((r: any) => r.id)) - expect(actual.map((r: any) => r.metadata.rank)).toEqual([12, 11, 10, 9, 8, 7]) - }) - - it('hybrid + orderBy asc: the ordering key is honoured in both directions', async () => { - const params = { where: { lane: 'alpha' }, orderBy: 'rank', order: 'asc' as const, limit: LIMIT } - const expected = await brain.find(params as any) - const actual = await brain.find({ ...params, query: QUERY } as any) - - expect(actual.map((r: any) => r.id)).toEqual(expected.map((r: any) => r.id)) - expect(actual.map((r: any) => r.metadata.rank)).toEqual([1, 2, 3, 4, 5, 6]) - }) - - it('hybrid + orderBy + offset: page two is page two of the ORDERING', async () => { - const params = { - where: { lane: 'alpha' }, - orderBy: 'rank', - order: 'desc' as const, - limit: LIMIT, - offset: LIMIT - } - const expected = await brain.find(params as any) - const actual = await brain.find({ ...params, query: QUERY } as any) - - expect(actual).toHaveLength(LIMIT) - expect(actual.map((r: any) => r.id)).toEqual(expected.map((r: any) => r.id)) - expect(actual.map((r: any) => r.metadata.rank)).toEqual([6, 5, 4, 3, 2, 1]) - }) - - it('hybrid + orderBy: paging walks the ordering monotonically, no row twice', async () => { - const seen: number[] = [] - for (let offset = 0; offset < ROWS; offset += LIMIT) { - const page = await brain.find({ - query: QUERY, - where: { lane: 'alpha' }, - orderBy: 'rank', - order: 'desc', - limit: LIMIT, - offset - } as any) - seen.push(...page.map((r: any) => r.metadata.rank)) - } - expect(seen).toHaveLength(ROWS) - expect(new Set(seen).size).toBe(ROWS) - // Strictly descending across every page boundary. - for (let i = 1; i < seen.length; i++) expect(seen[i]).toBeLessThan(seen[i - 1]) - }) - - it('vector + where + orderBy: field order, not distance order', async () => { - // The beam walk takes `limit * 2` = the whole universe here, so the page is - // the true top of the ordering — which distance order cannot produce. - const rows = await brain.find({ - vector: seededVector(1000), - where: { lane: 'alpha' }, - orderBy: 'rank', - order: 'desc', - limit: LIMIT - } as any) - expect(rows.map((r: any) => r.metadata.rank)).toEqual([12, 11, 10, 9, 8, 7]) - }) - - it('graph-first (query + connected + where) + orderBy: the neighbour set, ordered', async () => { - const actual = await brain.find({ - query: QUERY, - connected: { from: anchor, direction: 'out' as const }, - where: { lane: 'alpha' }, - orderBy: 'rank', - order: 'desc', - limit: GRAPH_LIMIT - } as any) - - expect(actual).toHaveLength(GRAPH_LIMIT) - const neighbours = new Set(neighbourIds) - for (const r of actual) expect(neighbours.has(r.id)).toBe(true) - - // The ordering covers the whole neighbour set, so the page holds the - // highest ranks AMONG THE NEIGHBOURS — not the ones the score ranking - // happened to surface first and the tail then sorted among themselves. - const expectedRanks = RANKS.slice(0, NEIGHBOURS) - .sort((a, b) => b - a) - .slice(0, GRAPH_LIMIT) - expect(expectedRanks).toEqual([12, 11, 9, 7]) - expect(actual.map((r: any) => r.metadata.rank)).toEqual(expectedRanks) - }) - - it('fusion + orderBy: the ordering survives the fusion rescore', async () => { - const actual = await brain.find({ - query: QUERY, - where: { lane: 'alpha' }, - fusion: 'weighted', - orderBy: 'rank', - order: 'desc', - limit: LIMIT - } as any) - expect(actual.map((r: any) => r.metadata.rank)).toEqual([12, 11, 10, 9, 8, 7]) - }) - - it('no orderBy: score order still stands (the default is untouched)', async () => { - const rows = await brain.find({ query: QUERY, where: { lane: 'alpha' }, limit: LIMIT } as any) - expect(rows).toHaveLength(LIMIT) - const scores = rows.map((r: any) => r.score) - for (let i = 1; i < scores.length; i++) expect(scores[i]).toBeLessThanOrEqual(scores[i - 1]) - }) -}) diff --git a/tests/integration/graphIndex-pagination.test.ts b/tests/integration/graphIndex-pagination.test.ts index 8ad4d6d8..32a7673c 100644 --- a/tests/integration/graphIndex-pagination.test.ts +++ b/tests/integration/graphIndex-pagination.test.ts @@ -9,34 +9,9 @@ * 8.0 BigInt boundary: entity ints in (resolved via the metadata index's * idMapper), entity/verb ints out (`bigint[]`). Entity ints map back to UUIDs * via `idMapper.getUuid(Number(int))`; verb ints via `verbIntsToIds()`. - * - * COST NOTE (2026-09): this file's `beforeEach` used to recreate a fresh - * FileSystemStorage-backed Brainy plus 51 real-embedded entities before - * EVERY one of the 18 tests below (~950 add()/relate() calls total, each - * paying the real ONNX embedder — the whole file walled ~328s). Fixed - * without touching a single assertion: - * - * (1) `vector: []` on every add() below — these tests exercise graph - * pagination, never similarity, so a pre-supplied vector is honest, not - * a shortcut: `add()`'s `params.vector || (await this.embed(...))` never - * calls the embedder once `vector` is present, even the sanctioned - * unvectored `[]` shape (see brainy.ts's add(), the zero-norm-law - * comment) — and the `vector.length > 0` gate on dimension-pinning means - * `[]` never poisons `this.dimensions` for later real embeds. - * (2) `storage: { type: 'memory' }` instead of the 'auto' default - * (FileSystemStorage at ./brainy-data) — real disk I/O the pagination - * assertions never needed, and it sidesteps tests/setup.ts's global - * per-test `rm -rf brainy-data`, which would otherwise corrupt a brain - * shared across a describe's beforeAll out from under it. - * (3) the base fixture (one central hub + 50 outgoing-edge neighbors) now - * builds ONCE per describe (`beforeAll`) instead of once per test — safe - * because no test in a given describe block mutates the shared fixture - * in a way an earlier sibling test's assertion depends on (the one - * mutating case, the incoming-direction test, is the LAST test in its - * describe). */ -import { describe, it, expect, beforeAll, afterAll } from 'vitest' +import { describe, it, expect, beforeEach } from 'vitest' import { Brainy } from '../../src/brainy.js' import { NounType, VerbType } from '../../src/types/graphTypes.js' @@ -64,21 +39,14 @@ describe('GraphAdjacencyIndex Pagination', () => { .map((i) => idMapper().getUuid(Number(i))) .filter((u: string | undefined): u is string => u !== undefined) - /** - * Builds one central hub + 50 neighbor entities (all outgoing edges from - * the hub), unvectored and on in-memory storage (see the file header). - * Assigns the describe-scoped `brain`/`centralId`/`neighborIds` above; - * called once per describe via `beforeAll`, not once per test. - */ - async function buildFixture(): Promise { + beforeEach(async () => { brain = new Brainy({ requireSubtype: false }) - await brain.init({ storage: { type: 'memory' } }) + await brain.init() // Create central entity centralId = await brain.add({ data: { name: 'Central Hub' }, - type: NounType.Thing, - vector: [] + type: NounType.Thing }) // Create 50 neighbor entities with relationships @@ -86,8 +54,7 @@ describe('GraphAdjacencyIndex Pagination', () => { for (let i = 0; i < 50; i++) { const neighborId = await brain.add({ data: { name: `Neighbor ${i}`, index: i }, - type: NounType.Thing, - vector: [] + type: NounType.Thing }) neighborIds.push(neighborId) @@ -98,14 +65,9 @@ describe('GraphAdjacencyIndex Pagination', () => { type: VerbType.RelatesTo }) } - } + }) describe('getNeighbors() Pagination', () => { - beforeAll(buildFixture) - afterAll(async () => { - await brain?.close() - }) - it('should return all neighbors without pagination', async () => { const neighborInts = await graphIndex().getNeighbors(entityInt(centralId)) const neighbors = intsToUuids(neighborInts) @@ -187,8 +149,7 @@ describe('GraphAdjacencyIndex Pagination', () => { // Create some incoming relationships const sourceId = await brain.add({ data: { name: 'Source' }, - type: NounType.Thing, - vector: [] + type: NounType.Thing }) await brain.relate({ @@ -208,11 +169,6 @@ describe('GraphAdjacencyIndex Pagination', () => { }) describe('getVerbIdsBySource() Pagination', () => { - beforeAll(buildFixture) - afterAll(async () => { - await brain?.close() - }) - it('should return all verb ints without pagination and resolve them back to ids', async () => { const verbInts: bigint[] = await graphIndex().getVerbIdsBySource(entityInt(centralId)) @@ -267,11 +223,6 @@ describe('GraphAdjacencyIndex Pagination', () => { }) describe('getVerbIdsByTarget() Pagination', () => { - beforeAll(buildFixture) - afterAll(async () => { - await brain?.close() - }) - it('should return all verb ints targeting an entity', async () => { // Pick a neighbor that's a target of relationships const targetId = neighborIds[0] @@ -285,16 +236,14 @@ describe('GraphAdjacencyIndex Pagination', () => { // Create entity with many incoming relationships const popularTarget = await brain.add({ data: { name: 'Popular Target' }, - type: NounType.Thing, - vector: [] + type: NounType.Thing }) // Create 30 relationships pointing to it for (let i = 0; i < 30; i++) { const sourceId = await brain.add({ data: { name: `Source ${i}` }, - type: NounType.Thing, - vector: [] + type: NounType.Thing }) await brain.relate({ from: sourceId, @@ -318,11 +267,6 @@ describe('GraphAdjacencyIndex Pagination', () => { }) describe('Performance with Pagination', () => { - beforeAll(buildFixture) - afterAll(async () => { - await brain?.close() - }) - it('should maintain sub-5ms performance with pagination', async () => { const central = entityInt(centralId) @@ -341,17 +285,11 @@ describe('GraphAdjacencyIndex Pagination', () => { }) describe('Real-World Use Cases', () => { - beforeAll(buildFixture) - afterAll(async () => { - await brain?.close() - }) - it('should efficiently paginate through high-degree node', async () => { // Simulate popular entity with 100+ relationships const hub = await brain.add({ data: { name: 'Popular Hub' }, - type: NounType.Thing, - vector: [] + type: NounType.Thing }) // Create 100 relationships @@ -359,8 +297,7 @@ describe('GraphAdjacencyIndex Pagination', () => { for (let i = 0; i < 100; i++) { const targetId = await brain.add({ data: { name: `Target ${i}` }, - type: NounType.Thing, - vector: [] + type: NounType.Thing }) targetIds.push(targetId) await brain.relate({ diff --git a/tests/integration/metadata-vector-exclusion.test.ts b/tests/integration/metadata-vector-exclusion.test.ts index 0ca25388..9e11f9dc 100644 --- a/tests/integration/metadata-vector-exclusion.test.ts +++ b/tests/integration/metadata-vector-exclusion.test.ts @@ -26,7 +26,6 @@ import { describe, it, expect, beforeEach, afterEach } from 'vitest' import { Brainy } from '../../src/brainy.js' import { NounType } from '../../src/types/graphTypes.js' import { existsSync, rmSync } from 'fs' -import { MetadataArrayTooLargeError, MAX_INDEXED_ARRAY_LENGTH } from '../../src/errors/brainyError.js' describe('Metadata Vector Exclusion Fix', () => { let brainy: Brainy @@ -156,56 +155,29 @@ describe('Metadata Vector Exclusion Fix', () => { expect(results[0].entity.metadata?.name).toBe('Bob') }) - it('should REFUSE an array over the indexing bound, by name', async () => { - // A large array (not a vector, just bulk data). This used to be SKIPPED in - // silence at a bound of 10 — the field simply vanished from the index and - // the row dropped out of every `where` on it, indistinguishably from "no - // row matches". The bound is now MAX_INDEXED_ARRAY_LENGTH and it REFUSES. + it('should skip indexing large arrays (>10 elements)', async () => { + // Add entity with a large array (not a vector, just bulk data). const largeArray = Array.from({ length: 100 }, (_, i) => `item${i}`) - const err = await brainy - .add({ - type: NounType.Document, - data: 'Doc with large array', - metadata: { - name: 'Doc with large array', - items: largeArray - } - }) - .catch((e: any) => e) + await brainy.add({ + type: NounType.Document, + data: 'Doc with large array', + metadata: { + name: 'Doc with large array', + items: largeArray + } + }) - expect(err).toBeInstanceOf(MetadataArrayTooLargeError) - expect(err.field).toBe('items') - expect(err.length).toBe(100) - expect(err.limit).toBe(MAX_INDEXED_ARRAY_LENGTH) - - // Nothing was indexed from the refused write — no 'items' field, and above - // all no per-element numeric fields (the original explosion class). + // Large arrays (> 10 elements) are deliberately skipped to avoid indexing + // bulk/vector-like payloads: 'items' must NOT appear, and the 100 elements + // must NOT have produced 100 indexed fields. const fields = await brainy.getAvailableFields() expect(fields).not.toContain('items') const numericFields = fields.filter(f => /(^|\.)\d+$/.test(f)) expect(numericFields).toEqual([]) - }) - it('should index an array UP TO the bound — the old limit of 10 was the bug', async () => { - await brainy.add({ - type: NounType.Document, - data: 'Doc with a long-but-legitimate tag list', - metadata: { - name: 'Doc with many tags', - items: Array.from({ length: MAX_INDEXED_ARRAY_LENGTH }, (_, i) => `item${i}`) - } - }) - - const fields = await brainy.getAvailableFields() - // The field IS indexed now, and still without per-element numeric fields. - expect(fields).toContain('items') - expect(fields.filter(f => /(^|\.)\d+$/.test(f))).toEqual([]) - - // And the eleventh element — the one the old bound silently dropped the - // whole field for — really is searchable. - const hits = await brainy.find({ where: { items: 'item10' } }) - expect(hits.length).toBeGreaterThan(0) + // The scalar 'name' field IS indexed. + expect(fields).toContain('name') }) it('should preserve HNSW vector search functionality', async () => { diff --git a/tests/integration/readonly-close-no-marker.test.ts b/tests/integration/readonly-close-no-marker.test.ts index ad9357db..7bcf99df 100644 --- a/tests/integration/readonly-close-no-marker.test.ts +++ b/tests/integration/readonly-close-no-marker.test.ts @@ -149,12 +149,12 @@ describe('a read-only brain writes no clean-shutdown evidence', () => { brain = null // The FILE SET under `_system/` is unchanged — a reader creates and - // removes nothing. This pin is specifically about the generation store's - // clean-shutdown evidence. The wider law — that a reader leaves EVERY - // file under `_system/` byte-identical, which this fix left open as a - // known residual (the metadata field registry and the three statistics - // files were still re-stamped by a reader's close) — is closed and pinned - // in `readonly-close-writes-nothing.test.ts`. + // removes nothing. (Other files under `_system/` — e.g. the metadata + // field registry, which stamps its own `lastUpdated` on every persist — + // are a pre-existing, separate concern outside this fix's scope: this + // pin is specifically about the generation store's clean-shutdown + // evidence, not about every subsystem's close() being a true no-op for + // a reader.) const after = snapshotDir(systemDir()) expect([...after.keys()].sort()).toEqual([...before.keys()].sort()) diff --git a/tests/integration/readonly-close-writes-nothing.test.ts b/tests/integration/readonly-close-writes-nothing.test.ts deleted file mode 100644 index 701a1974..00000000 --- a/tests/integration/readonly-close-writes-nothing.test.ts +++ /dev/null @@ -1,261 +0,0 @@ -/** - * @module tests/integration/readonly-close-writes-nothing - * @description A READ-ONLY BRAIN LEAVES `_system/` BYTE-IDENTICAL — the WHOLE - * directory, not just the clean-shutdown marker. - * - * `readonly-close-no-marker` closed the marker half of this law and named the - * rest as a known, out-of-scope residual: - * - * "Other files under `_system/` — e.g. the metadata field registry, which - * stamps its own `lastUpdated` on every persist — are a pre-existing, - * separate concern outside this fix's scope." - * - * This is that residual, closed. MEASURED on the base before the fix, a - * read-only open → read → close rewrote FOUR files: - * - * _system/__metadata_field_registry__.json.gz - * _system/type-statistics.json.gz - * _system/subtype-statistics.json.gz - * _system/verb-subtype-statistics.json.gz - * - * THE CAUSE was not the closes the marker fix guarded — it was Phase 1 of - * `closeDurableSteps`, where every component flush ran unconditionally. A flush - * is a write by definition: `MetadataIndexManager#flush()` saves the field - * registry "even with no dirty fields" (its own comment), and the storage - * adapter's count flush re-stamps the three statistics files. A session that - * committed nothing re-stamped all four. Phase 2's closes were ungated too — - * the graph index's close drains both LSM MemTables and stamps a watermark, - * and the optional vector/metadata `close` hooks (unimplemented in the - * reference engine, filled in by a native provider) persist buffered state. - * - * THE LAW. A reader writes nothing, anywhere under `_system/`, at open or at - * close. It still RELEASES what it holds: the graph index's auto-flush interval - * is cleared through `stopBackgroundFlush()`, the non-writing half of its - * close, so nothing outlives the session. - * - * WHY IT MATTERS beyond tidiness: `_system/` is where a store keeps its - * evidence about itself — what the writer committed, what the projections have - * seen. A reader that rewrites any of it is vouching for a state it only - * observed, and on shared or snapshot storage it mutates bytes another process - * owns. - */ - -import { describe, it, expect, beforeEach, afterEach } from 'vitest' -import { mkdtempSync, rmSync, readdirSync, readFileSync, statSync } from 'node:fs' -import { createHash } from 'node:crypto' -import { tmpdir } from 'node:os' -import { join } from 'node:path' -import { Brainy } from '../../src/brainy.js' -import { NounType, VerbType } from '../../src/types/graphTypes.js' - -/** Recursively hash every regular file under `dir`, keyed by its path relative to `dir`. */ -function snapshotDir(dir: string): Map { - const out = new Map() - const walk = (rel: string): void => { - const abs = rel ? join(dir, rel) : dir - let entries: string[] - try { - entries = readdirSync(abs) - } catch { - return - } - for (const name of entries) { - const childRel = rel ? join(rel, name) : name - const childAbs = join(dir, childRel) - const st = statSync(childAbs) - if (st.isDirectory()) { - walk(childRel) - } else if (st.isFile()) { - out.set(childRel, createHash('sha256').update(readFileSync(childAbs)).digest('hex')) - } - } - } - walk('') - return out -} - -/** Every path where `after` differs from `before`, labelled — the failure message. */ -function diff(before: Map, after: Map): string[] { - const lines: string[] = [] - for (const [path, hash] of after) { - if (!before.has(path)) lines.push(`ADDED ${path}`) - else if (before.get(path) !== hash) lines.push(`CHANGED ${path}`) - } - for (const path of before.keys()) if (!after.has(path)) lines.push(`REMOVED ${path}`) - return lines.sort() -} - -describe('a read-only brain writes nothing under `_system/`', () => { - let dir: string - let brain: Brainy | null = null - - beforeEach(() => { - dir = mkdtempSync(join(tmpdir(), 'brainy-readonly-writes-')) - }) - - afterEach(async () => { - if (brain) { - try { - await brain.close() - } catch { - /* already closed */ - } - brain = null - } - try { - rmSync(dir, { recursive: true, force: true }) - } catch { - /* ignore */ - } - }) - - const systemDir = () => join(dir, '_system') - - /** - * A writer seeds a store with nouns, verbs and queryable metadata — enough - * that the field registry, the statistics files and the graph index all hold - * real content — then closes cleanly. - */ - async function seedStore(): Promise { - const writer = new Brainy({ - requireSubtype: false, - storage: { type: 'filesystem', path: dir } - }) - await writer.init() - for (let i = 0; i < 6; i++) { - await writer.add({ - id: `seed-${i}`, - data: `seed entity ${i}`, - type: i % 2 === 0 ? NounType.Concept : NounType.Document, - metadata: { lane: i % 2 === 0 ? 'alpha' : 'beta', rank: i, tags: [`t${i}`, 'shared'] }, - vector: [] - }) - } - for (let i = 1; i < 6; i++) { - await writer.relate({ from: 'seed-0', to: `seed-${i}`, type: VerbType.RelatedTo }) - } - await writer.flush() - await writer.close() - } - - it('open → read → close leaves every file under `_system/` byte-identical', async () => { - await seedStore() - - const before = snapshotDir(systemDir()) - expect(before.size, 'the writer left a populated `_system/`').toBeGreaterThan(0) - - brain = await Brainy.openReadOnly({ storage: { type: 'filesystem', path: dir } }) - expect(brain.isReadOnly).toBe(true) - - // Exercise the read surface that drives each subsystem: statistics (counts), - // a metadata filter (field index + registry), a graph walk (adjacency), a - // vector search, and a direct get. - await brain.stats() - await brain.find({ where: { lane: 'alpha' }, limit: 10 } as any) - await brain.find({ where: { tags: 'shared' }, limit: 10 } as any) - await brain.find({ connected: { from: 'seed-0', direction: 'out' }, limit: 10 } as any) - await brain.get('seed-1') - - await brain.close() - brain = null - - const after = snapshotDir(systemDir()) - const changes = diff(before, after) - expect(changes, `a reader modified \`_system/\`:\n${changes.join('\n')}`).toEqual([]) - }, 120_000) - - it('names the four files that used to change — the measured shape of the defect', async () => { - await seedStore() - const before = snapshotDir(systemDir()) - - // These are the exact paths the base rewrote. Naming them keeps the pin - // honest about what it caught: if a future change reintroduces the write, - // the test above fails and this one says which subsystem did it. - const previouslyRewritten = [ - '__metadata_field_registry__.json.gz', - 'type-statistics.json.gz', - 'subtype-statistics.json.gz', - 'verb-subtype-statistics.json.gz' - ] - for (const name of previouslyRewritten) { - expect(before.has(name), `fixture must contain ${name}`).toBe(true) - } - - brain = await Brainy.openReadOnly({ storage: { type: 'filesystem', path: dir } }) - await brain.stats() - await brain.find({ where: { lane: 'alpha' }, limit: 10 } as any) - await brain.close() - brain = null - - const after = snapshotDir(systemDir()) - for (const name of previouslyRewritten) { - expect(after.get(name), `${name} was rewritten by a reader`).toBe(before.get(name)) - } - }, 120_000) - - it('a reader that only opens and closes — touching nothing — writes nothing', async () => { - await seedStore() - const before = snapshotDir(systemDir()) - - brain = await Brainy.openReadOnly({ storage: { type: 'filesystem', path: dir } }) - await brain.close() - brain = null - - const changes = diff(before, snapshotDir(systemDir())) - expect(changes, `an idle reader modified \`_system/\`:\n${changes.join('\n')}`).toEqual([]) - }, 120_000) - - it('two readers in sequence each leave the store exactly as they found it', async () => { - await seedStore() - const before = snapshotDir(systemDir()) - - for (let i = 0; i < 2; i++) { - const reader = await Brainy.openReadOnly({ storage: { type: 'filesystem', path: dir } }) - await reader.find({ where: { lane: 'beta' }, limit: 10 } as any) - await reader.close() - const changes = diff(before, snapshotDir(systemDir())) - expect(changes, `reader ${i + 1} modified \`_system/\`:\n${changes.join('\n')}`).toEqual([]) - } - }, 120_000) - - it('the store outside `_system/` is untouched too — a reader writes nowhere', async () => { - await seedStore() - const before = snapshotDir(dir) - - brain = await Brainy.openReadOnly({ storage: { type: 'filesystem', path: dir } }) - await brain.stats() - await brain.find({ where: { lane: 'alpha' }, limit: 10 } as any) - await brain.close() - brain = null - - const changes = diff(before, snapshotDir(dir)) - expect(changes, `a reader modified the store:\n${changes.join('\n')}`).toEqual([]) - }, 120_000) - - it('a WRITER still persists on close — the guard did not disarm the write path', async () => { - await seedStore() - const before = snapshotDir(systemDir()) - - const writer = new Brainy({ requireSubtype: false, storage: { type: 'filesystem', path: dir } }) - await writer.init() - await writer.add({ - id: 'after-reader', - data: 'a new row', - type: NounType.Concept, - metadata: { lane: 'gamma', rank: 99 }, - vector: [] - }) - await writer.close() - - // The writer's close DID move `_system/` — that is the whole point of the - // asymmetry, and the guard must not have flattened it. - expect(diff(before, snapshotDir(systemDir())).length).toBeGreaterThan(0) - - // And the row is really there on the next open. - const reopened = new Brainy({ requireSubtype: false, storage: { type: 'filesystem', path: dir } }) - await reopened.init() - brain = reopened - const hits = await reopened.find({ where: { lane: 'gamma' }, limit: 10 } as any) - expect(hits.length).toBe(1) - }, 120_000) -}) diff --git a/tests/unit/utils/metadataIndex-array-bound.test.ts b/tests/unit/utils/metadataIndex-array-bound.test.ts deleted file mode 100644 index cbbf6b63..00000000 --- a/tests/unit/utils/metadataIndex-array-bound.test.ts +++ /dev/null @@ -1,242 +0,0 @@ -/** - * @module tests/unit/utils/metadataIndex-array-bound - * @description THE INDEXABLE-ARRAY BOUND — a law with a name and a refusal, - * not a `continue`. - * - * THE DEFECT. An array-valued metadata field indexes one posting per element, - * so the index has always carried a ceiling. It was 10, and it was applied by a - * bare `continue` deep inside field extraction: - * - * if (Array.isArray(value) && value.length > 10) continue - * - * A row whose `tags` array held ELEVEN entries therefore had that field skipped - * entirely — no posting, no error, no warning. The row then failed to match - * every filtered search on `tags`, including `{ tags: 'a-tag-it-really-has' }`, - * and the caller had no way to tell that from "no row matches". Eleven tags is - * not an exotic shape; the eleventh tag made the row invisible. - * - * THE LAW. Arrays of scalars index up to {@link MAX_INDEXED_ARRAY_LENGTH} = 64, - * hardcoded (the zero-config law: no knob), which clears every legitimate - * multi-value field and stays far below any embedding width. Above it the WRITE - * IS REFUSED by name — `MetadataArrayTooLargeError`, carrying the field, the - * length and the bound — at `add`, `update`, `relate` and `updateRelation` - * alike. Nothing is skipped in silence. - * - * THE ONE PLACE THE BOUND STILL SKIPS is a row already on disk, written by an - * older engine under the old rule and read back by a rebuild, a catch-up fold - * or a remove. Refusing there would make an existing store un-rebuildable — so - * the row is admitted and the skipped field is NARRATED. Both sides are pinned. - */ -import { describe, it, expect, beforeEach, vi, afterEach } from 'vitest' -import { Brainy } from '../../../src/brainy' -import { NounType, VerbType } from '../../../src/types/graphTypes' -import { MetadataArrayTooLargeError, MAX_INDEXED_ARRAY_LENGTH } from '../../../src/errors/brainyError' -import { resolveEntityId } from '../../../src/utils/idNormalization' -import { prodLog } from '../../../src/utils/logger' - -/** `n` distinct scalar tags. */ -function tags(n: number, prefix = 't'): string[] { - return Array.from({ length: n }, (_, i) => `${prefix}${i}`) -} - -describe('the indexable-array bound', () => { - let brain: Brainy - - beforeEach(async () => { - brain = new Brainy({ requireSubtype: false, storage: { type: 'memory' } }) - await brain.init() - }) - - describe('BELOW the bound: the array indexes, every element of it', () => { - it('the eleven-element array that used to vanish is searchable', async () => { - // ELEVEN — one over the old silent limit, the whole shape of the defect. - await brain.add({ - id: 'eleven', - data: 'a row with eleven tags', - type: NounType.Document, - metadata: { tags: tags(11) }, - vector: [] - }) - - // Every element is a posting, including the eleventh. - for (const tag of tags(11)) { - const hits = await brain.find({ where: { tags: tag }, limit: 10 } as any) - expect(hits.map((r: any) => r.id)).toContain(resolveEntityId('eleven')) - } - }) - - it('indexes right up to the bound — all 64 elements', async () => { - await brain.add({ - id: 'at-bound', - data: 'a row at the bound', - type: NounType.Document, - metadata: { tags: tags(MAX_INDEXED_ARRAY_LENGTH) }, - vector: [] - }) - - // The first, the last, and one in the middle. - for (const tag of ['t0', `t${MAX_INDEXED_ARRAY_LENGTH - 1}`, 't31']) { - const hits = await brain.find({ where: { tags: tag }, limit: 10 } as any) - expect(hits.map((r: any) => r.id)).toContain(resolveEntityId('at-bound')) - } - }) - - it('a nested bag\'s array indexes under its dotted address', async () => { - await brain.add({ - id: 'nested', - data: 'a row with a nested tag list', - type: NounType.Document, - metadata: { facets: { labels: tags(20, 'l') } }, - vector: [] - }) - const hits = await brain.find({ where: { 'facets.labels': 'l19' }, limit: 10 } as any) - expect(hits.map((r: any) => r.id)).toContain(resolveEntityId('nested')) - }) - }) - - describe('ABOVE the bound: the write is refused, by name', () => { - const OVER = MAX_INDEXED_ARRAY_LENGTH + 1 - - it('add() throws a typed error naming the field, the length and the bound', async () => { - const err = await brain - .add({ - id: 'too-many', - data: 'a row with too many tags', - type: NounType.Document, - metadata: { tags: tags(OVER) }, - vector: [] - } as any) - .catch((e: any) => e) - - expect(err).toBeInstanceOf(MetadataArrayTooLargeError) - expect(err.field).toBe('tags') - expect(err.length).toBe(OVER) - expect(err.limit).toBe(MAX_INDEXED_ARRAY_LENGTH) - expect(err.type).toBe('VALIDATION') - // The message carries all three, and names the cures. - expect(err.message).toContain('tags') - expect(err.message).toContain(String(OVER)) - expect(err.message).toContain(String(MAX_INDEXED_ARRAY_LENGTH)) - expect(err.message).toContain('vector') - }) - - it('the refused row is not written at all — no half-indexed ghost', async () => { - await expect( - brain.add({ - id: 'refused', - data: 'refused', - type: NounType.Document, - metadata: { tags: tags(OVER) }, - vector: [] - } as any) - ).rejects.toBeInstanceOf(MetadataArrayTooLargeError) - - expect(await brain.get('refused')).toBeNull() - const hits = await brain.find({ where: { tags: 't0' }, limit: 10 } as any) - expect(hits.map((r: any) => r.id)).not.toContain(resolveEntityId('refused')) - }) - - it('a 384-float embedding parked in the metadata bag is refused, not swallowed', async () => { - const err = await brain - .add({ - id: 'bag-vector', - data: 'an embedding in the wrong place', - type: NounType.Document, - metadata: { embedding: Array.from({ length: 384 }, (_, i) => i / 384) }, - vector: [] - } as any) - .catch((e: any) => e) - - expect(err).toBeInstanceOf(MetadataArrayTooLargeError) - expect(err.field).toBe('embedding') - expect(err.length).toBe(384) - }) - - it('update() refuses it too', async () => { - await brain.add({ - id: 'grow', - data: 'starts small', - type: NounType.Document, - metadata: { tags: tags(3) }, - vector: [] - }) - await expect( - brain.update({ id: 'grow', metadata: { tags: tags(OVER) } } as any) - ).rejects.toBeInstanceOf(MetadataArrayTooLargeError) - - // And the row keeps the values it had. - const hits = await brain.find({ where: { tags: 't1' }, limit: 10 } as any) - expect(hits.map((r: any) => r.id)).toContain(resolveEntityId('grow')) - }) - - it('relate() refuses it on a verb\'s metadata', async () => { - await brain.add({ id: 'a', data: 'a', type: NounType.Thing, vector: [] }) - await brain.add({ id: 'b', data: 'b', type: NounType.Thing, vector: [] }) - await expect( - brain.relate({ - from: 'a', - to: 'b', - type: VerbType.RelatedTo, - metadata: { tags: tags(OVER) } - } as any) - ).rejects.toBeInstanceOf(MetadataArrayTooLargeError) - }) - - it('a nested oversize array is refused under its dotted address', async () => { - const err = await brain - .add({ - id: 'nested-over', - data: 'nested and too long', - type: NounType.Document, - metadata: { facets: { labels: tags(OVER, 'l') } }, - vector: [] - } as any) - .catch((e: any) => e) - expect(err).toBeInstanceOf(MetadataArrayTooLargeError) - expect(err.field).toBe('facets.labels') - }) - }) - - describe('a row already on disk is admitted, and the skip is NARRATED', () => { - afterEach(() => { - vi.restoreAllMocks() - }) - - it('extraction over an old oversize row warns by field, length and bound', async () => { - const warn = vi.spyOn(prodLog, 'warn').mockImplementation(() => {}) - const index = (brain as any).metadataIndex - - // The shape an older engine persisted: the write door never saw it, so - // this reaches extraction directly — exactly as a rebuild or a remove - // reading the row back would. - const fields = index.extractIndexableFields({ - metadata: { tags: tags(MAX_INDEXED_ARRAY_LENGTH + 5), keep: 'me' } - }) - - // The oversize field contributes nothing... - expect(fields.filter((f: any) => f.field === 'tags')).toHaveLength(0) - // ...the rest of the row indexes normally — the row is not rejected... - expect(fields.some((f: any) => f.field === 'keep' && f.value === 'me')).toBe(true) - // ...and the skip is said out loud, with everything needed to act on it. - expect(warn).toHaveBeenCalled() - const said = warn.mock.calls.map((c: any[]) => String(c[0])).join('\n') - expect(said).toContain('tags') - expect(said).toContain(String(MAX_INDEXED_ARRAY_LENGTH + 5)) - expect(said).toContain(String(MAX_INDEXED_ARRAY_LENGTH)) - expect(said).toContain('NOT indexed') - }) - - it('an at-bound row on disk is indexed in full and says nothing', async () => { - const warn = vi.spyOn(prodLog, 'warn').mockImplementation(() => {}) - const index = (brain as any).metadataIndex - - const fields = index.extractIndexableFields({ - metadata: { tags: tags(MAX_INDEXED_ARRAY_LENGTH) } - }) - expect(fields.filter((f: any) => f.field === 'tags')).toHaveLength(MAX_INDEXED_ARRAY_LENGTH) - - const said = warn.mock.calls.map((c: any[]) => String(c[0])).join('\n') - expect(said).not.toContain('indexing bound') - }) - }) -}) diff --git a/tests/unit/utils/metadataIndex-sparse-range-collation.test.ts b/tests/unit/utils/metadataIndex-sparse-range-collation.test.ts deleted file mode 100644 index d6d00568..00000000 --- a/tests/unit/utils/metadataIndex-sparse-range-collation.test.ts +++ /dev/null @@ -1,258 +0,0 @@ -/** - * @module tests/unit/utils/metadataIndex-sparse-range-collation - * @description RANGE QUERIES ON THE LEGACY SPARSE INDEX — order, or a refusal. - * Never a confidently ordered wrong answer. - * - * THE TWO RANGE PATHS. `getIdsForRange` routes a `gte` / `lt` / `between` two - * ways. The column store compares RAW values and is correct. The legacy sparse - * chunk index — the pre-7.20.0 fallback, still read for workspaces that have - * not been rebuilt — compared `normalizeValue()` output, and `normalizeValue` - * carries an escape hatch that destroys order on purpose: a string over 100 - * characters is replaced by a short hash so it can serve as a filesystem-safe - * key. Ordering hashes ranks rows by digest. - * - * THE DEFECT, IN TWO SHAPES. - * - * (a) A LONG BOUND against ordinary values. `where: { title: { gte: } }` collapsed the BOUND to `__HASH_…`, whose - * leading underscores sort below every letter — so a bound that should - * have excluded everything matched the entire field instead. This is the - * shape that reaches a caller who never stored a long value at all. - * - * (b) LONG VALUES in the index. A field whose values ran long was persisted - * hashed, so its order is not recoverable from this index at all. The old - * code compared the digests anyway and returned a subset chosen by hash. - * - * THE LAW. Bounds are normalized WITHOUT the hash escape hatch, so a long - * bound stays comparable — (a) is simply fixed. Where the persisted KEY is a - * hash, the order does not exist to be computed, and the query throws a typed - * `BrainyError('INVALID_QUERY')` naming the field and the cure — (b) is - * refused by name. Loud beats wrong. - * - * THE FIXTURE is a genuine legacy index: it is written through the same - * `ChunkManager` / `SparseIndex` doors a pre-7.20.0 engine wrote through, with - * keys normalized exactly as that engine normalized them, into a field the - * column store does not serve. The chunk WRITE path was removed in 11be039, so - * this is the only way the shape the read path exists for can be built. - * - * NOT CLAIMED HERE. The persisted keys are also lower-cased and trimmed by - * `normalizeValue`, so this path's string ranges are case-INSENSITIVE where - * the column store's are not. The raw values are not in the index to compare — - * that divergence is a property of the bytes on disk and it ends when the - * column store adopts the field. It is named in `getIdsFromChunksForRange`'s - * doc comment rather than papered over. - */ -import { describe, it, expect, beforeEach } from 'vitest' -import { Brainy } from '../../../src/brainy' -import { NounType } from '../../../src/types/graphTypes' -import { SparseIndex, ChunkManager } from '../../../src/utils/metadataIndexChunking' -import { BrainyError } from '../../../src/errors/brainyError' - -/** The field the legacy index covers — deliberately never given to a row, so - * the column store never learns it and the sparse fallback is the only path. */ -const FIELD = 'legacyTitle' - -/** - * Write a legacy sparse index for `field` exactly as a pre-7.20.0 engine did: - * one chunk, keys normalized through the index's own `normalizeValue`, ids as - * roaring bitmaps, a zone map and a bloom filter over the chunk. - * - * @param brain - The live brain whose metadata index gains the legacy field. - * @param field - Field name to index. - * @param valueToIds - Raw value → the entity ids that carried it. - */ -async function writeLegacySparseIndex( - brain: any, - field: string, - valueToIds: Array<[string, string[]]> -): Promise { - const index = brain.metadataIndex - const chunkManager: ChunkManager = index.chunkManager - const sparseIndex = new SparseIndex(field) - - // The keys a pre-7.20.0 writer persisted: normalizeValue output, hash escape - // hatch and all. This is what makes the fixture the real shape. - const chunk = await chunkManager.createChunk(field) - for (const [value, ids] of valueToIds) { - const key = index.normalizeValue(value, field) - for (const id of ids) await chunkManager.addToChunk(chunk, key, id) - } - await chunkManager.saveChunk(chunk) - - sparseIndex.registerChunk( - { - chunkId: chunk.chunkId, - field, - valueCount: chunk.entries.size, - idCount: Array.from(chunk.entries.values()).reduce((s: number, b: any) => s + b.size, 0), - zoneMap: (chunkManager as any).calculateZoneMap(chunk), - lastUpdated: Date.now(), - splitThreshold: 80, - mergeThreshold: 20 - }, - chunkManager.createBloomFilter(chunk) - ) - - await index.saveSparseIndex(field, sparseIndex) -} - -/** A deterministic string of `n` characters starting with `lead`. */ -function longString(lead: string, n: number): string { - return lead + 'x'.repeat(n - lead.length) -} - -describe('legacy sparse index: range queries order values, or refuse', () => { - let brain: Brainy - let index: any - let ids: string[] - - beforeEach(async () => { - brain = new Brainy({ requireSubtype: false, storage: { type: 'memory' } }) - await brain.init() - index = (brain as any).metadataIndex - - // Rows exist (so the id mapper can resolve them) but carry NO `legacyTitle` - // — the column store must not serve the field the pins query. - ids = [] - for (let i = 0; i < 3; i++) { - const id = `row-${i}` - await brain.add({ id, data: `row ${i}`, type: NounType.Thing, metadata: { lane: 'a' }, vector: [] }) - ids.push(id) - } - expect(index.columnStore.hasField(FIELD)).toBe(false) - }) - - describe('(a) a long BOUND against ordinary short values', () => { - // 'apple' < 'mango' < 'zebra', and every bound below is compared against - // these three raw keys. - beforeEach(async () => { - await writeLegacySparseIndex(brain, FIELD, [ - ['apple', [ids[0]]], - ['mango', [ids[1]]], - ['zebra', [ids[2]]] - ]) - }) - - it('the fixture: the values are stored raw, the long bound is what hashes', () => { - expect(index.normalizeValue('apple', FIELD)).toBe('apple') - // The bound is what the old code collapsed — and a digest sorts below - // every letter, which is exactly why `gte` matched everything. - const bound = longString('zzz', 120) - expect(index.normalizeValue(bound, FIELD)).toMatch(/^__HASH_/) - expect(index.normalizeValue(bound, FIELD) < 'apple').toBe(true) - }) - - it('gte a bound above every value matches NOTHING (it used to match all)', async () => { - const bound = longString('zzz', 120) - const matched = await index.getIdsForRange(FIELD, bound, undefined, true, true) - expect(matched).toEqual([]) - }) - - it('lte a bound above every value matches EVERY value', async () => { - const bound = longString('zzz', 120) - const matched = await index.getIdsForRange(FIELD, undefined, bound, true, true) - expect(matched).toHaveLength(3) - }) - - it('gte a long bound below every value matches every value', async () => { - const bound = longString('aaa', 120) - const matched = await index.getIdsForRange(FIELD, bound, undefined, true, true) - expect(matched).toHaveLength(3) - }) - - it('a long bound orders BETWEEN the values, not below all of them', async () => { - // 'mmm…' sits between 'mango' and 'zebra'. - const bound = longString('mmm', 120) - const matched = await index.getIdsForRange(FIELD, bound, undefined, true, true) - expect(matched).toHaveLength(1) - }) - - it('short bounds are unchanged — the ordinary case still orders correctly', async () => { - expect(await index.getIdsForRange(FIELD, 'b', undefined, true, true)).toHaveLength(2) - expect(await index.getIdsForRange(FIELD, undefined, 'n', true, true)).toHaveLength(2) - expect(await index.getIdsForRange(FIELD, 'b', 'n', true, true)).toHaveLength(1) - // Strict bounds stay strict. - expect(await index.getIdsForRange(FIELD, 'mango', undefined, false, true)).toHaveLength(1) - expect(await index.getIdsForRange(FIELD, 'mango', undefined, true, true)).toHaveLength(2) - }) - }) - - describe('(b) long VALUES — the index holds hashes, so the range is refused', () => { - beforeEach(async () => { - await writeLegacySparseIndex(brain, FIELD, [ - [longString('alpha', 140), [ids[0]]], - [longString('mike', 140), [ids[1]]], - [longString('zulu', 140), [ids[2]]] - ]) - }) - - it('the fixture: the persisted keys really are hashes', async () => { - const chunk = await index.chunkManager.loadChunk(FIELD, 0) - const keys = Array.from(chunk.entries.keys()) as string[] - expect(keys).toHaveLength(3) - for (const k of keys) expect(k).toMatch(/^__HASH_/) - // And their digest order is NOT their value order — the wrong answer the - // old code returned was wrong, not merely arbitrary. - const digestOrder = [...keys].sort() - const valueOrder = [ - index.normalizeValue(longString('alpha', 140), FIELD), - index.normalizeValue(longString('mike', 140), FIELD), - index.normalizeValue(longString('zulu', 140), FIELD) - ] - expect(digestOrder).not.toEqual(valueOrder) - }) - - it('a range over the hashed field throws a typed refusal naming the field', async () => { - await expect( - index.getIdsForRange(FIELD, longString('mike', 140), undefined, true, true) - ).rejects.toThrow(BrainyError) - - const err = await index - .getIdsForRange(FIELD, longString('mike', 140), undefined, true, true) - .catch((e: any) => e) - expect(err).toBeInstanceOf(BrainyError) - expect(err.type).toBe('INVALID_QUERY') - expect(err.message).toContain(FIELD) - expect(err.message).toContain('hash') - // The cure is named, not left to the caller to guess. - expect(err.message).toContain('repairIndex') - }) - - it('every range shape refuses — gte, lte and between alike', async () => { - const lo = longString('alpha', 140) - const hi = longString('zulu', 140) - for (const [min, max] of [ - [lo, undefined], - [undefined, hi], - [lo, hi] - ] as Array<[any, any]>) { - const err = await index.getIdsForRange(FIELD, min, max, true, true).catch((e: any) => e) - expect(err).toBeInstanceOf(BrainyError) - expect(err.type).toBe('INVALID_QUERY') - } - }) - - it('EQUALITY still works on the hashed field — only ordering is refused', async () => { - const matched = await index.getIds(FIELD, longString('mike', 140)) - expect(matched).toHaveLength(1) - }) - }) - - describe('numeric ranges on the legacy path are untouched', () => { - beforeEach(async () => { - await writeLegacySparseIndex(brain, FIELD, [ - ['5', [ids[0]]], - ['50', [ids[1]]], - ['500', [ids[2]]] - ]) - }) - - it('numbers still compare numerically, not lexicographically', async () => { - // The whole point of compareNormalizedValues: "50" < "500" numerically - // even though "500" < "50" would hold as strings by prefix. - expect(await index.getIdsForRange(FIELD, 10, undefined, true, true)).toHaveLength(2) - expect(await index.getIdsForRange(FIELD, undefined, 100, true, true)).toHaveLength(2) - expect(await index.getIdsForRange(FIELD, 10, 100, true, true)).toHaveLength(1) - }) - }) -})