20 changed files with 75 additions and 2216 deletions
--- a/docs/FIND_SYSTEM.md
+++ b/docs/FIND_SYSTEM.md
@ -369,71 +369,6 @@ return results.slice(offset, offset + limit)
 // → Auto-correction: Use most likely alternative based on affinity data
 ```
 ## Field Projection (`fields`)
 `find()` and `get()` accept a `fields` list. Without it they return the whole
 record; with it they return only the fields you name — and, where the index can
 supply them, without opening the canonical record at all.
 ```ts
 // A list page: two user fields and one engine scalar. No document bodies.
 await brain.find({
  where: { kind: 'post' },
  fields: ['title', 'slug', 'system.createdAt'],
  limit: 50
 })
 await brain.get(id, { fields: ['title'] })
 ```
 ### Why it exists
 A list view that renders a title and a date does not need the body, but without
 a projection every row hydrates its full record and throws almost all of it
 away. On a posts list that is the dominant cost of the query.
 ### The rules
 | | |
 |---|---|
 | **`fields` absent** | The full record, byte-identical to before. Nothing changes. |
 | **Field names** | The one addressing law: a bare name is user metadata (`'title'`), `system.*` is an engine scalar (`'system.createdAt'`). |
 | **A field the row lacks** | Simply **absent** from the result. Never an error. |
 | **Identity** | Every row keeps its `id` (and `score` on `find`) regardless — a row you cannot identify is not a row. |
 | **Where values come from** | The **column store**, which holds raw values. Never the sparse index, which buckets timestamps for range queries. |
 | **A field the column cannot serve** | The canonical record is read for that field only. Correct, just not free. |
 ### Missing fields are absent, not errors
 This is deliberate and differs from `orderBy`, which throws
 `UnresolvableFieldError` for an unknown field. A typo in `orderBy` silently
 changes the ordering, so it must be loud. A projection asks "give me these if
 you have them", and an optional field must not turn a list into a failure — so
 `fields` uses the permissive path.
 ### Cost
 When every named field is column-served, a projected page performs **zero**
 canonical reads. When one is not, only that read happens and the rest still come
 from the index. Both are pinned by counting reads rather than timing them, in
 `tests/integration/find-fields-projection.test.ts`.
 ### `related()` takes no `fields`
 A `Relation` carries `from` and `to` as **ids** and hydrates no entity record,
 so there is nothing for a projection to trim. Projecting the endpoints would be
 a new capability rather than a projection of an existing one.
 ### For engine implementers
 Projection is served through an optional provider door,
 `getScalarsForIds(ids, fields)` on `MetadataIndexProvider`. The contract is in
 `src/plugin.ts`; the short version is **return only what you can serve exactly,
 and say what you served**. The caller diffs the answer against the request and
 reads records for the remainder, so omission costs a read while a wrong value is
 a wrong answer nobody can see. An engine without the door still works — every
 field falls back to the record.
 ## Performance Characteristics
 ### Query Performance by Type
--- a/docs/api-contract.json
+++ b/docs/api-contract.json
@ -1507,7 +1507,6 @@
    "BrainyError",
    "DerivedArtifactMissingError",
    "GraphIndexNotReadyError",
    "MetadataArrayTooLargeError",
    "MetadataIndexNotReadyError",
    "MigrationInProgressError",
    "ProtectedArtifactError",
--- a/src/brainy.ts
+++ b/src/brainy.ts
@ -4193,16 +4193,6 @@ export class Brainy<T = any> implements BrainyInterface<T> {
    }
    // Route to metadata-only or full entity based on options
    // A PROJECTED get goes through the same seam every list page uses, so a
    // detail read of two scalars costs an index read rather than a record read.
    // It is checked before `includeVectors` because the two are incompatible by
    // construction: a projection returns the named fields, and a vector is not
    // one of them unless it was named.
    if (options?.fields !== undefined && options.fields.length > 0) {
      const page = await this.#hydratePage([id], options.fields)
      return page.get(id) ?? null
    }
    const includeVectors = options?.includeVectors ?? false  // Default: metadata-only (fast)
    if (includeVectors) {
@ -4249,170 +4239,6 @@ export class Brainy<T = any> implements BrainyInterface<T> {
   * const children = childIds.map(id => childrenMap.get(id)).filter(Boolean)
   * ```
   */
  /**
   * **The projection seam** — hydrate a page of ids under an optional `fields`
   * projection, opening the canonical record only when the index cannot serve
   * what was asked for.
   *
   * Without a projection this is exactly `batchGet`, byte for byte: the whole
   * point is that `fields` absent changes nothing.
   *
   * With one, the order is: ask the index for the named scalars in a single
   * batched door; see which requested fields it actually served; and read
   * records ONLY if something is still missing — and only to fill those fields.
   * A page whose every requested field is index-served performs zero canonical
   * reads, which is the whole reason the door exists.
   *
   * `guardFields` are fetched ALONGSIDE the projection and trimmed off before
   * the caller sees them. find()'s index-integrity guard re-validates every row
   * against its own predicate, and it reads the entity to do so — so a row
   * projected down to `title` would fail a `where: { kind }` it genuinely
   * matches, and the whole page would vanish. The fields a filter names are
   * fields the index can serve by definition, so carrying them costs nothing
   * and keeps the guard honest.
   *
   * A field nothing can supply is simply absent from the row. That is the
   * permissive law: a projection asks "these, if you have them", and an
   * optional field must not turn a list into an exception. It deliberately does
   * NOT route through the strict address resolver, which throws
   * `UnresolvableFieldError` for an unknown key — that strictness is right for
   * `orderBy`, where a typo silently changes the order, and wrong here, where
   * the honest answer is "this row does not have that".
   *
   * @param ids - Canonical ids for the page.
   * @param fields - The projection, or undefined for the full record.
   * @returns `id → entity`, projected when `fields` was given.
   */
  /**
   * The index keys find()'s integrity guard reads when it re-validates a row.
   *
   * The guard calls `entityMatchesFind(entity, params)`, so a projected entity
   * must still carry whatever the params constrain — otherwise a row that
   * genuinely matches is dropped for lacking the evidence. These are fetched
   * with the projection and trimmed off before the caller sees them.
   *
   * @param params - The find params.
   * @returns Index keys to carry through hydration.
   */
  #guardFieldsFor(params: FindParams<T>): string[] {
    const keys: string[] = []
    if (params.where && typeof params.where === 'object') {
      // Top-level where keys only: nested `anyOf`/`allOf` branches are carried
      // by their own keys when the guard walks them, and a filter whose
      // evidence is missing keeps the row (the guard's own catch) rather than
      // dropping it.
      for (const key of Object.keys(params.where as Record<string, unknown>)) {
        if (key === 'anyOf' || key === 'allOf' || key === 'not') continue
        keys.push(key)
      }
    }
    if (params.type !== undefined) keys.push('system.type')
    if (params.subtype !== undefined) keys.push('system.subtype')
    if (params.service !== undefined) keys.push('system.service')
    if (params.excludeVFS === true) keys.push('vfsType', 'isVFSEntity')
    return keys
  }
  async #hydratePage(
    ids: string[],
    fields?: readonly string[],
    guardFields: readonly string[] = []
  ): Promise<Map<string, Entity<T>>> {
    if (fields === undefined || fields.length === 0) return this.batchGet(ids)
    const wanted = [...new Set([...fields, ...guardFields])]
    const provider = this.metadataIndex as unknown as MetadataIndexProvider
    let served = new Map<string, Record<string, unknown>>()
    if (typeof provider.getScalarsForIds === 'function') {
      served = await provider.getScalarsForIds(ids, wanted)
    }
    // Which ids still owe a field? Only those cost a record read, and a page
    // that owes nothing costs none at all.
    const owing: string[] = []
    for (const id of ids) {
      const row = served.get(id)
      if (row === undefined || wanted.some((f) => !(f in row))) owing.push(id)
    }
    // The records are read for the OWED fields only; everything the index
    // already served is used as-is, so a body field pulls its own record and
    // no more than that.
    const records = owing.length > 0 ? await this.batchGet(owing) : new Map<string, Entity<T>>()
    const out = new Map<string, Entity<T>>()
    for (const id of ids) {
      const fromIndex = served.get(id)
      const record = records.get(id)
      // An id neither the index nor storage knows is not a row.
      if (fromIndex === undefined && record === undefined) continue
      out.set(id, this.#projectEntity(id, wanted, fromIndex, record))
    }
    return out
  }
  /**
   * Build one projected entity: `id`, plus exactly the requested fields that
   * something could supply.
   *
   * Values come from the index first and the record second, and they must agree
   * — the index only reports what it can serve exactly, so a field it served is
   * the record's value. A field neither has is omitted rather than set to
   * `undefined`: absent and present-and-undefined are different answers, and a
   * caller checking `'slug' in row.metadata` deserves the true one.
   *
   * @param id - The entity id, always present on the result.
   * @param fields - The requested index keys.
   * @param fromIndex - What the index served for this id, if anything.
   * @param record - The canonical entity, if one had to be read.
   * @returns The projected entity.
   */
  #projectEntity(
    id: string,
    fields: readonly string[],
    fromIndex: Record<string, unknown> | undefined,
    record: Entity<T> | undefined
  ): Entity<T> {
    const projected: Record<string, unknown> = { id }
    const metadata: Record<string, unknown> = {}
    let sawMetadata = false
    for (const field of fields) {
      let value: unknown
      let found = false
      if (fromIndex !== undefined && field in fromIndex) {
        value = fromIndex[field]
        found = true
      } else if (record !== undefined) {
        if (field.startsWith('system.')) {
          const inner = field.slice('system.'.length)
          const bag = record as unknown as Record<string, unknown>
          if (inner in bag && bag[inner] !== undefined) {
            value = bag[inner]
            found = true
          }
        } else {
          const bag = (record.metadata ?? {}) as Record<string, unknown>
          if (field in bag) {
            value = bag[field]
            found = true
          }
        }
      }
      if (!found) continue
      if (field.startsWith('system.')) {
        projected[field.slice('system.'.length)] = value
      } else {
        metadata[field] = value
        sawMetadata = true
      }
    }
    if (sawMetadata) projected.metadata = metadata
    return projected as unknown as Entity<T>
  }
  async batchGet(ids: string[], options?: GetOptions): Promise<Map<string, Entity<T>>> {
    // Canonical read (see get): resolves by id from storage, no derived index.
    await this.ensureInitialized({ needs: [] })
@ -8211,7 +8037,7 @@ export class Brainy<T = any> implements BrainyInterface<T> {
        // Batch-load entities for 10x faster cloud storage performance
        // GCS: 10 entities = 1×50ms vs 10×50ms = 500ms (10x faster)
-        const entitiesMap = await this.#hydratePage(pageIds, params.fields, this.#guardFieldsFor(params))
+        const entitiesMap = await this.batchGet(pageIds)
        for (const id of pageIds) {
          const entity = entitiesMap.get(id)
          if (entity) {
@ -8248,7 +8074,7 @@ export class Brainy<T = any> implements BrainyInterface<T> {
          if (hiddenIds.size > 0) allUuids = allUuids.filter((id) => !hiddenIds.has(id))
          const pageIds = allUuids.slice(offset, offset + limit)
-          const entitiesMap = await this.#hydratePage(pageIds, params.fields, this.#guardFieldsFor(params))
+          const entitiesMap = await this.batchGet(pageIds)
          for (const id of pageIds) {
            const entity = entitiesMap.get(id)
            if (entity) {
@ -8276,7 +8102,7 @@ export class Brainy<T = any> implements BrainyInterface<T> {
          const pageIds = filteredIds.slice(offset, offset + limit)
          // Batch-load entities for 10x faster cloud storage performance
-          const entitiesMap = await this.#hydratePage(pageIds, params.fields, this.#guardFieldsFor(params))
+          const entitiesMap = await this.batchGet(pageIds)
          for (const id of pageIds) {
            const entity = entitiesMap.get(id)
            if (entity) {
@ -8486,23 +8312,7 @@ export class Brainy<T = any> implements BrainyInterface<T> {
          // Rank by score (top offset+limit), then drop the offset — identical ordering
          // to a full `sort((a, b) => b.score - a.score)` + slice, but the native
          // `sort:topK` provider can compute only the page instead of the full sort.
-          //
+          if (results.length >= offset + limit) {
          // ONLY when score IS the requested order. An explicit `orderBy` names a
          // different ordering key, and this block cannot serve it: it ranks by
          // score and CUTS the page, so the tail's `orderBy` sort below either
          // never runs at all (the early return, when there is no `connected` /
          // `fusion` work left) or runs over a page that score already chose —
          // ordering eight rows relevance picked instead of the eight the field
          // ordering asks for. Both readings were silent: `find({ query, where,
          // orderBy })` answered in score order while `find({ where, orderBy })`
          // answered in field order, and nothing said the request had been dropped.
          //
          // With `orderBy` present the candidate set falls through UNCUT to the
          // tail, which orders it in full and pages that ordering — "page last",
          // the graph-first law applied to ordering rather than to filtering. The
          // set is bounded by the legs (the text matches inside the universe plus
          // the beam walk's `limit * 2`), not by the store.
          if (!params.orderBy && results.length >= offset + limit) {
            const k = offset + limit
            const order = rankIndicesByScore(results.map(r => r.score), k, true)
            results = reorderByIndices(results, order).slice(offset, k)
@ -8527,7 +8337,7 @@ export class Brainy<T = any> implements BrainyInterface<T> {
          // Batch-load entities for current page - O(page_size) instead of O(total_results)
          // GCS: 10 entities = 1×50ms vs 10×50ms = 500ms (10x faster)
-          const entitiesMap = await this.#hydratePage(pageIds, params.fields, this.#guardFieldsFor(params))
+          const entitiesMap = await this.batchGet(pageIds)
          for (const id of pageIds) {
            const entity = entitiesMap.get(id)
            if (entity) {
@ -8555,7 +8365,7 @@ export class Brainy<T = any> implements BrainyInterface<T> {
              // Batch-load entities for paginated results (10x faster on GCS)
              const sortedResults: Result<T>[] = []
-              const entitiesMap = await this.#hydratePage(pageIds, params.fields, this.#guardFieldsFor(params))
+              const entitiesMap = await this.batchGet(pageIds)
              for (const id of pageIds) {
                const entity = entitiesMap.get(id)
                if (entity) {
@ -8660,28 +8470,6 @@ export class Brainy<T = any> implements BrainyInterface<T> {
      })
    }
    // PROJECTION TRIM — applied once, here, AFTER the integrity guard, so every
    // find() path is trimmed uniformly and the guard still saw the evidence it
    // needs. Hydration carried the guard's fields alongside the projection;
    // this removes them, leaving exactly what the caller named.
    //
    // Rows that reached here from a path the seam does not hydrate (a vector or
    // text leg builds its own entities) are trimmed from what they already
    // hold, so the ANSWER is the same everywhere — only the cost differs, and
    // only on the paths that still read a record.
    if (params.fields !== undefined && params.fields.length > 0 && result.length > 0) {
      const named = [...new Set(params.fields)]
      result = result.map((r) => {
        const projected = this.#projectEntity(
          r.id,
          named,
          undefined,
          r.entity as unknown as Entity<T>
        )
        return { ...r, entity: projected } as typeof r
      })
    }
    // includeVectors — opt-in vector hydration. Default (false) keeps the perf
    // contract: every result path above builds entities via the metadata-only
    // fast path, so `entity.vector` is the empty stub. When requested, fetch the
@ -13342,7 +13130,7 @@ export class Brainy<T = any> implements BrainyInterface<T> {
      }
      return this._flushQueued
    }
-    return this.#startFlushLeader()
+    return this.startFlushLeader()
  }
  /**
@ -13351,23 +13139,16 @@ export class Brainy<T = any> implements BrainyInterface<T> {
   * the ONE queued waiter (if any) is promoted. The `finally` callback returns
   * nothing on purpose: a callback that returned the promoted run's promise
   * would make the leader await its own follower.
   *
   * ECMAScript-private (`#`), not TypeScript `private`: `private` is erased at
   * compile time, so the method would still land on the prototype — and the
   * contract manifest reads the surface the BUILD exposes, so it would emit
   * this as a contract door. A door is a promise every engine implementing the
   * contract must keep; this is the flush gate's own bookkeeping. `#` keeps it
   * off the prototype, where the emitter cannot see it.
   * @returns The leader's own promise, settling on its own body alone.
   */
-  #startFlushLeader(): Promise<void> {
+  private startFlushLeader(): Promise<void> {
    const run = this._runFlush()
    // `finally` and not `then`: a failed flush must still open the gate, or
    // one rejection would wedge every later flush behind a promise nobody
    // will ever settle.
    const gated: Promise<void> = run.finally(() => {
      if (this._flushInFlight === gated) this._flushInFlight = null
-      this.#promoteQueuedFlush()
+      this.promoteQueuedFlush()
    })
    this._flushInFlight = gated
    return gated
@ -13378,12 +13159,9 @@ export class Brainy<T = any> implements BrainyInterface<T> {
   * leader and settle its deferred from that run. Never throws into the
   * leader's `finally`: a synchronous failure starting the promoted run is
   * reported to the waiter, which must be settled on every path.
   *
   * ECMAScript-private for the same reason as the leader starter above:
   * internals are not doors.
   * @returns Nothing.
   */
-  #promoteQueuedFlush(): void {
+  private promoteQueuedFlush(): void {
    const settle = this._flushQueuedSettle
    if (!settle) return
    // Clear BEFORE starting, so the promoted run's own joiners queue afresh
@ -13391,7 +13169,7 @@ export class Brainy<T = any> implements BrainyInterface<T> {
    this._flushQueued = null
    this._flushQueuedSettle = null
    try {
-      this.#startFlushLeader().then(settle.resolve, settle.reject)
+      this.startFlushLeader().then(settle.resolve, settle.reject)
    } catch (error) {
      settle.reject(error)
    }
@ -17064,7 +16842,7 @@ export class Brainy<T = any> implements BrainyInterface<T> {
      ordered = valued.map((v) => v.id)
    }
    const pageIds = ordered.slice(offset, offset + limit)
-    const entitiesMap = await this.#hydratePage(pageIds, params.fields, this.#guardFieldsFor(params))
+    const entitiesMap = await this.batchGet(pageIds)
    const results: Result<T>[] = []
    for (const id of pageIds) {
      const entity = entitiesMap.get(id)
@ -20844,45 +20622,34 @@ export class Brainy<T = any> implements BrainyInterface<T> {
    // Phase 1: Flush ALL components in parallel to persist buffered data
    // This is critical when cor native providers buffer data in Rust memory
    //
    // READ-ONLY GUARD, applied to EVERY flush here. A flush is a write by
    // definition, and a reader has nothing of its own to persist — but these
    // calls were not conditional, so a read-only open → read → close REWROTE
    // four files under `_system/`: the metadata field registry (whose flush()
    // saves it unconditionally, "even with no dirty fields"), and the three
    // type/subtype statistics files the storage adapter's count flush stamps.
    // Every one of them was re-stamped on a session that committed nothing.
    // A reader must leave `_system/` exactly as it found it — the same law the
    // clean-shutdown marker already lives under (see the generation-store
    // guard below and `Brainy.openReadOnly`).
    await Promise.all([
      // Flush HNSW dirty nodes (deferred persistence mode)
      (async () => {
-        if (this.index && !this.isReadOnly && typeof this.index.flush === 'function') {
+        if (this.index && typeof this.index.flush === 'function') {
          await this.index.flush()
        }
      })(),
      // Flush metadata index (field indexes + EntityIdMapper)
      (async () => {
-        if (this.metadataIndex && !this.isReadOnly && typeof this.metadataIndex.flush === 'function') {
+        if (this.metadataIndex && typeof this.metadataIndex.flush === 'function') {
          await this.metadataIndex.flush()
        }
      })(),
      // Flush graph adjacency index (LSM trees)
      (async () => {
-        if (this.graphIndex && !this.isReadOnly && typeof this.graphIndex.flush === 'function') {
+        if (this.graphIndex && typeof this.graphIndex.flush === 'function') {
          await this.graphIndex.flush()
        }
      })(),
      // Flush storage adapter counts
      (async () => {
-        if (this.storage && !this.isReadOnly && typeof this.storage.flushCounts === 'function') {
+        if (this.storage && typeof this.storage.flushCounts === 'function') {
          await this.storage.flushCounts()
        }
      })(),
      // Flush aggregation index state
      (async () => {
-        if (this._aggregationIndex && !this.isReadOnly) {
+        if (this._aggregationIndex) {
          await this._aggregationIndex.flush()
        }
      })(),
@ -20931,37 +20698,21 @@ export class Brainy<T = any> implements BrainyInterface<T> {
    // Phase 2: Close components to release resources (timers, file handles)
    // Data is already safe on disk from Phase 1
    //
    // READ-ONLY GUARD, same law as Phase 1. Each of these closes is a WRITER:
    // the graph index drains both LSM MemTables to SSTables and stamps its
    // watermark, and the vector/metadata `close` hooks — optional doors the
    // reference engine leaves unimplemented, but which a native provider fills
    // in — persist their buffered state. None of that is a reader's to write.
    //
    // A reader still has to RELEASE what it holds, which is why this is a
    // branch rather than a skip: `stopBackgroundFlush()` is the non-writing
    // half of the graph index's close, clearing the auto-flush interval that
    // would otherwise outlive the session. The optional hooks have no
    // non-writing counterpart to call, and a provider that buffers nothing on
    // a read-only open has nothing to release.
    await Promise.all([
      (async () => {
-        if (!this.graphIndex) return
+        if (this.graphIndex && typeof this.graphIndex.close === 'function') {
        if (this.isReadOnly) {
          this.graphIndex.stopBackgroundFlush()
        } else if (typeof this.graphIndex.close === 'function') {
          await this.graphIndex.close()
        }
      })(),
      (async () => {
        const index = this.index as JsHnswVectorIndex & VectorIndexOptionalHooks
-        if (index && !this.isReadOnly && typeof index.close === 'function') {
+        if (index && typeof index.close === 'function') {
          await index.close()
        }
      })(),
      (async () => {
        const metadataIndex = this.metadataIndex as MetadataIndexManager & MetadataIndexOptionalHooks
-        if (metadataIndex && !this.isReadOnly && typeof metadataIndex.close === 'function') {
+        if (metadataIndex && typeof metadataIndex.close === 'function') {
          await metadataIndex.close()
        }
      })(),
--- a/src/errors/brainyError.ts
+++ b/src/errors/brainyError.ts
@ -405,68 +405,3 @@ export class MigrationInProgressError extends BrainyError {
        }
    }
 }
 /**
 * THE INDEXABLE-ARRAY BOUND. An array-valued metadata field indexes one posting
 * per element, so an unbounded array is an unbounded write — a 384-float
 * embedding parked in the metadata bag would mint 384 postings for one row.
 * The bound exists to keep that out of the index.
 *
 * 64 is hardcoded on purpose (the zero-config law: no knob). It sits far above
 * every legitimate multi-value field the engine has seen — tags, authors,
 * categories, labels, participant lists — and far below any real embedding
 * width, so the two populations do not overlap and no caller has to tune it.
 *
 * It replaces a limit of 10 that was applied SILENTLY: a row whose `tags` array
 * held eleven entries had that field skipped entirely and dropped out of every
 * filtered search on it, with no error, no warning and no way to tell the
 * difference from "no row matches". A rule this consequential is a law with a
 * name and a refusal, not a `continue`.
 */
 export const MAX_INDEXED_ARRAY_LENGTH = 64
 /**
 * A metadata field carries an array longer than {@link MAX_INDEXED_ARRAY_LENGTH}.
 *
 * Thrown at the WRITE door (`add` / `update` / `relate` / `updateRelation`), so
 * the caller learns at the moment of writing that the field will not be
 * searchable — rather than discovering it later as rows that quietly fail to
 * match. Carries the field, its length and the bound so a handler can report
 * or repair without parsing the message.
 *
 * The cure is one of: store the long array outside the indexed bag (`data`
 * carries arbitrary content and is not indexed element-wise); pass an embedding
 * as the first-class `vector` parameter, which is where a vector belongs; or
 * shorten the field to the values that are actually queried.
 */
 export class MetadataArrayTooLargeError extends BrainyError {
    /** The metadata field whose array is too long (its full dotted address). */
    public readonly field: string
    /** How many elements that array holds. */
    public readonly length: number
    /** The bound it exceeded — {@link MAX_INDEXED_ARRAY_LENGTH}. */
    public readonly limit: number
    constructor(site: string, field: string, length: number, limit: number) {
        super(
            `${site}: metadata field '${field}' holds ${length} array elements, ` +
                `over the ${limit}-element indexing bound. An array field indexes one ` +
                `posting per element, so an unbounded array is an unbounded write. ` +
                `This write is refused rather than indexed partially or skipped silently ` +
                `— a skipped field drops the row out of every filtered search on '${field}' ` +
                `with no way to tell that from "nothing matched". ` +
                `Cures: put the long array in 'data' (stored, not indexed element-wise); ` +
                `pass an embedding as the first-class 'vector' parameter; or keep only ` +
                `the values you actually query in '${field}'.`,
            'VALIDATION',
            false
        )
        this.name = 'MetadataArrayTooLargeError'
        this.field = field
        this.length = length
        this.limit = limit
        if (Error.captureStackTrace) {
            Error.captureStackTrace(this, MetadataArrayTooLargeError)
        }
    }
 }
--- a/src/graph/graphAdjacencyIndex.ts
+++ b/src/graph/graphAdjacencyIndex.ts
@ -1105,31 +1105,13 @@ export class GraphAdjacencyIndex implements GraphIndexProvider {
  }
  /**
-   * Stop the auto-flush interval WITHOUT writing anything.
+   * Clean shutdown
   *
   * The non-writing half of {@link close}, for a shutdown that must leave the
   * store byte-identical — a read-only brain's close. `close()` itself is a
   * writer: it drains both LSM MemTables to SSTables and stamps the watermark,
   * which is exactly right for a writer and forbidden for a reader. A reader
   * still has to release this interval, though: it is the one piece of this
   * index that outlives the close and could fire against a store the session no
   * longer owns.
   *
   * @returns Nothing.
   */
-  stopBackgroundFlush(): void {
+  async close(): Promise<void> {
    if (this.flushTimer) {
      clearInterval(this.flushTimer)
      this.flushTimer = undefined
    }
  }
  /**
   * Clean shutdown — drains both trees and stamps the watermark. THIS WRITES;
   * a read-only brain must call {@link stopBackgroundFlush} instead.
   */
  async close(): Promise<void> {
    this.stopBackgroundFlush()
    // Close both LSM-trees (will flush MemTables to SSTables)
    if (this.initialized) {
--- a/src/index.ts
+++ b/src/index.ts
@ -203,7 +203,7 @@ export { EntityNotFoundError, RelationNotFoundError } from './errors/notFound.js
 // Base error + typed migration-lock error — thrown by any data-plane call while a
 // brain runs its one-time 7.x→8.0 upgrade; catch to answer HTTP 503 + Retry-After.
-export { BrainyError, MigrationInProgressError, GraphIndexNotReadyError, MetadataIndexNotReadyError, VectorIndexNotReadyError, ProtectedArtifactError, DerivedArtifactMissingError, MetadataArrayTooLargeError, MAX_INDEXED_ARRAY_LENGTH } from './errors/brainyError.js'
+export { BrainyError, MigrationInProgressError, GraphIndexNotReadyError, MetadataIndexNotReadyError, VectorIndexNotReadyError, ProtectedArtifactError, DerivedArtifactMissingError } from './errors/brainyError.js'
 export type { BrainyErrorType } from './errors/brainyError.js'
 // ============= 8.0 Db API — generational MVCC =============
--- a/src/indexes/columnStore/ColumnStore.ts
+++ b/src/indexes/columnStore/ColumnStore.ts
@ -292,58 +292,6 @@ export class ColumnStore implements ColumnStoreProvider {
    return result
  }
  /**
   * Read this column's value for each of `entityIntIds` — the per-id read
   * behind `find({ fields })`.
   *
   * Every other read door here answers "which entities have this value". A
   * projection asks the opposite — "what value does this entity have" — and
   * without it a projection has to go to the canonical record for a field the
   * column is already holding.
   *
   * The column is walked ONCE and the wanted ids are picked out as they pass,
   * so the cost is O(column) per field rather than O(ids x column). Later
   * sources win: the tail buffer holds writes newer than any segment, and
   * within the segments a later one supersedes an earlier, exactly as `filter`
   * treats them.
   *
   * Values are EXACT — this store keeps raw values, not the bucketed form the
   * sparse index uses for range queries — which is what makes it safe to
   * project from. Deleted entities are skipped; an id with no value in this
   * column is simply absent from the result.
   *
   * @param field - Field name to read.
   * @param entityIntIds - Entity integer ids to read values for.
   * @returns `entityIntId -> value` for the ids this column holds.
   */
  async valuesForIds(
    field: string,
    entityIntIds: Iterable<number>
  ): Promise<Map<number, number | string>> {
    const wanted = new Set<number>(entityIntIds)
    const out = new Map<number, number | string>()
    if (wanted.size === 0 || !this.hasField(field)) return out
    const deleted = this.deletedEntities.get(field)
    const take = (entry: { value: number | string; entityIntId: number }): void => {
      if (!wanted.has(entry.entityIntId)) return
      if (deleted && deleted.has(entry.entityIntId)) return
      out.set(entry.entityIntId, entry.value)
    }
    // Segments oldest -> newest, then the tail: a later write overwrites an
    // earlier one for the same id.
    const cursors = await this.getSegmentCursors(field)
    for (const cursor of cursors) {
      for (const entry of cursor.iterateForward()) take(entry)
    }
    const tailCursor = this.getTailBufferCursor(field)
    if (tailCursor) {
      for (const entry of tailCursor.iterateForward()) take(entry)
    }
    return out
  }
  /**
   * Range filter: find entities where field is within the bounds.
   *
--- a/src/neural/embeddedPatterns.ts
+++ b/src/neural/embeddedPatterns.ts
@ -2,7 +2,7 @@
 * 🧠 BRAINY EMBEDDED PATTERNS
 * 
 * AUTO-GENERATED - DO NOT EDIT
- * Generated: 2026-08-27T09:18:45-07:00
+ * Generated: 2025-09-29T10:10:00-07:00
 * Patterns: 220
 * Coverage: 94-98% of all queries
 * 
--- a/src/plugin.ts
+++ b/src/plugin.ts
@ -495,45 +495,6 @@ export interface MetadataIndexProvider {
    query: string,
    ids: readonly string[]
  ): Promise<Array<{ id: string; matchCount: number }>>
  /**
   * @description OPTIONAL: read named SCALAR fields for many ids at once, from
   * the index's own value storage, WITHOUT touching the canonical record.
   *
   * This is the door behind `find/get/related({ fields })`. A list view that
   * needs a title and a slug currently hydrates the whole record for every row
   * — document bodies included — and then discards almost all of it. Serving
   * the named scalars from the index turns that into an index read.
   *
   * ## The contract, and the one rule that makes it safe
   *
   * **Return only what you can serve EXACTLY, and say what you served.** The
   * answer is a per-id map of the fields this index actually resolved; the
   * caller diffs it against what was requested and reads the canonical record
   * for the remainder. An implementation must therefore OMIT a field rather
   * than approximate it — and omission costs only a record read, while a wrong
   * value is a wrong answer nobody can see.
   *
   * That rule is not hypothetical. This engine's own index buckets
   * `system.createdAt` and `system.updatedAt` to the minute for range queries,
   * so it cannot serve them exactly and omits them. An engine whose column
   * store holds raw values can serve the same fields — so the two answer
   * differently in COST and identically in CONTENT, which is the only
   * difference a projection door is allowed to have.
   *
   * A field absent from an entity is simply absent from that entity's map. It
   * is never an error, and never a `null` standing in for one: absent and
   * present-and-null are different answers.
   *
   * @param ids - Canonical entity ids to read.
   * @param fields - Index KEYS (bare = user metadata, `system.*` = engine
   *   scalar), already address-resolved by the caller.
   * @returns `id → { field: value }` for the fields this index served exactly.
   *   Ids with nothing to serve may be omitted entirely.
   */
  getScalarsForIds?(
    ids: readonly string[],
    fields: readonly string[]
  ): Promise<Map<string, Record<string, unknown>>>
  getSortedIdsForFilter(filter: any, orderBy: string, order?: 'asc' | 'desc', topK?: number): Promise<string[]>
  getFilterValues(field: string): Promise<string[]>
  getFilterFields(): Promise<string[]>
--- a/src/types/brainy.types.ts
+++ b/src/types/brainy.types.ts
@ -561,33 +561,6 @@ export interface UpdateRelationParams<T = any> {
 * refusal with the fix in hand beats a silent behavior flip.
 */
 export interface FindParams<T = any> {
  /**
   * **Field projection** — return only these fields on each row, instead of the
   * whole record.
   *
   * A list view that shows a title and a slug does not need the document body,
   * yet without a projection every row hydrates its full record and throws
   * almost all of it away. Naming the fields lets them be served from the index
   * itself: a scalar the index holds exactly is read from the index, and the
   * canonical record is opened ONLY when a requested field cannot be.
   *
   * Field names follow the one addressing law: a bare name is the user's
   * metadata (`'title'`), and `system.*` is an engine scalar
   * (`'system.createdAt'`).
   *
   * - **Absent** ⇒ the full record, exactly as before.
   * - A requested field the entity does not carry is simply **absent** from the
   *   row. It is never an error — a projection asks "give me these if you have
   *   them", so an optional field must not turn a list into a failure.
   * - Every returned row carries `id` (and, on `find`, `score`) regardless: a
   *   row you cannot identify is not a row.
   *
   * @example
   * // A list page: two user fields and one engine scalar, no document bodies.
   * await brain.find({ where: { kind: 'post' }, fields: ['title', 'slug', 'system.createdAt'], limit: 50 })
   */
  fields?: readonly string[]
  // Vector Intelligence
  /** Natural language or semantic search query (embedded and matched via HNSW + text index) */
  query?: string
@ -816,12 +789,6 @@ export interface SimilarParams<T = any> {
 * Added string ID shorthand syntax
 */
 export interface RelatedParams {
  // NOTE: `fields` is deliberately NOT offered here. A Relation carries `from`
  // and `to` as IDS and hydrates no entity record, so there is nothing for a
  // projection to trim — the param would be decorative. Projecting the
  // ENDPOINTS would be a new capability (related() hydrating entities), not a
  // projection of an existing one, and it belongs in its own decision.
  /**
   * Filter by source entity ID
   *
@ -1447,33 +1414,6 @@ export interface ImportResult {
 *
 */
 export interface GetOptions {
  /**
   * **Field projection** — return only these fields on each row, instead of the
   * whole record.
   *
   * A list view that shows a title and a slug does not need the document body,
   * yet without a projection every row hydrates its full record and throws
   * almost all of it away. Naming the fields lets them be served from the index
   * itself: a scalar the index holds exactly is read from the index, and the
   * canonical record is opened ONLY when a requested field cannot be.
   *
   * Field names follow the one addressing law: a bare name is the user's
   * metadata (`'title'`), and `system.*` is an engine scalar
   * (`'system.createdAt'`).
   *
   * - **Absent** ⇒ the full record, exactly as before.
   * - A requested field the entity does not carry is simply **absent** from the
   *   row. It is never an error — a projection asks "give me these if you have
   *   them", so an optional field must not turn a list into a failure.
   * - Every returned row carries `id` (and, on `find`, `score`) regardless: a
   *   row you cannot identify is not a row.
   *
   * @example
   * // A list page: two user fields and one engine scalar, no document bodies.
   * await brain.find({ where: { kind: 'post' }, fields: ['title', 'slug', 'system.createdAt'], limit: 50 })
   */
  fields?: readonly string[]
  /**
   * Include 384-dimensional vector embeddings in the response
   *
--- a/src/utils/metadataIndex.ts
+++ b/src/utils/metadataIndex.ts
@ -40,7 +40,7 @@ import {
 import { EntityIdMapper } from './entityIdMapper.js'
 import { RoaringBitmap32, roaringLibraryInitialize } from './roaring/index.js'
 import { FieldTypeInference, FieldType } from './fieldTypeInference.js'
-import { BrainyError, MAX_INDEXED_ARRAY_LENGTH } from '../errors/brainyError.js'
+import { BrainyError } from '../errors/brainyError.js'
 /**
 * Fields whose values are stored in the sparse index as BUCKETED values
@ -289,10 +289,8 @@ export class MetadataIndexManager implements MetadataIndexProvider {
      // No name-based exclude/allow lists — the field-addressing law: every
      // user field indexes, whatever its name ('content', 'data', 'id',
      // 'vector', … included). Bulk payloads are kept out by uniform value-
-      // SHAPE rules in extractIndexableFields (arrays longer than
+      // SHAPE rules in extractIndexableFields (arrays >10 never become
-      // MAX_INDEXED_ARRAY_LENGTH never become posting scalars, and the write
+      // posting scalars; >100-char values index hashed), never by name.
      // door refuses them by name; >100-char values index hashed), never by
      // field name.
    }
    // Initialize metadata cache with similar config to search cache
@ -963,41 +961,9 @@ export class MetadataIndexManager implements MetadataIndexProvider {
  }
  /**
-   * Get IDs for a range using the legacy chunked sparse index (zone maps +
+   * Get IDs for a range using chunked sparse index with zone maps and roaring bitmaps
-   * roaring bitmaps). Lazy-loaded via UnifiedCache.
+   * Now fully lazy-loaded via UnifiedCache (no local sparseIndices Map)
-   *
+   * Normalize min/max for timestamp bucketing before comparison
   * ORDER IS NOT A KEY. This path compares NORMALIZED values, and
   * {@link normalizeValue} carries an escape hatch that is order-destroying by
   * design: a string over 100 characters is replaced by {@link hashValue}'s
   * digest so it can be used as a filesystem-safe key. Feeding that digest to
   * an ORDERING comparison — which is what a `gte` / `lt` / `between` does —
   * ranks rows by hash. The result is not empty and not an error: it is a
   * confidently ordered wrong answer, and it disagrees with the column-store
   * path (`getIdsForRange` above), which compares raw values and is correct.
   *
   * Two changes hold the line here:
   *
   * 1. THE BOUNDS ARE NEVER HASHED. They are normalized with `allowHash =
   *    false`, so a long bound stays comparable instead of collapsing to a
   *    digest. This alone fixes the common shape — a long bound queried
   *    against ordinary short values, where the digest sorts below every
   *    letter and `gte` therefore matched the entire store.
   *
   * 2. A HASHED KEY IS REFUSED, NEVER GUESSED. The persisted keys are whatever
   *    the pre-7.20.0 writer normalized them to, so a field whose values ran
   *    long is stored hashed and its order is simply not recoverable from this
   *    index. Rather than compare digests, the query throws a typed
   *    `BrainyError('INVALID_QUERY')` naming the field, the bound and the cure.
   *    Loud beats wrong.
   *
   * KNOWN, NAMED DIVERGENCE. The persisted keys are also lower-cased and
   * trimmed by `normalizeValue`, so this path's string ranges are
   * CASE-INSENSITIVE where the column store's are not. That is a property of
   * the bytes a pre-7.20.0 engine wrote, not of the comparison: the raw values
   * are not in the index to compare. The bounds are normalized into the same
   * case-folded space so the comparison is at least self-consistent, and the
   * divergence disappears with the field itself once the column store adopts
   * it. See the module note on `getIdsFromChunks` for the path's lifetime.
   */
  private async getIdsFromChunksForRange(
    field: string,
@ -1013,27 +979,9 @@ export class MetadataIndexManager implements MetadataIndexProvider {
    }
    // Normalize min/max for consistent comparison with indexed values
-    // (indexed values are bucketed for timestamps, so we must bucket the query
+    // (indexed values are bucketed for timestamps, so we must bucket the query bounds too)
-    // bounds too) — but NEVER through the hash escape hatch, which would make
+    const normalizedMin = min !== undefined ? this.normalizeValue(min, field) : undefined
-    // the bound incomparable. See the doc comment above.
+    const normalizedMax = max !== undefined ? this.normalizeValue(max, field) : undefined
    const normalizedMin = min !== undefined ? this.normalizeValue(min, field, false) : undefined
    const normalizedMax = max !== undefined ? this.normalizeValue(max, field, false) : undefined
    // REFUSE BEFORE SELECTING. Chunk selection itself orders values: it tests
    // the bounds against each chunk's zone-map min/max. If those are hashes the
    // selection is already meaningless — and its failure mode is an EMPTY
    // answer (no chunk appears to overlap), which is the quietest wrong answer
    // of all. So the key space is checked here, before a single chunk is
    // chosen, and again per key below for a chunk whose zone map happens to
    // read clean.
    for (const chunkId of sparseIndex.getAllChunkIds()) {
      const zoneMap = sparseIndex.getChunk(chunkId)?.zoneMap
      for (const bound of [zoneMap?.min, zoneMap?.max]) {
        if (typeof bound === 'string' && MetadataIndexManager.isHashedValue(bound)) {
          throw MetadataIndexManager.rangeOverHashedIndex(field)
        }
      }
    }
    // Find candidate chunks using zone maps
    const candidateChunkIds = sparseIndex.findChunksForRange(normalizedMin, normalizedMax)
@ -1048,13 +996,6 @@ export class MetadataIndexManager implements MetadataIndexProvider {
      const chunk = await this.chunkManager.loadChunk(field, chunkId)
      if (chunk) {
        for (const [value, bitmap] of chunk.entries) {
          // A hashed key carries no order. Refuse the range rather than rank by
          // digest — the whole answer is unsound, so failing on the first one
          // is the honest outcome.
          if (MetadataIndexManager.isHashedValue(value)) {
            throw MetadataIndexManager.rangeOverHashedIndex(field)
          }
          // Check if value is in range using numeric-aware comparison
          // (normalizeValue converts numbers to strings, so we must compare numerically)
          let inRange = true
@ -1083,25 +1024,6 @@ export class MetadataIndexManager implements MetadataIndexProvider {
    return this.idMapper.intsIterableToUuids(allIntIds)
  }
  /**
   * The refusal a range query gets when the legacy sparse index holds hashed
   * keys for the field. Names the field and the cure; never a wrong answer.
   */
  private static rangeOverHashedIndex(field: string): BrainyError {
    return new BrainyError(
      `Range query on field "${field}" cannot be served by the legacy sparse index: ` +
        `its values were persisted as hashes (values over 100 characters are stored ` +
        `hashed to stay within filesystem name limits), and a hash carries no order — ` +
        `comparing them would return a confidently ordered wrong answer. ` +
        `Equality (\`where: { ${field}: value }\`) still works on this index. ` +
        `To range over this field, let the column store adopt it: run ` +
        `brain.repairIndex({ rebuild: ['metadata'] }), which rebuilds the field into ` +
        `the column store, where ranges compare raw values.`,
      'INVALID_QUERY',
      false
    )
  }
  /**
   * Get roaring bitmap for a field-value pair without converting to UUIDs
   * This is used for fast multi-field intersection queries using hardware-accelerated bitmap AND
@ -1269,17 +1191,8 @@ export class MetadataIndexManager implements MetadataIndexProvider {
   * value-based detection (DuckDB-inspired). Analyzes actual data values, not names.
   *
   * NO FALLBACKS - Pure value-based detection only.
   *
   * @param value - The value to normalize.
   * @param field - Optional field name (drives the per-field statistics strategy).
   * @param allowHash - Whether the >100-character escape hatch may fire. TRUE
   *   everywhere a normalized value is used as a KEY (equality postings, chunk
   *   entries, filenames) — that is what the hash exists for. FALSE on the
   *   ORDER-comparing path: a hash is deliberately order-destroying, so a
   *   bound that hashes can only be compared as nonsense. See
   *   {@link isHashedValue} and `getIdsFromChunksForRange`.
   */
-  private normalizeValue(value: any, field?: string, allowHash: boolean = true): string {
+  private normalizeValue(value: any, field?: string): string {
    if (value === null || value === undefined) return '__NULL__'
    if (typeof value === 'boolean') return value ? '__TRUE__' : '__FALSE__'
@ -1337,34 +1250,21 @@ export class MetadataIndexManager implements MetadataIndexProvider {
    // Default normalization
    if (typeof value === 'number') return value.toString()
    if (Array.isArray(value)) {
-      const joined = value.map(v => this.normalizeValue(v, field, allowHash)).join(',')
+      const joined = value.map(v => this.normalizeValue(v, field)).join(',')
      // Hash very long array values to avoid filesystem limits
-      if (allowHash && joined.length > 100) {
+      if (joined.length > 100) {
        return this.hashValue(joined)
      }
      return joined
    }
    const stringValue = String(value).toLowerCase().trim()
    // Hash very long string values to avoid filesystem limits
-    if (allowHash && stringValue.length > 100) {
+    if (stringValue.length > 100) {
      return this.hashValue(stringValue)
    }
    return stringValue
  }
  /**
   * Is this normalized value a HASH rather than the value itself?
   *
   * {@link hashValue} is an escape hatch for filesystem name limits, and it is
   * deliberately order-destroying: two values whose hashes compare one way
   * routinely compare the other way themselves. Anything that ORDERS normalized
   * values has to know when it is holding one, because comparing hashes yields
   * a confident, wrong answer rather than an error.
   */
  private static isHashedValue(normalized: string): boolean {
    return normalized.startsWith('__HASH_')
  }
  /**
   * Create a short hash for long values to avoid filesystem filename limits
   */
@ -1389,10 +1289,9 @@ export class MetadataIndexManager implements MetadataIndexProvider {
   *   'content', 'vector' in a bag are ordinary user fields)
   * - Record-frame plumbing (vector, connections, level, data, _rev, id)
   *   never indexes — that is namespace routing, not a name carve-out
-   * - Value-SHAPE rules apply uniformly to all names: arrays longer than
+   * - Value-SHAPE rules apply uniformly to all names: arrays >10 never
-   *   MAX_INDEXED_ARRAY_LENGTH never become posting scalars (and say so —
+   *   become posting scalars; purely numeric key names (array indices)
-   *   the write door refuses them outright); purely numeric key names
+   *   skip; >100-char values index hashed (normalizeValue)
   *   (array indices) skip; >100-char values index hashed (normalizeValue)
   */
  private extractIndexableFields(data: any): Array<{ field: string, value: any }> {
    const fields: Array<{ field: string, value: any }> = []
@ -1454,37 +1353,13 @@ export class MetadataIndexManager implements MetadataIndexProvider {
        // This catches vectors stored as objects: {0: 0.1, 1: 0.2, ...}
        if (/^\d+$/.test(key)) continue
-        // THE INDEXABLE-ARRAY BOUND ({@link MAX_INDEXED_ARRAY_LENGTH}). An
+        // Skip large arrays (> 10 elements) - likely vectors or bulk data
-        // array field mints one posting per element, so the index has always
+        if (Array.isArray(value) && value.length > 10) continue
        // carried a ceiling — it was 10, and it was applied by this bare
        // `continue`: an eleven-element `tags` array had its whole field
        // skipped and the row dropped out of every filtered search on it, with
        // no error, no warning, and nothing to distinguish that from "no row
        // matches". The ceiling is not the defect; the silence was.
        //
        // The write door refuses this shape by name now
        // (`MetadataArrayTooLargeError`, thrown from paramValidation's
        // `rejectOversizeIndexArrays`), so a live add/update never reaches
        // here over the bound. Reaching it means the row is ALREADY on disk —
        // written by an older engine under the old rule — and this is a
        // rebuild, a catch-up fold or a remove reading it back. Refusing there
        // would make an existing store un-rebuildable, so the row is admitted
        // and the skipped field is NARRATED instead. Never silent, either way.
        if (Array.isArray(value) && value.length > MAX_INDEXED_ARRAY_LENGTH) {
          prodLog.warn(
            `[brainy] metadata field '${fullKey}' holds ${value.length} array elements, ` +
            `over the ${MAX_INDEXED_ARRAY_LENGTH}-element indexing bound — the field is ` +
            `NOT indexed for this row, so it will not match a where-clause on '${fullKey}'. ` +
            `This row predates the bound (the write door refuses this shape now). ` +
            `Move the long array into 'data', or pass an embedding as the 'vector' parameter.`
          )
          continue
        }
        if (value && typeof value === 'object' && !Array.isArray(value)) {
          // Recurse into nested objects (but not arrays), keeping the frame
          extract(value, fullKey, frame)
-        } else if (Array.isArray(value)) {
+        } else if (Array.isArray(value) && value.length <= 10) {
          // Small arrays: index as multi-value field (all with same field name)
          // Example: tags: ["javascript", "node"] → field="tags", value="javascript" + field="tags", value="node"
          for (const item of value) {
@ -2930,67 +2805,6 @@ export class MetadataIndexManager implements MetadataIndexProvider {
    return order === 'asc' ? comparison : -comparison
  }
  /**
   * Read named scalar fields for many ids from the COLUMN STORE, without
   * touching the canonical record — the `find({ fields })` door.
   *
   * ## Why the column store and not the sparse index
   *
   * The column store keeps RAW values; the sparse index keeps a normalized,
   * bucketed form built for range queries — `system.createdAt` is indexed at
   * minute precision there. A projection served from the sparse index would
   * hand back a value that differs from the record's, which is a wrong answer
   * nobody can see. So this door reads the column store, and a field the
   * column store does not hold is OMITTED rather than approximated.
   *
   * ## Why batched
   *
   * `getFieldValueForEntity` answers one (id, field) pair by walking the
   * field's storage; called per row it re-walks the same column for every id.
   * This walks each column ONCE and picks out every requested id as it passes:
   * O(fields x column) instead of O(ids x fields x column).
   *
   * Omission is always safe — it costs the caller a record read. The caller
   * diffs what it asked for against what came back and reads records for the
   * remainder, so an index that can serve nothing is slow, never wrong.
   *
   * @param ids - Canonical entity ids.
   * @param fields - Index keys (bare = user metadata, `system.*` = engine scalar).
   * @returns `id -> { field: value }` for exactly the pairs this index served.
   */
  async getScalarsForIds(
    ids: readonly string[],
    fields: readonly string[]
  ): Promise<Map<string, Record<string, unknown>>> {
    const out = new Map<string, Record<string, unknown>>()
    if (ids.length === 0 || fields.length === 0) return out
    // int -> id, so a column hit resolves back to the caller's id. An id the
    // mapper does not know cannot be in any column, so it is simply absent.
    const idByInt = new Map<number, string>()
    for (const id of ids) {
      const intId = this.idMapper.getInt(id)
      if (intId !== undefined) idByInt.set(intId, id)
    }
    if (idByInt.size === 0) return out
    for (const field of fields) {
      if (!this.columnStore.hasField(field)) continue
      const values = await this.columnStore.valuesForIds(field, idByInt.keys())
      for (const [intId, value] of values) {
        const id = idByInt.get(intId)
        if (id === undefined) continue
        let row = out.get(id)
        if (row === undefined) {
          row = {}
          out.set(id, row)
        }
        row[field] = value
      }
    }
    return out
  }
  async getFieldValueForEntity(entityId: string, field: string): Promise<any> {
    // `field` arrives as a FROZEN INDEX KEY (bare = user metadata;
    // 'system.<field>' = engine scalar). Storage fallbacks read the matching
--- a/src/utils/paramValidation.ts
+++ b/src/utils/paramValidation.ts
@ -18,7 +18,6 @@ import { findCallerLocation } from './callerLocation.js'
 import * as os from 'node:os'
 import * as fs from 'node:fs'
 import { parseFieldAddress, UnsupportedFindOptionError } from '../db/fieldAddressing.js'
 import { MAX_INDEXED_ARRAY_LENGTH, MetadataArrayTooLargeError } from '../errors/brainyError.js'
 const getSystemMemory = (): number => {
  if (os) {
@ -539,53 +538,8 @@ function rejectForgedSystemKeys(metadata: Record<string, unknown> | undefined, s
  }
 }
 /**
 * THE INDEXABLE-ARRAY BOUND, enforced at the write door.
 *
 * An array-valued metadata field indexes one posting per element, so the index
 * has always carried a ceiling. It used to be 10, and it was applied by a bare
 * `continue` deep inside field extraction: a row whose `tags` array held eleven
 * entries had that field skipped entirely and dropped out of every filtered
 * search on it — no error, no warning, and no way for the caller to tell the
 * difference from "no row matches". Silence is the defect; the ceiling is not.
 *
 * The bound is now {@link MAX_INDEXED_ARRAY_LENGTH}, high enough that every
 * legitimate multi-value field clears it, and it REFUSES here instead of
 * dropping data downstream. Refusing at the write door is what makes it
 * actionable: the caller learns at the moment of writing, with the field, the
 * length and the bound in hand.
 *
 * Scope is the caller's own metadata bag — the values that become postings.
 * Nested bags are walked, because a nested field indexes under its dotted
 * address exactly like a top-level one. Arrays of OBJECTS are not walked: the
 * index only ever makes postings from an array's scalar elements.
 *
 * @param metadata - The caller's metadata bag (undefined is fine).
 * @param site - The write door's name, for the message ('add()', 'update()', …).
 * @throws {MetadataArrayTooLargeError} Naming the field, its length and the bound.
 */
 function rejectOversizeIndexArrays(metadata: Record<string, unknown> | undefined, site: string): void {
  if (!metadata) return
  const walk = (bag: Record<string, unknown>, prefix: string): void => {
    for (const [key, value] of Object.entries(bag)) {
      const address = prefix ? `${prefix}.${key}` : key
      if (Array.isArray(value)) {
        if (value.length > MAX_INDEXED_ARRAY_LENGTH) {
          throw new MetadataArrayTooLargeError(site, address, value.length, MAX_INDEXED_ARRAY_LENGTH)
        }
      } else if (value && typeof value === 'object') {
        walk(value as Record<string, unknown>, address)
      }
    }
  }
  walk(metadata, '')
 }
 export function validateAddParams(params: AddParams): void {
  rejectForgedSystemKeys(params.metadata as Record<string, unknown> | undefined, 'add()')
  rejectOversizeIndexArrays(params.metadata as Record<string, unknown> | undefined, 'add()')
  // 'data' is ABSENT only when null/undefined — an empty string ('') is real
  // content (a legitimate empty file's first write) and must not be treated
  // as missing. Falsy-but-present values (0, false, '') all count as present;
@ -654,7 +608,6 @@ export function validateAddParams(params: AddParams): void {
 */
 export function validateUpdateParams(params: UpdateParams): void {
  rejectForgedSystemKeys(params.metadata as Record<string, unknown> | undefined, 'update()')
  rejectOversizeIndexArrays(params.metadata as Record<string, unknown> | undefined, 'update()')
  // Same absent-vs-empty distinction as validateAddParams: '' is a real new
  // value (e.g. truncating a file to empty content via overwrite), only
  // null/undefined means "no new data was given".
@ -729,7 +682,6 @@ export function validateUpdateParams(params: UpdateParams): void {
 */
 export function validateRelateParams(params: RelateParams): void {
  rejectForgedSystemKeys(params.metadata as Record<string, unknown> | undefined, 'relate()')
  rejectOversizeIndexArrays(params.metadata as Record<string, unknown> | undefined, 'relate()')
  // 8.0 verb-id contract (L.7): verb ids are UUIDs, generated by brainy.
  // RelateParams has no `id` field — an untyped caller passing one would
  // previously have it silently ignored (a generated UUID was used instead).
@ -779,7 +731,6 @@ export function validateRelateParams(params: RelateParams): void {
 */
 export function validateUpdateRelationParams(params: UpdateRelationParams): void {
  rejectForgedSystemKeys(params.metadata as Record<string, unknown> | undefined, 'updateRelation()')
  rejectOversizeIndexArrays(params.metadata as Record<string, unknown> | undefined, 'updateRelation()')
  if (!params.id) {
    throw new Error('id is required for updateRelation')
  }
--- a/tests/integration/find-fields-projection.test.ts
+++ b/tests/integration/find-fields-projection.test.ts
@ -1,261 +0,0 @@
 /**
 * @module tests/integration/find-fields-projection
 * @description **Field projection** — `find/get({ fields })` returns only the
 * named fields, and serves them from the index when it can.
 *
 * A list view that shows a title and a slug does not need the document body,
 * yet without a projection every row hydrates its whole record and discards
 * almost all of it. These pins hold the two halves of the fix:
 *
 * **The answer.** A projected row is a SUBSET of the full row — for every
 * requested field, the projected value equals the value the same query returns
 * unprojected. Absent `fields` is byte-identical to today. A requested field the
 * entity does not carry is simply absent, never an error. `system.*` resolves to
 * the engine scalar, a bare name to the user's metadata.
 *
 * **The cost.** When every requested field is index-served, the canonical
 * record is never opened — asserted by counting reads, not by timing them, so
 * it cannot flake into a false green. When one requested field is NOT
 * index-served (a body field, or a bucketed timestamp), exactly the owing rows
 * are read and the rest are still served from the index.
 */
 import { describe, it, expect, beforeAll, vi } from 'vitest'
 import { Brainy } from '../../src/brainy'
 import { NounType } from '../../src/types/graphTypes'
 import { generateTestVector } from '../helpers/test-factory'
 /** Rows carrying a title, a slug, and a large body nobody wants in a list. */
 const ROWS = 12
 const BODY = 'x'.repeat(4096)
 describe('find/get({ fields }) — projection', () => {
  let brain: Brainy<any>
  const ids: string[] = []
  beforeAll(async () => {
    brain = new Brainy({ requireSubtype: false, storage: { type: 'memory' } })
    await brain.init()
    for (let i = 0; i < ROWS; i++) {
      ids.push(
        await brain.add({
          id: `post-${i}`,
          data: `post ${i}`,
          type: NounType.Thing,
          metadata: {
            kind: 'post',
            title: `Title ${i}`,
            slug: `slug-${i}`,
            rank: i,
            body: BODY,
            // Only some rows carry this, so "missing is absent" is exercised
            // by real data rather than by a name nothing ever had.
            ...(i % 2 === 0 ? { featured: true } : {})
          },
          vector: generateTestVector()
        })
      )
    }
    // Persist so the column store holds the values a projection reads from.
    await brain.flush()
  })
  /** Count canonical record reads for one call. */
  const countingReads = async <R>(body: () => Promise<R>): Promise<{ out: R; reads: number }> => {
    const spy = vi.spyOn(brain as any, 'batchGet')
    try {
      const out = await body()
      const reads = spy.mock.calls.reduce(
        (n, call) => n + ((call[0] as string[] | undefined)?.length ?? 0),
        0
      )
      return { out, reads }
    } finally {
      spy.mockRestore()
    }
  }
  it('absent fields is byte-identical to today', async () => {
    const params = { where: { kind: 'post' }, limit: 5 }
    const a = await brain.find({ ...params })
    const b = await brain.find({ ...params, fields: undefined })
    expect(JSON.stringify(b)).toBe(JSON.stringify(a))
  })
  it('a projected row is a SUBSET of the full row, field for field', async () => {
    const shapes: Array<Record<string, unknown>> = [
      { where: { kind: 'post' }, limit: 6 },
      { where: { kind: 'post' }, limit: 6, offset: 3 },
      { where: { kind: 'post' }, orderBy: 'rank', order: 'asc', limit: 6 },
      { where: { kind: 'post' }, orderBy: 'rank', order: 'desc', limit: 4 }
    ]
    for (const shape of shapes) {
      const full = await brain.find(shape as never)
      const projected = await brain.find({ ...shape, fields: ['title', 'slug'] } as never)
      expect(projected.map((r) => r.id), JSON.stringify(shape)).toEqual(full.map((r) => r.id))
      for (let i = 0; i < full.length; i++) {
        const fullMeta = (full[i].entity.metadata ?? {}) as Record<string, unknown>
        const projMeta = (projected[i].entity.metadata ?? {}) as Record<string, unknown>
        expect(projMeta.title, `${JSON.stringify(shape)} row ${i}`).toEqual(fullMeta.title)
        expect(projMeta.slug).toEqual(fullMeta.slug)
      }
    }
  })
  it('returns ONLY the named fields — the body never rides along', async () => {
    const rows = await brain.find({ where: { kind: 'post' }, fields: ['title'], limit: 4 })
    expect(rows).toHaveLength(4)
    for (const r of rows) {
      const meta = (r.entity.metadata ?? {}) as Record<string, unknown>
      expect(Object.keys(meta)).toEqual(['title'])
      expect(meta.body).toBeUndefined()
      // Identity always survives a projection: a row you cannot identify is
      // not a row.
      expect(typeof r.id).toBe('string')
      expect(r.entity.id).toBe(r.id)
    }
  })
  it('a missing field is simply ABSENT — never an error', async () => {
    // `featured` exists on half the rows; `no-such-field` on none. Neither
    // throws, and neither appears as an explicit undefined.
    const rows = await brain.find({
      where: { kind: 'post' },
      fields: ['title', 'featured', 'no-such-field'],
      limit: ROWS
    })
    expect(rows.length).toBeGreaterThan(0)
    let withFeatured = 0
    for (const r of rows) {
      const meta = (r.entity.metadata ?? {}) as Record<string, unknown>
      expect('no-such-field' in meta).toBe(false)
      if ('featured' in meta) withFeatured += 1
    }
    // Real data, not a name nothing ever had: some rows carry it, some do not.
    expect(withFeatured).toBeGreaterThan(0)
    expect(withFeatured).toBeLessThan(rows.length)
  })
  it('a strict address resolver is NOT on this path', async () => {
    // orderBy throws UnresolvableFieldError for an unknown user key, because a
    // typo there silently changes the order. A projection must not inherit that
    // strictness: the honest answer to "give me this if you have it" is silence.
    await expect(
      brain.find({ where: { kind: 'post' }, fields: ['definitely-not-a-field'], limit: 2 })
    ).resolves.toBeInstanceOf(Array)
  })
  it('system.* resolves to the engine scalar, a bare name to user metadata', async () => {
    const full = await brain.find({ where: { kind: 'post' }, limit: 3 })
    const rows = await brain.find({
      where: { kind: 'post' },
      fields: ['system.createdAt', 'title'],
      limit: 3
    })
    for (let i = 0; i < rows.length; i++) {
      expect((rows[i].entity as any).createdAt).toEqual((full[i].entity as any).createdAt)
      const meta = (rows[i].entity.metadata ?? {}) as Record<string, unknown>
      expect(meta.title).toEqual((full[i].entity.metadata as any).title)
      // The engine scalar lands at the top level, not in the metadata bag —
      // the two address spaces never shadow each other.
      expect('system.createdAt' in meta).toBe(false)
      expect('createdAt' in meta).toBe(false)
    }
  })
  it('reads NO canonical record when every requested field is index-served', async () => {
    // The cost pin, counted rather than timed. `title` and `slug` are ordinary
    // indexed user fields, so the index can serve them exactly.
    const { out, reads } = await countingReads(() =>
      brain.find({ where: { kind: 'post' }, fields: ['title', 'slug'], limit: ROWS })
    )
    expect(out.length).toBeGreaterThan(0)
    expect(reads).toBe(0)
  })
  it('reads records only for the fields the column cannot serve', async () => {
    // `system.data` is NOT a column the store holds (verified against
    // getIndexedFields), so the record must be opened for it — while `title`,
    // which the column does hold, still comes from the index.
    const { out, reads } = await countingReads(() =>
      brain.find({ where: { kind: 'post' }, fields: ['title', 'system.data'], limit: 4 })
    )
    expect(out).toHaveLength(4)
    expect(reads).toBe(4)
    for (const r of out) {
      const meta = (r.entity.metadata ?? {}) as Record<string, unknown>
      expect(Object.keys(meta)).toEqual(['title'])
      expect(typeof (r.entity as any).data).toBe('string')
    }
  })
  it('a large field the column DOES hold costs no record read', async () => {
    // Worth pinning because it is the venue case: the body is column-served on
    // this engine, so a list that projects around it pays nothing for it, and
    // a list that projects it still pays no record read.
    const { reads } = await countingReads(() =>
      brain.find({ where: { kind: 'post' }, fields: ['body'], limit: 4 })
    )
    expect(reads).toBe(0)
  })
  it('projects a vector-leg find too — the ANSWER is uniform, only the cost is not', async () => {
    // The seam hydrates the metadata and graph page paths. A vector or text leg
    // builds its own entities, so those rows are trimmed after the integrity
    // guard instead. That difference is a COST difference, and this pin exists
    // so it can never quietly become an ANSWER difference.
    const rows = await brain.find({ query: 'post', fields: ['title'], limit: 3 })
    for (const r of rows) {
      const meta = (r.entity.metadata ?? {}) as Record<string, unknown>
      expect(Object.keys(meta)).toEqual(['title'])
      expect(meta.body).toBeUndefined()
      expect(r.entity.id).toBe(r.id)
    }
  })
  it('get({ fields }) projects a single row through the same seam', async () => {
    const full = await brain.get(ids[0])
    const projected = await brain.get(ids[0], { fields: ['title', 'slug'] })
    expect(projected).not.toBeNull()
    expect(projected!.id).toBe(full!.id)
    const fullMeta = (full!.metadata ?? {}) as Record<string, unknown>
    const projMeta = (projected!.metadata ?? {}) as Record<string, unknown>
    expect(projMeta.title).toEqual(fullMeta.title)
    expect(projMeta.slug).toEqual(fullMeta.slug)
    expect(Object.keys(projMeta).sort()).toEqual(['slug', 'title'])
    expect((projected as any).body).toBeUndefined()
  })
  it('get({ fields }) reads no record when the index serves the fields', async () => {
    const { reads } = await countingReads(() => brain.get(ids[1], { fields: ['title'] }))
    expect(reads).toBe(0)
  })
  it('the door serves EXACT values — the column, never the bucketed index', async () => {
    // The sparse index buckets `system.createdAt` to the minute for range
    // queries; the column store keeps raw ms. Serving a projection from the
    // former would hand back a value that differs from the record's, so the
    // door reads the column — and this pin is what proves which one it read.
    const index = (brain as any).metadataIndex
    const sample = ids.slice(0, 3)
    const served = await index.getScalarsForIds(sample, ['title', 'system.createdAt'])
    expect(served.size).toBe(sample.length)
    for (const id of sample) {
      const row = served.get(id)!
      const record = await brain.get(id)
      expect(row.title).toEqual((record!.metadata as any).title)
      // Exact to the millisecond — a bucketed value would be rounded down to
      // the minute and this would fail.
      expect(row['system.createdAt']).toEqual((record as any).createdAt)
    }
  })
  it('a field the column store does not hold is OMITTED, not approximated', async () => {
    const index = (brain as any).metadataIndex
    const served = await index.getScalarsForIds(ids.slice(0, 2), ['title', 'system.data'])
    for (const [, row] of served) {
      expect('title' in row).toBe(true)
      // Omission is what makes the caller read the record for it.
      expect('system.data' in row).toBe(false)
    }
  })
 })
--- a/tests/integration/find-orderby-every-path.test.ts
+++ b/tests/integration/find-orderby-every-path.test.ts
@ -1,244 +0,0 @@
 /**
 * @module tests/integration/find-orderby-every-path
 * @description `orderBy` IS THE ORDER — on every find() path, not just the
 * metadata-only one.
 *
 * THE DEFECT. `find({ where, orderBy })` (metadata only) answered in field
 * order. `find({ query, where, orderBy })` and `find({ vector, where, orderBy })`
 * answered in SCORE order, silently: the vector/filter block ranked the fused
 * candidates by score, cut the page, and returned early — the tail's `orderBy`
 * sort sat below that early return and never ran. Nothing threw, nothing warned,
 * and the two paths disagreed about what "ordered by rank" means. A caller
 * paging `orderBy: 'rank', order: 'desc'` over a hybrid find got relevance
 * order wearing an ordering request's clothes.
 *
 * Where `connected` or `fusion` kept the tail alive the defect changed shape
 * rather than disappearing: the block had already CUT the page by score, so the
 * tail ordered the rows relevance had chosen instead of the rows the ordering
 * asks for — a correctly sorted page of the wrong rows.
 *
 * The early cut fires only once the candidate set reaches `offset + limit`
 * rows, which is why small fixtures never saw it: below that threshold the
 * block falls through and the tail's sort does apply. That is the whole shape
 * of the bug — an ordering that is correct until there is enough data to matter.
 *
 * THE LAW. An explicit `orderBy` displaces score as the ordering key on every
 * path. The candidate set the path produced is ordered IN FULL and the page is
 * cut from that ordering — the graph-first law's "page last", applied to
 * ordering rather than to filtering. Score-ranked early paging is for the
 * default (no `orderBy`) case only, where score IS the requested order.
 *
 * THE PIN. Differential, against the metadata-only path — the one path that
 * always honoured `orderBy`.
 *
 * WHAT THE DIFFERENTIAL CAN AND CANNOT CLAIM. `orderBy` orders the candidate
 * set; it does not enlarge it. The hybrid legs are bounded by construction (the
 * text leg and the beam walk each take `limit * 2`), so a differential against
 * the metadata-only path — whose universe is every matching row — is only
 * meaningful where those bounds provably cover the universe. The fixture is
 * sized so they do (12 rows, `limit` 6 → a `limit * 2` = 12-row text leg), and
 * the covering is ASSERTED from the leg's own output rather than assumed. This
 * pin is about ordering, and it says nothing about recall.
 */
 import { describe, it, expect, beforeAll } from 'vitest'
 import { Brainy } from '../../src/brainy'
 import { NounType, VerbType } from '../../src/types/graphTypes'
 import { resolveEntityId } from '../../src/utils/idNormalization'
 /** Embedding width of the default model — the row vectors must match it. */
 const DIM = 384
 /** A deterministic, per-row-distinct unit vector (no embedder in the fixture). */
 function seededVector(seed: number): number[] {
  const v = new Array<number>(DIM)
  for (let i = 0; i < DIM; i++) {
    v[i] = Math.sin((i + 1) * 0.11 + seed * 0.37) * 0.5 + Math.cos((i + 1) * 0.05 + seed * 0.13) * 0.3
  }
  const magnitude = Math.sqrt(v.reduce((sum, x) => sum + x * x, 0))
  return v.map((x) => x / magnitude)
 }
 /**
 * Ranks, shuffled — so no scoring order can reproduce them by luck, and the
 * ordering the pins assert is visibly not the insertion order either.
 */
 const RANKS = [7, 3, 11, 1, 9, 5, 12, 2, 10, 4, 8, 6]
 const ROWS = RANKS.length
 /** The page size every pin uses: `limit * 2` covers the whole universe. */
 const LIMIT = 6
 /** The neighbour subset — the graph-first universe — and its own page size. */
 const NEIGHBOURS = 8
 const GRAPH_LIMIT = 4
 describe('find(): orderBy is the order on every path', () => {
  let brain: Brainy<any>
  const QUERY = 'orbital telemetry'
  const anchor = 'ordering-anchor'
  const neighbourIds: string[] = []
  beforeAll(async () => {
    brain = new Brainy({ requireSubtype: false, storage: { type: 'memory' } })
    await brain.init()
    let seed = 1
    await brain.add({
      id: anchor,
      data: 'ground station anchor record',
      type: NounType.Thing,
      metadata: { lane: 'anchor', rank: 0 },
      vector: seededVector(seed++)
    })
    for (let i = 0; i < ROWS; i++) {
      const id = `row-${i}`
      await brain.add({
        id,
        // EVERY row carries both query words, so the text leg reaches all of
        // them and the hybrid candidate set covers the whole universe.
        data: `orbital telemetry packet ${i} recorded downlink`,
        type: NounType.Document,
        metadata: { lane: 'alpha', rank: RANKS[i] },
        vector: seededVector(seed++)
      })
      if (i < NEIGHBOURS) {
        await brain.relate({ from: anchor, to: id, type: VerbType.RelatedTo })
        neighbourIds.push(resolveEntityId(id))
      }
    }
  })
  it('the fixture: the hybrid candidate set covers the whole filter universe', async () => {
    const universe: string[] = await (brain as any).filterIdsBelted({ lane: 'alpha' })
    expect(universe).toHaveLength(ROWS)
    // The text leg is bounded at `limit * 2`; the fixture is sized so that
    // bound reaches every row in the universe. This is the precondition the
    // differential below rests on — asserted from the leg itself.
    const textScored = await (brain as any).executeTextSearchScored(QUERY, LIMIT * 2, universe)
    expect(textScored).toHaveLength(ROWS)
    // And the candidate set is large enough to trigger the score-ranked early
    // cut this pin exists to keep out of an ordered query's way.
    expect(ROWS).toBeGreaterThanOrEqual(LIMIT)
  })
  it('metadata-only + orderBy: the reference ordering', async () => {
    const rows = await brain.find({
      where: { lane: 'alpha' },
      orderBy: 'rank',
      order: 'desc',
      limit: LIMIT
    } as any)
    expect(rows.map((r: any) => r.metadata.rank)).toEqual([12, 11, 10, 9, 8, 7])
  })
  it('hybrid (query + where) + orderBy: the same page as the metadata-only path', async () => {
    const params = { where: { lane: 'alpha' }, orderBy: 'rank', order: 'desc' as const, limit: LIMIT }
    const expected = await brain.find(params as any)
    const actual = await brain.find({ ...params, query: QUERY } as any)
    expect(actual).toHaveLength(expected.length)
    expect(actual.map((r: any) => r.id)).toEqual(expected.map((r: any) => r.id))
    expect(actual.map((r: any) => r.metadata.rank)).toEqual([12, 11, 10, 9, 8, 7])
  })
  it('hybrid + orderBy asc: the ordering key is honoured in both directions', async () => {
    const params = { where: { lane: 'alpha' }, orderBy: 'rank', order: 'asc' as const, limit: LIMIT }
    const expected = await brain.find(params as any)
    const actual = await brain.find({ ...params, query: QUERY } as any)
    expect(actual.map((r: any) => r.id)).toEqual(expected.map((r: any) => r.id))
    expect(actual.map((r: any) => r.metadata.rank)).toEqual([1, 2, 3, 4, 5, 6])
  })
  it('hybrid + orderBy + offset: page two is page two of the ORDERING', async () => {
    const params = {
      where: { lane: 'alpha' },
      orderBy: 'rank',
      order: 'desc' as const,
      limit: LIMIT,
      offset: LIMIT
    }
    const expected = await brain.find(params as any)
    const actual = await brain.find({ ...params, query: QUERY } as any)
    expect(actual).toHaveLength(LIMIT)
    expect(actual.map((r: any) => r.id)).toEqual(expected.map((r: any) => r.id))
    expect(actual.map((r: any) => r.metadata.rank)).toEqual([6, 5, 4, 3, 2, 1])
  })
  it('hybrid + orderBy: paging walks the ordering monotonically, no row twice', async () => {
    const seen: number[] = []
    for (let offset = 0; offset < ROWS; offset += LIMIT) {
      const page = await brain.find({
        query: QUERY,
        where: { lane: 'alpha' },
        orderBy: 'rank',
        order: 'desc',
        limit: LIMIT,
        offset
      } as any)
      seen.push(...page.map((r: any) => r.metadata.rank))
    }
    expect(seen).toHaveLength(ROWS)
    expect(new Set(seen).size).toBe(ROWS)
    // Strictly descending across every page boundary.
    for (let i = 1; i < seen.length; i++) expect(seen[i]).toBeLessThan(seen[i - 1])
  })
  it('vector + where + orderBy: field order, not distance order', async () => {
    // The beam walk takes `limit * 2` = the whole universe here, so the page is
    // the true top of the ordering — which distance order cannot produce.
    const rows = await brain.find({
      vector: seededVector(1000),
      where: { lane: 'alpha' },
      orderBy: 'rank',
      order: 'desc',
      limit: LIMIT
    } as any)
    expect(rows.map((r: any) => r.metadata.rank)).toEqual([12, 11, 10, 9, 8, 7])
  })
  it('graph-first (query + connected + where) + orderBy: the neighbour set, ordered', async () => {
    const actual = await brain.find({
      query: QUERY,
      connected: { from: anchor, direction: 'out' as const },
      where: { lane: 'alpha' },
      orderBy: 'rank',
      order: 'desc',
      limit: GRAPH_LIMIT
    } as any)
    expect(actual).toHaveLength(GRAPH_LIMIT)
    const neighbours = new Set(neighbourIds)
    for (const r of actual) expect(neighbours.has(r.id)).toBe(true)
    // The ordering covers the whole neighbour set, so the page holds the
    // highest ranks AMONG THE NEIGHBOURS — not the ones the score ranking
    // happened to surface first and the tail then sorted among themselves.
    const expectedRanks = RANKS.slice(0, NEIGHBOURS)
      .sort((a, b) => b - a)
      .slice(0, GRAPH_LIMIT)
    expect(expectedRanks).toEqual([12, 11, 9, 7])
    expect(actual.map((r: any) => r.metadata.rank)).toEqual(expectedRanks)
  })
  it('fusion + orderBy: the ordering survives the fusion rescore', async () => {
    const actual = await brain.find({
      query: QUERY,
      where: { lane: 'alpha' },
      fusion: 'weighted',
      orderBy: 'rank',
      order: 'desc',
      limit: LIMIT
    } as any)
    expect(actual.map((r: any) => r.metadata.rank)).toEqual([12, 11, 10, 9, 8, 7])
  })
  it('no orderBy: score order still stands (the default is untouched)', async () => {
    const rows = await brain.find({ query: QUERY, where: { lane: 'alpha' }, limit: LIMIT } as any)
    expect(rows).toHaveLength(LIMIT)
    const scores = rows.map((r: any) => r.score)
    for (let i = 1; i < scores.length; i++) expect(scores[i]).toBeLessThanOrEqual(scores[i - 1])
  })
 })
--- a/tests/integration/graphIndex-pagination.test.ts
+++ b/tests/integration/graphIndex-pagination.test.ts
@ -9,34 +9,9 @@
 * 8.0 BigInt boundary: entity ints in (resolved via the metadata index's
 * idMapper), entity/verb ints out (`bigint[]`). Entity ints map back to UUIDs
 * via `idMapper.getUuid(Number(int))`; verb ints via `verbIntsToIds()`.
 *
 * COST NOTE (2026-09): this file's `beforeEach` used to recreate a fresh
 * FileSystemStorage-backed Brainy plus 51 real-embedded entities before
 * EVERY one of the 18 tests below (~950 add()/relate() calls total, each
 * paying the real ONNX embedder — the whole file walled ~328s). Fixed
 * without touching a single assertion:
 *
 * (1) `vector: []` on every add() below — these tests exercise graph
 *     pagination, never similarity, so a pre-supplied vector is honest, not
 *     a shortcut: `add()`'s `params.vector || (await this.embed(...))` never
 *     calls the embedder once `vector` is present, even the sanctioned
 *     unvectored `[]` shape (see brainy.ts's add(), the zero-norm-law
 *     comment) — and the `vector.length > 0` gate on dimension-pinning means
 *     `[]` never poisons `this.dimensions` for later real embeds.
 * (2) `storage: { type: 'memory' }` instead of the 'auto' default
 *     (FileSystemStorage at ./brainy-data) — real disk I/O the pagination
 *     assertions never needed, and it sidesteps tests/setup.ts's global
 *     per-test `rm -rf brainy-data`, which would otherwise corrupt a brain
 *     shared across a describe's beforeAll out from under it.
 * (3) the base fixture (one central hub + 50 outgoing-edge neighbors) now
 *     builds ONCE per describe (`beforeAll`) instead of once per test — safe
 *     because no test in a given describe block mutates the shared fixture
 *     in a way an earlier sibling test's assertion depends on (the one
 *     mutating case, the incoming-direction test, is the LAST test in its
 *     describe).
 */
-import { describe, it, expect, beforeAll, afterAll } from 'vitest'
+import { describe, it, expect, beforeEach } from 'vitest'
 import { Brainy } from '../../src/brainy.js'
 import { NounType, VerbType } from '../../src/types/graphTypes.js'
@ -64,21 +39,14 @@ describe('GraphAdjacencyIndex Pagination', () => {
      .map((i) => idMapper().getUuid(Number(i)))
      .filter((u: string | undefined): u is string => u !== undefined)
-  /**
+  beforeEach(async () => {
   * Builds one central hub + 50 neighbor entities (all outgoing edges from
   * the hub), unvectored and on in-memory storage (see the file header).
   * Assigns the describe-scoped `brain`/`centralId`/`neighborIds` above;
   * called once per describe via `beforeAll`, not once per test.
   */
  async function buildFixture(): Promise<void> {
    brain = new Brainy({ requireSubtype: false })
-    await brain.init({ storage: { type: 'memory' } })
+    await brain.init()
    // Create central entity
    centralId = await brain.add({
      data: { name: 'Central Hub' },
-      type: NounType.Thing,
+      type: NounType.Thing
      vector: []
    })
    // Create 50 neighbor entities with relationships
@ -86,8 +54,7 @@ describe('GraphAdjacencyIndex Pagination', () => {
    for (let i = 0; i < 50; i++) {
      const neighborId = await brain.add({
        data: { name: `Neighbor ${i}`, index: i },
-        type: NounType.Thing,
+        type: NounType.Thing
        vector: []
      })
      neighborIds.push(neighborId)
@ -98,14 +65,9 @@ describe('GraphAdjacencyIndex Pagination', () => {
        type: VerbType.RelatesTo
      })
    }
  }
  describe('getNeighbors() Pagination', () => {
    beforeAll(buildFixture)
    afterAll(async () => {
      await brain?.close()
  })
  describe('getNeighbors() Pagination', () => {
    it('should return all neighbors without pagination', async () => {
      const neighborInts = await graphIndex().getNeighbors(entityInt(centralId))
      const neighbors = intsToUuids(neighborInts)
@ -187,8 +149,7 @@ describe('GraphAdjacencyIndex Pagination', () => {
      // Create some incoming relationships
      const sourceId = await brain.add({
        data: { name: 'Source' },
-        type: NounType.Thing,
+        type: NounType.Thing
        vector: []
      })
      await brain.relate({
@ -208,11 +169,6 @@ describe('GraphAdjacencyIndex Pagination', () => {
  })
  describe('getVerbIdsBySource() Pagination', () => {
    beforeAll(buildFixture)
    afterAll(async () => {
      await brain?.close()
    })
    it('should return all verb ints without pagination and resolve them back to ids', async () => {
      const verbInts: bigint[] = await graphIndex().getVerbIdsBySource(entityInt(centralId))
@ -267,11 +223,6 @@ describe('GraphAdjacencyIndex Pagination', () => {
  })
  describe('getVerbIdsByTarget() Pagination', () => {
    beforeAll(buildFixture)
    afterAll(async () => {
      await brain?.close()
    })
    it('should return all verb ints targeting an entity', async () => {
      // Pick a neighbor that's a target of relationships
      const targetId = neighborIds[0]
@ -285,16 +236,14 @@ describe('GraphAdjacencyIndex Pagination', () => {
      // Create entity with many incoming relationships
      const popularTarget = await brain.add({
        data: { name: 'Popular Target' },
-        type: NounType.Thing,
+        type: NounType.Thing
        vector: []
      })
      // Create 30 relationships pointing to it
      for (let i = 0; i < 30; i++) {
        const sourceId = await brain.add({
          data: { name: `Source ${i}` },
-          type: NounType.Thing,
+          type: NounType.Thing
          vector: []
        })
        await brain.relate({
          from: sourceId,
@ -318,11 +267,6 @@ describe('GraphAdjacencyIndex Pagination', () => {
  })
  describe('Performance with Pagination', () => {
    beforeAll(buildFixture)
    afterAll(async () => {
      await brain?.close()
    })
    it('should maintain sub-5ms performance with pagination', async () => {
      const central = entityInt(centralId)
@ -341,17 +285,11 @@ describe('GraphAdjacencyIndex Pagination', () => {
  })
  describe('Real-World Use Cases', () => {
    beforeAll(buildFixture)
    afterAll(async () => {
      await brain?.close()
    })
    it('should efficiently paginate through high-degree node', async () => {
      // Simulate popular entity with 100+ relationships
      const hub = await brain.add({
        data: { name: 'Popular Hub' },
-        type: NounType.Thing,
+        type: NounType.Thing
        vector: []
      })
      // Create 100 relationships
@ -359,8 +297,7 @@ describe('GraphAdjacencyIndex Pagination', () => {
      for (let i = 0; i < 100; i++) {
        const targetId = await brain.add({
          data: { name: `Target ${i}` },
-          type: NounType.Thing,
+          type: NounType.Thing
          vector: []
        })
        targetIds.push(targetId)
        await brain.relate({
--- a/tests/integration/metadata-vector-exclusion.test.ts
+++ b/tests/integration/metadata-vector-exclusion.test.ts
@ -26,7 +26,6 @@ import { describe, it, expect, beforeEach, afterEach } from 'vitest'
 import { Brainy } from '../../src/brainy.js'
 import { NounType } from '../../src/types/graphTypes.js'
 import { existsSync, rmSync } from 'fs'
 import { MetadataArrayTooLargeError, MAX_INDEXED_ARRAY_LENGTH } from '../../src/errors/brainyError.js'
 describe('Metadata Vector Exclusion Fix', () => {
  let brainy: Brainy
@ -156,15 +155,11 @@ describe('Metadata Vector Exclusion Fix', () => {
    expect(results[0].entity.metadata?.name).toBe('Bob')
  })
-  it('should REFUSE an array over the indexing bound, by name', async () => {
+  it('should skip indexing large arrays (>10 elements)', async () => {
-    // A large array (not a vector, just bulk data). This used to be SKIPPED in
+    // Add entity with a large array (not a vector, just bulk data).
    // silence at a bound of 10 — the field simply vanished from the index and
    // the row dropped out of every `where` on it, indistinguishably from "no
    // row matches". The bound is now MAX_INDEXED_ARRAY_LENGTH and it REFUSES.
    const largeArray = Array.from({ length: 100 }, (_, i) => `item${i}`)
-    const err = await brainy
+    await brainy.add({
      .add({
      type: NounType.Document,
      data: 'Doc with large array',
      metadata: {
@ -172,40 +167,17 @@ describe('Metadata Vector Exclusion Fix', () => {
        items: largeArray
      }
    })
      .catch((e: any) => e)
-    expect(err).toBeInstanceOf(MetadataArrayTooLargeError)
+    // Large arrays (> 10 elements) are deliberately skipped to avoid indexing
-    expect(err.field).toBe('items')
+    // bulk/vector-like payloads: 'items' must NOT appear, and the 100 elements
-    expect(err.length).toBe(100)
+    // must NOT have produced 100 indexed fields.
    expect(err.limit).toBe(MAX_INDEXED_ARRAY_LENGTH)
    // Nothing was indexed from the refused write — no 'items' field, and above
    // all no per-element numeric fields (the original explosion class).
    const fields = await brainy.getAvailableFields()
    expect(fields).not.toContain('items')
    const numericFields = fields.filter(f => /(^|\.)\d+$/.test(f))
    expect(numericFields).toEqual([])
  })
-  it('should index an array UP TO the bound — the old limit of 10 was the bug', async () => {
+    // The scalar 'name' field IS indexed.
-    await brainy.add({
+    expect(fields).toContain('name')
      type: NounType.Document,
      data: 'Doc with a long-but-legitimate tag list',
      metadata: {
        name: 'Doc with many tags',
        items: Array.from({ length: MAX_INDEXED_ARRAY_LENGTH }, (_, i) => `item${i}`)
      }
    })
    const fields = await brainy.getAvailableFields()
    // The field IS indexed now, and still without per-element numeric fields.
    expect(fields).toContain('items')
    expect(fields.filter(f => /(^|\.)\d+$/.test(f))).toEqual([])
    // And the eleventh element — the one the old bound silently dropped the
    // whole field for — really is searchable.
    const hits = await brainy.find({ where: { items: 'item10' } })
    expect(hits.length).toBeGreaterThan(0)
  })
  it('should preserve HNSW vector search functionality', async () => {
--- a/tests/integration/readonly-close-no-marker.test.ts
+++ b/tests/integration/readonly-close-no-marker.test.ts
@ -149,12 +149,12 @@ describe('a read-only brain writes no clean-shutdown evidence', () => {
    brain = null
    // The FILE SET under `_system/` is unchanged — a reader creates and
-    // removes nothing. This pin is specifically about the generation store's
+    // removes nothing. (Other files under `_system/` — e.g. the metadata
-    // clean-shutdown evidence. The wider law — that a reader leaves EVERY
+    // field registry, which stamps its own `lastUpdated` on every persist —
-    // file under `_system/` byte-identical, which this fix left open as a
+    // are a pre-existing, separate concern outside this fix's scope: this
-    // known residual (the metadata field registry and the three statistics
+    // pin is specifically about the generation store's clean-shutdown
-    // files were still re-stamped by a reader's close) — is closed and pinned
+    // evidence, not about every subsystem's close() being a true no-op for
-    // in `readonly-close-writes-nothing.test.ts`.
+    // a reader.)
    const after = snapshotDir(systemDir())
    expect([...after.keys()].sort()).toEqual([...before.keys()].sort())
--- a/tests/integration/readonly-close-writes-nothing.test.ts
+++ b/tests/integration/readonly-close-writes-nothing.test.ts
@ -1,261 +0,0 @@
 /**
 * @module tests/integration/readonly-close-writes-nothing
 * @description A READ-ONLY BRAIN LEAVES `_system/` BYTE-IDENTICAL — the WHOLE
 * directory, not just the clean-shutdown marker.
 *
 * `readonly-close-no-marker` closed the marker half of this law and named the
 * rest as a known, out-of-scope residual:
 *
 *   "Other files under `_system/` — e.g. the metadata field registry, which
 *    stamps its own `lastUpdated` on every persist — are a pre-existing,
 *    separate concern outside this fix's scope."
 *
 * This is that residual, closed. MEASURED on the base before the fix, a
 * read-only open → read → close rewrote FOUR files:
 *
 *   _system/__metadata_field_registry__.json.gz
 *   _system/type-statistics.json.gz
 *   _system/subtype-statistics.json.gz
 *   _system/verb-subtype-statistics.json.gz
 *
 * THE CAUSE was not the closes the marker fix guarded — it was Phase 1 of
 * `closeDurableSteps`, where every component flush ran unconditionally. A flush
 * is a write by definition: `MetadataIndexManager#flush()` saves the field
 * registry "even with no dirty fields" (its own comment), and the storage
 * adapter's count flush re-stamps the three statistics files. A session that
 * committed nothing re-stamped all four. Phase 2's closes were ungated too —
 * the graph index's close drains both LSM MemTables and stamps a watermark,
 * and the optional vector/metadata `close` hooks (unimplemented in the
 * reference engine, filled in by a native provider) persist buffered state.
 *
 * THE LAW. A reader writes nothing, anywhere under `_system/`, at open or at
 * close. It still RELEASES what it holds: the graph index's auto-flush interval
 * is cleared through `stopBackgroundFlush()`, the non-writing half of its
 * close, so nothing outlives the session.
 *
 * WHY IT MATTERS beyond tidiness: `_system/` is where a store keeps its
 * evidence about itself — what the writer committed, what the projections have
 * seen. A reader that rewrites any of it is vouching for a state it only
 * observed, and on shared or snapshot storage it mutates bytes another process
 * owns.
 */
 import { describe, it, expect, beforeEach, afterEach } from 'vitest'
 import { mkdtempSync, rmSync, readdirSync, readFileSync, statSync } from 'node:fs'
 import { createHash } from 'node:crypto'
 import { tmpdir } from 'node:os'
 import { join } from 'node:path'
 import { Brainy } from '../../src/brainy.js'
 import { NounType, VerbType } from '../../src/types/graphTypes.js'
 /** Recursively hash every regular file under `dir`, keyed by its path relative to `dir`. */
 function snapshotDir(dir: string): Map<string, string> {
  const out = new Map<string, string>()
  const walk = (rel: string): void => {
    const abs = rel ? join(dir, rel) : dir
    let entries: string[]
    try {
      entries = readdirSync(abs)
    } catch {
      return
    }
    for (const name of entries) {
      const childRel = rel ? join(rel, name) : name
      const childAbs = join(dir, childRel)
      const st = statSync(childAbs)
      if (st.isDirectory()) {
        walk(childRel)
      } else if (st.isFile()) {
        out.set(childRel, createHash('sha256').update(readFileSync(childAbs)).digest('hex'))
      }
    }
  }
  walk('')
  return out
 }
 /** Every path where `after` differs from `before`, labelled — the failure message. */
 function diff(before: Map<string, string>, after: Map<string, string>): string[] {
  const lines: string[] = []
  for (const [path, hash] of after) {
    if (!before.has(path)) lines.push(`ADDED   ${path}`)
    else if (before.get(path) !== hash) lines.push(`CHANGED ${path}`)
  }
  for (const path of before.keys()) if (!after.has(path)) lines.push(`REMOVED ${path}`)
  return lines.sort()
 }
 describe('a read-only brain writes nothing under `_system/`', () => {
  let dir: string
  let brain: Brainy | null = null
  beforeEach(() => {
    dir = mkdtempSync(join(tmpdir(), 'brainy-readonly-writes-'))
  })
  afterEach(async () => {
    if (brain) {
      try {
        await brain.close()
      } catch {
        /* already closed */
      }
      brain = null
    }
    try {
      rmSync(dir, { recursive: true, force: true })
    } catch {
      /* ignore */
    }
  })
  const systemDir = () => join(dir, '_system')
  /**
   * A writer seeds a store with nouns, verbs and queryable metadata — enough
   * that the field registry, the statistics files and the graph index all hold
   * real content — then closes cleanly.
   */
  async function seedStore(): Promise<void> {
    const writer = new Brainy({
      requireSubtype: false,
      storage: { type: 'filesystem', path: dir }
    })
    await writer.init()
    for (let i = 0; i < 6; i++) {
      await writer.add({
        id: `seed-${i}`,
        data: `seed entity ${i}`,
        type: i % 2 === 0 ? NounType.Concept : NounType.Document,
        metadata: { lane: i % 2 === 0 ? 'alpha' : 'beta', rank: i, tags: [`t${i}`, 'shared'] },
        vector: []
      })
    }
    for (let i = 1; i < 6; i++) {
      await writer.relate({ from: 'seed-0', to: `seed-${i}`, type: VerbType.RelatedTo })
    }
    await writer.flush()
    await writer.close()
  }
  it('open → read → close leaves every file under `_system/` byte-identical', async () => {
    await seedStore()
    const before = snapshotDir(systemDir())
    expect(before.size, 'the writer left a populated `_system/`').toBeGreaterThan(0)
    brain = await Brainy.openReadOnly({ storage: { type: 'filesystem', path: dir } })
    expect(brain.isReadOnly).toBe(true)
    // Exercise the read surface that drives each subsystem: statistics (counts),
    // a metadata filter (field index + registry), a graph walk (adjacency), a
    // vector search, and a direct get.
    await brain.stats()
    await brain.find({ where: { lane: 'alpha' }, limit: 10 } as any)
    await brain.find({ where: { tags: 'shared' }, limit: 10 } as any)
    await brain.find({ connected: { from: 'seed-0', direction: 'out' }, limit: 10 } as any)
    await brain.get('seed-1')
    await brain.close()
    brain = null
    const after = snapshotDir(systemDir())
    const changes = diff(before, after)
    expect(changes, `a reader modified \`_system/\`:\n${changes.join('\n')}`).toEqual([])
  }, 120_000)
  it('names the four files that used to change — the measured shape of the defect', async () => {
    await seedStore()
    const before = snapshotDir(systemDir())
    // These are the exact paths the base rewrote. Naming them keeps the pin
    // honest about what it caught: if a future change reintroduces the write,
    // the test above fails and this one says which subsystem did it.
    const previouslyRewritten = [
      '__metadata_field_registry__.json.gz',
      'type-statistics.json.gz',
      'subtype-statistics.json.gz',
      'verb-subtype-statistics.json.gz'
    ]
    for (const name of previouslyRewritten) {
      expect(before.has(name), `fixture must contain ${name}`).toBe(true)
    }
    brain = await Brainy.openReadOnly({ storage: { type: 'filesystem', path: dir } })
    await brain.stats()
    await brain.find({ where: { lane: 'alpha' }, limit: 10 } as any)
    await brain.close()
    brain = null
    const after = snapshotDir(systemDir())
    for (const name of previouslyRewritten) {
      expect(after.get(name), `${name} was rewritten by a reader`).toBe(before.get(name))
    }
  }, 120_000)
  it('a reader that only opens and closes — touching nothing — writes nothing', async () => {
    await seedStore()
    const before = snapshotDir(systemDir())
    brain = await Brainy.openReadOnly({ storage: { type: 'filesystem', path: dir } })
    await brain.close()
    brain = null
    const changes = diff(before, snapshotDir(systemDir()))
    expect(changes, `an idle reader modified \`_system/\`:\n${changes.join('\n')}`).toEqual([])
  }, 120_000)
  it('two readers in sequence each leave the store exactly as they found it', async () => {
    await seedStore()
    const before = snapshotDir(systemDir())
    for (let i = 0; i < 2; i++) {
      const reader = await Brainy.openReadOnly({ storage: { type: 'filesystem', path: dir } })
      await reader.find({ where: { lane: 'beta' }, limit: 10 } as any)
      await reader.close()
      const changes = diff(before, snapshotDir(systemDir()))
      expect(changes, `reader ${i + 1} modified \`_system/\`:\n${changes.join('\n')}`).toEqual([])
    }
  }, 120_000)
  it('the store outside `_system/` is untouched too — a reader writes nowhere', async () => {
    await seedStore()
    const before = snapshotDir(dir)
    brain = await Brainy.openReadOnly({ storage: { type: 'filesystem', path: dir } })
    await brain.stats()
    await brain.find({ where: { lane: 'alpha' }, limit: 10 } as any)
    await brain.close()
    brain = null
    const changes = diff(before, snapshotDir(dir))
    expect(changes, `a reader modified the store:\n${changes.join('\n')}`).toEqual([])
  }, 120_000)
  it('a WRITER still persists on close — the guard did not disarm the write path', async () => {
    await seedStore()
    const before = snapshotDir(systemDir())
    const writer = new Brainy({ requireSubtype: false, storage: { type: 'filesystem', path: dir } })
    await writer.init()
    await writer.add({
      id: 'after-reader',
      data: 'a new row',
      type: NounType.Concept,
      metadata: { lane: 'gamma', rank: 99 },
      vector: []
    })
    await writer.close()
    // The writer's close DID move `_system/` — that is the whole point of the
    // asymmetry, and the guard must not have flattened it.
    expect(diff(before, snapshotDir(systemDir())).length).toBeGreaterThan(0)
    // And the row is really there on the next open.
    const reopened = new Brainy({ requireSubtype: false, storage: { type: 'filesystem', path: dir } })
    await reopened.init()
    brain = reopened
    const hits = await reopened.find({ where: { lane: 'gamma' }, limit: 10 } as any)
    expect(hits.length).toBe(1)
  }, 120_000)
 })
--- a/tests/unit/utils/metadataIndex-array-bound.test.ts
+++ b/tests/unit/utils/metadataIndex-array-bound.test.ts
@ -1,242 +0,0 @@
 /**
 * @module tests/unit/utils/metadataIndex-array-bound
 * @description THE INDEXABLE-ARRAY BOUND — a law with a name and a refusal,
 * not a `continue`.
 *
 * THE DEFECT. An array-valued metadata field indexes one posting per element,
 * so the index has always carried a ceiling. It was 10, and it was applied by a
 * bare `continue` deep inside field extraction:
 *
 *     if (Array.isArray(value) && value.length > 10) continue
 *
 * A row whose `tags` array held ELEVEN entries therefore had that field skipped
 * entirely — no posting, no error, no warning. The row then failed to match
 * every filtered search on `tags`, including `{ tags: 'a-tag-it-really-has' }`,
 * and the caller had no way to tell that from "no row matches". Eleven tags is
 * not an exotic shape; the eleventh tag made the row invisible.
 *
 * THE LAW. Arrays of scalars index up to {@link MAX_INDEXED_ARRAY_LENGTH} = 64,
 * hardcoded (the zero-config law: no knob), which clears every legitimate
 * multi-value field and stays far below any embedding width. Above it the WRITE
 * IS REFUSED by name — `MetadataArrayTooLargeError`, carrying the field, the
 * length and the bound — at `add`, `update`, `relate` and `updateRelation`
 * alike. Nothing is skipped in silence.
 *
 * THE ONE PLACE THE BOUND STILL SKIPS is a row already on disk, written by an
 * older engine under the old rule and read back by a rebuild, a catch-up fold
 * or a remove. Refusing there would make an existing store un-rebuildable — so
 * the row is admitted and the skipped field is NARRATED. Both sides are pinned.
 */
 import { describe, it, expect, beforeEach, vi, afterEach } from 'vitest'
 import { Brainy } from '../../../src/brainy'
 import { NounType, VerbType } from '../../../src/types/graphTypes'
 import { MetadataArrayTooLargeError, MAX_INDEXED_ARRAY_LENGTH } from '../../../src/errors/brainyError'
 import { resolveEntityId } from '../../../src/utils/idNormalization'
 import { prodLog } from '../../../src/utils/logger'
 /** `n` distinct scalar tags. */
 function tags(n: number, prefix = 't'): string[] {
  return Array.from({ length: n }, (_, i) => `${prefix}${i}`)
 }
 describe('the indexable-array bound', () => {
  let brain: Brainy<any>
  beforeEach(async () => {
    brain = new Brainy({ requireSubtype: false, storage: { type: 'memory' } })
    await brain.init()
  })
  describe('BELOW the bound: the array indexes, every element of it', () => {
    it('the eleven-element array that used to vanish is searchable', async () => {
      // ELEVEN — one over the old silent limit, the whole shape of the defect.
      await brain.add({
        id: 'eleven',
        data: 'a row with eleven tags',
        type: NounType.Document,
        metadata: { tags: tags(11) },
        vector: []
      })
      // Every element is a posting, including the eleventh.
      for (const tag of tags(11)) {
        const hits = await brain.find({ where: { tags: tag }, limit: 10 } as any)
        expect(hits.map((r: any) => r.id)).toContain(resolveEntityId('eleven'))
      }
    })
    it('indexes right up to the bound — all 64 elements', async () => {
      await brain.add({
        id: 'at-bound',
        data: 'a row at the bound',
        type: NounType.Document,
        metadata: { tags: tags(MAX_INDEXED_ARRAY_LENGTH) },
        vector: []
      })
      // The first, the last, and one in the middle.
      for (const tag of ['t0', `t${MAX_INDEXED_ARRAY_LENGTH - 1}`, 't31']) {
        const hits = await brain.find({ where: { tags: tag }, limit: 10 } as any)
        expect(hits.map((r: any) => r.id)).toContain(resolveEntityId('at-bound'))
      }
    })
    it('a nested bag\'s array indexes under its dotted address', async () => {
      await brain.add({
        id: 'nested',
        data: 'a row with a nested tag list',
        type: NounType.Document,
        metadata: { facets: { labels: tags(20, 'l') } },
        vector: []
      })
      const hits = await brain.find({ where: { 'facets.labels': 'l19' }, limit: 10 } as any)
      expect(hits.map((r: any) => r.id)).toContain(resolveEntityId('nested'))
    })
  })
  describe('ABOVE the bound: the write is refused, by name', () => {
    const OVER = MAX_INDEXED_ARRAY_LENGTH + 1
    it('add() throws a typed error naming the field, the length and the bound', async () => {
      const err = await brain
        .add({
          id: 'too-many',
          data: 'a row with too many tags',
          type: NounType.Document,
          metadata: { tags: tags(OVER) },
          vector: []
        } as any)
        .catch((e: any) => e)
      expect(err).toBeInstanceOf(MetadataArrayTooLargeError)
      expect(err.field).toBe('tags')
      expect(err.length).toBe(OVER)
      expect(err.limit).toBe(MAX_INDEXED_ARRAY_LENGTH)
      expect(err.type).toBe('VALIDATION')
      // The message carries all three, and names the cures.
      expect(err.message).toContain('tags')
      expect(err.message).toContain(String(OVER))
      expect(err.message).toContain(String(MAX_INDEXED_ARRAY_LENGTH))
      expect(err.message).toContain('vector')
    })
    it('the refused row is not written at all — no half-indexed ghost', async () => {
      await expect(
        brain.add({
          id: 'refused',
          data: 'refused',
          type: NounType.Document,
          metadata: { tags: tags(OVER) },
          vector: []
        } as any)
      ).rejects.toBeInstanceOf(MetadataArrayTooLargeError)
      expect(await brain.get('refused')).toBeNull()
      const hits = await brain.find({ where: { tags: 't0' }, limit: 10 } as any)
      expect(hits.map((r: any) => r.id)).not.toContain(resolveEntityId('refused'))
    })
    it('a 384-float embedding parked in the metadata bag is refused, not swallowed', async () => {
      const err = await brain
        .add({
          id: 'bag-vector',
          data: 'an embedding in the wrong place',
          type: NounType.Document,
          metadata: { embedding: Array.from({ length: 384 }, (_, i) => i / 384) },
          vector: []
        } as any)
        .catch((e: any) => e)
      expect(err).toBeInstanceOf(MetadataArrayTooLargeError)
      expect(err.field).toBe('embedding')
      expect(err.length).toBe(384)
    })
    it('update() refuses it too', async () => {
      await brain.add({
        id: 'grow',
        data: 'starts small',
        type: NounType.Document,
        metadata: { tags: tags(3) },
        vector: []
      })
      await expect(
        brain.update({ id: 'grow', metadata: { tags: tags(OVER) } } as any)
      ).rejects.toBeInstanceOf(MetadataArrayTooLargeError)
      // And the row keeps the values it had.
      const hits = await brain.find({ where: { tags: 't1' }, limit: 10 } as any)
      expect(hits.map((r: any) => r.id)).toContain(resolveEntityId('grow'))
    })
    it('relate() refuses it on a verb\'s metadata', async () => {
      await brain.add({ id: 'a', data: 'a', type: NounType.Thing, vector: [] })
      await brain.add({ id: 'b', data: 'b', type: NounType.Thing, vector: [] })
      await expect(
        brain.relate({
          from: 'a',
          to: 'b',
          type: VerbType.RelatedTo,
          metadata: { tags: tags(OVER) }
        } as any)
      ).rejects.toBeInstanceOf(MetadataArrayTooLargeError)
    })
    it('a nested oversize array is refused under its dotted address', async () => {
      const err = await brain
        .add({
          id: 'nested-over',
          data: 'nested and too long',
          type: NounType.Document,
          metadata: { facets: { labels: tags(OVER, 'l') } },
          vector: []
        } as any)
        .catch((e: any) => e)
      expect(err).toBeInstanceOf(MetadataArrayTooLargeError)
      expect(err.field).toBe('facets.labels')
    })
  })
  describe('a row already on disk is admitted, and the skip is NARRATED', () => {
    afterEach(() => {
      vi.restoreAllMocks()
    })
    it('extraction over an old oversize row warns by field, length and bound', async () => {
      const warn = vi.spyOn(prodLog, 'warn').mockImplementation(() => {})
      const index = (brain as any).metadataIndex
      // The shape an older engine persisted: the write door never saw it, so
      // this reaches extraction directly — exactly as a rebuild or a remove
      // reading the row back would.
      const fields = index.extractIndexableFields({
        metadata: { tags: tags(MAX_INDEXED_ARRAY_LENGTH + 5), keep: 'me' }
      })
      // The oversize field contributes nothing...
      expect(fields.filter((f: any) => f.field === 'tags')).toHaveLength(0)
      // ...the rest of the row indexes normally — the row is not rejected...
      expect(fields.some((f: any) => f.field === 'keep' && f.value === 'me')).toBe(true)
      // ...and the skip is said out loud, with everything needed to act on it.
      expect(warn).toHaveBeenCalled()
      const said = warn.mock.calls.map((c: any[]) => String(c[0])).join('\n')
      expect(said).toContain('tags')
      expect(said).toContain(String(MAX_INDEXED_ARRAY_LENGTH + 5))
      expect(said).toContain(String(MAX_INDEXED_ARRAY_LENGTH))
      expect(said).toContain('NOT indexed')
    })
    it('an at-bound row on disk is indexed in full and says nothing', async () => {
      const warn = vi.spyOn(prodLog, 'warn').mockImplementation(() => {})
      const index = (brain as any).metadataIndex
      const fields = index.extractIndexableFields({
        metadata: { tags: tags(MAX_INDEXED_ARRAY_LENGTH) }
      })
      expect(fields.filter((f: any) => f.field === 'tags')).toHaveLength(MAX_INDEXED_ARRAY_LENGTH)
      const said = warn.mock.calls.map((c: any[]) => String(c[0])).join('\n')
      expect(said).not.toContain('indexing bound')
    })
  })
 })
--- a/tests/unit/utils/metadataIndex-sparse-range-collation.test.ts
+++ b/tests/unit/utils/metadataIndex-sparse-range-collation.test.ts
@ -1,258 +0,0 @@
 /**
 * @module tests/unit/utils/metadataIndex-sparse-range-collation
 * @description RANGE QUERIES ON THE LEGACY SPARSE INDEX — order, or a refusal.
 * Never a confidently ordered wrong answer.
 *
 * THE TWO RANGE PATHS. `getIdsForRange` routes a `gte` / `lt` / `between` two
 * ways. The column store compares RAW values and is correct. The legacy sparse
 * chunk index — the pre-7.20.0 fallback, still read for workspaces that have
 * not been rebuilt — compared `normalizeValue()` output, and `normalizeValue`
 * carries an escape hatch that destroys order on purpose: a string over 100
 * characters is replaced by a short hash so it can serve as a filesystem-safe
 * key. Ordering hashes ranks rows by digest.
 *
 * THE DEFECT, IN TWO SHAPES.
 *
 * (a) A LONG BOUND against ordinary values. `where: { title: { gte: <a
 *     120-character string> } }` collapsed the BOUND to `__HASH_…`, whose
 *     leading underscores sort below every letter — so a bound that should
 *     have excluded everything matched the entire field instead. This is the
 *     shape that reaches a caller who never stored a long value at all.
 *
 * (b) LONG VALUES in the index. A field whose values ran long was persisted
 *     hashed, so its order is not recoverable from this index at all. The old
 *     code compared the digests anyway and returned a subset chosen by hash.
 *
 * THE LAW. Bounds are normalized WITHOUT the hash escape hatch, so a long
 * bound stays comparable — (a) is simply fixed. Where the persisted KEY is a
 * hash, the order does not exist to be computed, and the query throws a typed
 * `BrainyError('INVALID_QUERY')` naming the field and the cure — (b) is
 * refused by name. Loud beats wrong.
 *
 * THE FIXTURE is a genuine legacy index: it is written through the same
 * `ChunkManager` / `SparseIndex` doors a pre-7.20.0 engine wrote through, with
 * keys normalized exactly as that engine normalized them, into a field the
 * column store does not serve. The chunk WRITE path was removed in 11be039, so
 * this is the only way the shape the read path exists for can be built.
 *
 * NOT CLAIMED HERE. The persisted keys are also lower-cased and trimmed by
 * `normalizeValue`, so this path's string ranges are case-INSENSITIVE where
 * the column store's are not. The raw values are not in the index to compare —
 * that divergence is a property of the bytes on disk and it ends when the
 * column store adopts the field. It is named in `getIdsFromChunksForRange`'s
 * doc comment rather than papered over.
 */
 import { describe, it, expect, beforeEach } from 'vitest'
 import { Brainy } from '../../../src/brainy'
 import { NounType } from '../../../src/types/graphTypes'
 import { SparseIndex, ChunkManager } from '../../../src/utils/metadataIndexChunking'
 import { BrainyError } from '../../../src/errors/brainyError'
 /** The field the legacy index covers — deliberately never given to a row, so
 *  the column store never learns it and the sparse fallback is the only path. */
 const FIELD = 'legacyTitle'
 /**
 * Write a legacy sparse index for `field` exactly as a pre-7.20.0 engine did:
 * one chunk, keys normalized through the index's own `normalizeValue`, ids as
 * roaring bitmaps, a zone map and a bloom filter over the chunk.
 *
 * @param brain - The live brain whose metadata index gains the legacy field.
 * @param field - Field name to index.
 * @param valueToIds - Raw value → the entity ids that carried it.
 */
 async function writeLegacySparseIndex(
  brain: any,
  field: string,
  valueToIds: Array<[string, string[]]>
 ): Promise<void> {
  const index = brain.metadataIndex
  const chunkManager: ChunkManager = index.chunkManager
  const sparseIndex = new SparseIndex(field)
  // The keys a pre-7.20.0 writer persisted: normalizeValue output, hash escape
  // hatch and all. This is what makes the fixture the real shape.
  const chunk = await chunkManager.createChunk(field)
  for (const [value, ids] of valueToIds) {
    const key = index.normalizeValue(value, field)
    for (const id of ids) await chunkManager.addToChunk(chunk, key, id)
  }
  await chunkManager.saveChunk(chunk)
  sparseIndex.registerChunk(
    {
      chunkId: chunk.chunkId,
      field,
      valueCount: chunk.entries.size,
      idCount: Array.from(chunk.entries.values()).reduce((s: number, b: any) => s + b.size, 0),
      zoneMap: (chunkManager as any).calculateZoneMap(chunk),
      lastUpdated: Date.now(),
      splitThreshold: 80,
      mergeThreshold: 20
    },
    chunkManager.createBloomFilter(chunk)
  )
  await index.saveSparseIndex(field, sparseIndex)
 }
 /** A deterministic string of `n` characters starting with `lead`. */
 function longString(lead: string, n: number): string {
  return lead + 'x'.repeat(n - lead.length)
 }
 describe('legacy sparse index: range queries order values, or refuse', () => {
  let brain: Brainy<any>
  let index: any
  let ids: string[]
  beforeEach(async () => {
    brain = new Brainy({ requireSubtype: false, storage: { type: 'memory' } })
    await brain.init()
    index = (brain as any).metadataIndex
    // Rows exist (so the id mapper can resolve them) but carry NO `legacyTitle`
    // — the column store must not serve the field the pins query.
    ids = []
    for (let i = 0; i < 3; i++) {
      const id = `row-${i}`
      await brain.add({ id, data: `row ${i}`, type: NounType.Thing, metadata: { lane: 'a' }, vector: [] })
      ids.push(id)
    }
    expect(index.columnStore.hasField(FIELD)).toBe(false)
  })
  describe('(a) a long BOUND against ordinary short values', () => {
    // 'apple' < 'mango' < 'zebra', and every bound below is compared against
    // these three raw keys.
    beforeEach(async () => {
      await writeLegacySparseIndex(brain, FIELD, [
        ['apple', [ids[0]]],
        ['mango', [ids[1]]],
        ['zebra', [ids[2]]]
      ])
    })
    it('the fixture: the values are stored raw, the long bound is what hashes', () => {
      expect(index.normalizeValue('apple', FIELD)).toBe('apple')
      // The bound is what the old code collapsed — and a digest sorts below
      // every letter, which is exactly why `gte` matched everything.
      const bound = longString('zzz', 120)
      expect(index.normalizeValue(bound, FIELD)).toMatch(/^__HASH_/)
      expect(index.normalizeValue(bound, FIELD) < 'apple').toBe(true)
    })
    it('gte a bound above every value matches NOTHING (it used to match all)', async () => {
      const bound = longString('zzz', 120)
      const matched = await index.getIdsForRange(FIELD, bound, undefined, true, true)
      expect(matched).toEqual([])
    })
    it('lte a bound above every value matches EVERY value', async () => {
      const bound = longString('zzz', 120)
      const matched = await index.getIdsForRange(FIELD, undefined, bound, true, true)
      expect(matched).toHaveLength(3)
    })
    it('gte a long bound below every value matches every value', async () => {
      const bound = longString('aaa', 120)
      const matched = await index.getIdsForRange(FIELD, bound, undefined, true, true)
      expect(matched).toHaveLength(3)
    })
    it('a long bound orders BETWEEN the values, not below all of them', async () => {
      // 'mmm…' sits between 'mango' and 'zebra'.
      const bound = longString('mmm', 120)
      const matched = await index.getIdsForRange(FIELD, bound, undefined, true, true)
      expect(matched).toHaveLength(1)
    })
    it('short bounds are unchanged — the ordinary case still orders correctly', async () => {
      expect(await index.getIdsForRange(FIELD, 'b', undefined, true, true)).toHaveLength(2)
      expect(await index.getIdsForRange(FIELD, undefined, 'n', true, true)).toHaveLength(2)
      expect(await index.getIdsForRange(FIELD, 'b', 'n', true, true)).toHaveLength(1)
      // Strict bounds stay strict.
      expect(await index.getIdsForRange(FIELD, 'mango', undefined, false, true)).toHaveLength(1)
      expect(await index.getIdsForRange(FIELD, 'mango', undefined, true, true)).toHaveLength(2)
    })
  })
  describe('(b) long VALUES — the index holds hashes, so the range is refused', () => {
    beforeEach(async () => {
      await writeLegacySparseIndex(brain, FIELD, [
        [longString('alpha', 140), [ids[0]]],
        [longString('mike', 140), [ids[1]]],
        [longString('zulu', 140), [ids[2]]]
      ])
    })
    it('the fixture: the persisted keys really are hashes', async () => {
      const chunk = await index.chunkManager.loadChunk(FIELD, 0)
      const keys = Array.from(chunk.entries.keys()) as string[]
      expect(keys).toHaveLength(3)
      for (const k of keys) expect(k).toMatch(/^__HASH_/)
      // And their digest order is NOT their value order — the wrong answer the
      // old code returned was wrong, not merely arbitrary.
      const digestOrder = [...keys].sort()
      const valueOrder = [
        index.normalizeValue(longString('alpha', 140), FIELD),
        index.normalizeValue(longString('mike', 140), FIELD),
        index.normalizeValue(longString('zulu', 140), FIELD)
      ]
      expect(digestOrder).not.toEqual(valueOrder)
    })
    it('a range over the hashed field throws a typed refusal naming the field', async () => {
      await expect(
        index.getIdsForRange(FIELD, longString('mike', 140), undefined, true, true)
      ).rejects.toThrow(BrainyError)
      const err = await index
        .getIdsForRange(FIELD, longString('mike', 140), undefined, true, true)
        .catch((e: any) => e)
      expect(err).toBeInstanceOf(BrainyError)
      expect(err.type).toBe('INVALID_QUERY')
      expect(err.message).toContain(FIELD)
      expect(err.message).toContain('hash')
      // The cure is named, not left to the caller to guess.
      expect(err.message).toContain('repairIndex')
    })
    it('every range shape refuses — gte, lte and between alike', async () => {
      const lo = longString('alpha', 140)
      const hi = longString('zulu', 140)
      for (const [min, max] of [
        [lo, undefined],
        [undefined, hi],
        [lo, hi]
      ] as Array<[any, any]>) {
        const err = await index.getIdsForRange(FIELD, min, max, true, true).catch((e: any) => e)
        expect(err).toBeInstanceOf(BrainyError)
        expect(err.type).toBe('INVALID_QUERY')
      }
    })
    it('EQUALITY still works on the hashed field — only ordering is refused', async () => {
      const matched = await index.getIds(FIELD, longString('mike', 140))
      expect(matched).toHaveLength(1)
    })
  })
  describe('numeric ranges on the legacy path are untouched', () => {
    beforeEach(async () => {
      await writeLegacySparseIndex(brain, FIELD, [
        ['5', [ids[0]]],
        ['50', [ids[1]]],
        ['500', [ids[2]]]
      ])
    })
    it('numbers still compare numerically, not lexicographically', async () => {
      // The whole point of compareNormalizedValues: "50" < "500" numerically
      // even though "500" < "50" would hold as strings by prefix.
      expect(await index.getIdsForRange(FIELD, 10, undefined, true, true)).toHaveLength(2)
      expect(await index.getIdsForRange(FIELD, undefined, 100, true, true)).toHaveLength(2)
      expect(await index.getIdsForRange(FIELD, 10, 100, true, true)).toHaveLength(1)
    })
  })
 })