perf(sort): ordered reads never do per-row storage round-trips — the 199-317s production scan class dies structurally

BRAINY-PROD-LATENCY-TRIAD Track A1 (David-approved plan): the sort path's
value resolution goes BATCHED — one chunked metadata-record batch pass
serves any N, replacing the serial per-row getNoun loop (62-98ms x 3,224
rows = the measured 199-317 second silent scan on self prod). The
metadata record carries every sortable value: system scalars EXACT
(bucketed-index precision loss can never force a per-row disk read
again) and the user bag via the shape-aware split, both record eras.

- resolveOrderValuesBatch: the one sanctioned value source for ordered
  reads (batch door: getNounMetadataBatch -> getMetadataBatch -> chunked
  parallel; never serial).
- Column top-K page re-sort and the no-column fallback both rewired.
- B2 down-payment: the no-column fallback ANNOUNCES itself once per
  field past 500 rows - silent degradation is illegal.
- THE CALL-SHAPE PIN (tests/unit/utils/metadataIndex-sort-callshape):
  zero vector-record reads, batch calls only, latency-blind so it holds
  on any machine - the serial loop cannot quietly return. Ordering
  contract re-pinned through the batch path (nulls last both directions,
  ties by id, never drop).

(! = perf contract change only; no API change. Gates: unit 1904/1904,
integration 758, conformance 27/27.)

src/utils/metadataIndex.ts

@@ -5,7 +5,8 @@
  */
 
 import { StorageAdapter, resolveEntityField, NounMetadata, VerbMetadata } from '../coreTypes.js'
-import { SYSTEM_ENTITY_SCALARS, parseFieldAddress, UnresolvableFieldError } from '../db/fieldAddressing.js'
+import { SYSTEM_ENTITY_SCALARS, parseFieldAddress, UnresolvableFieldError, type FieldAddress } from '../db/fieldAddressing.js'
+import { splitNounMetadataRecord } from '../types/reservedFields.js'
 import { ColumnStore } from '../indexes/columnStore/ColumnStore.js'
 import type { MetadataIndexProvider } from '../plugin.js'
 import { MetadataIndexCache, MetadataIndexCacheConfig } from './metadataIndexCache.js'
@@ -2207,6 +2208,98 @@ export class MetadataIndexManager implements MetadataIndexProvider {
    * @returns Promise<string[]> - Entity IDs sorted by specified field
    *
    */
+  /**
+   * Resolve the orderBy value for MANY entities in BATCHED metadata-record
+   * reads — the sort path's one sanctioned value source (BRAINY-PROD-LATENCY-TRIAD).
+   *
+   * THE ASYMPTOTIC LAW THIS ENFORCES: an ordered read never does per-row
+   * storage round-trips. The previous shape — `await getFieldValueForEntity`
+   * per id, each opening the VECTOR record serially — cost 62–98ms × N on a
+   * production filesystem brain: 3,224 rows took 199–317 SECONDS, silently.
+   * The metadata RECORD (smaller, cached, batch-readable) carries everything
+   * a sort can address: the ten system scalars top-level — EXACT values, no
+   * bucketing loss — and the user's bag (v2 nested or legacy flat, resolved
+   * through the shape-aware split). One batched read pass serves any N.
+   *
+   * The call-shape is pinned by tests (zero per-row reads, batch calls only)
+   * so the serial loop cannot quietly return.
+   *
+   * @param ids - Entity ids to resolve (any size; reads are chunk-batched).
+   * @param orderAddress - The parsed orderBy address (system or metadata scope).
+   * @returns id → value map; ids whose record is missing map to `undefined`
+   *   (they sort LAST per the ordering contract — never dropped).
+   */
+  private async resolveOrderValuesBatch(
+    ids: string[],
+    orderAddress: FieldAddress
+  ): Promise<Map<string, unknown>> {
+    const values = new Map<string, unknown>()
+    if (ids.length === 0) return values
+
+    // Batch door, best first: BaseStorage's getNounMetadataBatch (native
+    // batch or parallel reads inside), then the adapter-optional
+    // getMetadataBatch, then chunked-parallel single reads — NEVER serial.
+    const storage = this.storage as StorageAdapter & {
+      getNounMetadataBatch?(ids: string[]): Promise<Map<string, NounMetadata>>
+    }
+    const CHUNK = 500
+    const records = new Map<string, NounMetadata>()
+    for (let i = 0; i < ids.length; i += CHUNK) {
+      const chunk = ids.slice(i, i + CHUNK)
+      if (typeof storage.getNounMetadataBatch === 'function') {
+        const batch = await storage.getNounMetadataBatch(chunk)
+        for (const [id, rec] of batch) records.set(id, rec)
+      } else if (typeof storage.getMetadataBatch === 'function') {
+        const batch = await storage.getMetadataBatch(chunk)
+        for (const [id, rec] of batch) records.set(id, rec)
+      } else {
+        const loaded = await Promise.all(
+          chunk.map(async (id) => [id, await storage.getNounMetadata(id)] as const)
+        )
+        for (const [id, rec] of loaded) if (rec) records.set(id, rec)
+      }
+    }
+
+    for (const id of ids) {
+      const record = records.get(id)
+      if (!record) {
+        values.set(id, undefined)
+        continue
+      }
+      // Shape-aware split serves both record eras: engine scalars from the
+      // reserved half (EXACT timestamps — the bucketed index is never
+      // consulted here), user fields from the bag.
+      const { reserved, custom } = splitNounMetadataRecord(
+        record as Record<string, unknown>
+      )
+      if (orderAddress.scope === 'system') {
+        values.set(
+          id,
+          orderAddress.field === 'type'
+            ? reserved.noun
+            : (reserved as Record<string, unknown>)[orderAddress.field]
+        )
+      } else {
+        let value: unknown = custom[orderAddress.field]
+        if (value === undefined && orderAddress.field.includes('.')) {
+          // Dotted user path: traverse INSIDE the bag.
+          value = orderAddress.field
+            .split('.')
+            .reduce<unknown>(
+              (o, seg) =>
+                o && typeof o === 'object' ? (o as Record<string, unknown>)[seg] : undefined,
+              custom
+            )
+        }
+        values.set(id, value)
+      }
+    }
+    return values
+  }
+
+  /** Once-per-field flag for the fallback-degradation announcement. */
+  private static announcedFallbackSorts = new Set<string>()
+
   async getSortedIdsForFilter(
     filter: any,
     orderBy: string,
@@ -2274,12 +2367,12 @@ export class MetadataIndexManager implements MetadataIndexProvider {
       // ORDERING CONTRACT (cross-engine, sealed): rows missing the field are
       // NEVER dropped — they sort LAST in both directions — and ties break by
       // id ascending. The column only contains rows that HAVE the field, so
-      // (1) re-sort the page deterministically (value, then id) with K cheap
-      // value reads, and (2) append the filtered rows the column omitted,
-      // id-ascending, filling any remaining page budget.
-      const page = await Promise.all(
-        sortedUuids.map(async id => ({ id, value: await this.getFieldValueForEntity(id, orderKey) }))
-      )
+      // (1) re-sort the page deterministically (value, then id) via ONE
+      // batched value resolution — never per-row reads — and (2) append the
+      // filtered rows the column omitted, id-ascending, filling any
+      // remaining page budget.
+      const pageValues = await this.resolveOrderValuesBatch(sortedUuids, orderAddress)
+      const page = sortedUuids.map(id => ({ id, value: pageValues.get(id) }))
       page.sort((a, b) => this.compareAddressedValues(a.value, b.value, a.id, b.id, order))
       let result = page.map(p => p.id)
 
@@ -2293,20 +2386,32 @@ export class MetadataIndexManager implements MetadataIndexProvider {
       return topK !== undefined ? result.slice(0, topK) : result
     }
 
-    // Fallback: sparse index path (for fields not yet in column store).
-    // Requires a non-empty filter because it reads O(k) entity values from storage.
+    // Fallback: no column serves this field. BOUNDED + ANNOUNCED, never
+    // silent (the B2 no-silent-degradation law, BRAINY-PROD-LATENCY-TRIAD):
+    // O(N) in row count but served by BATCHED metadata-record reads — the
+    // serial per-row getNoun loop that turned 3,224 rows into a 199–317s
+    // scan is dead, and the call-shape pin keeps it dead.
     const filteredIds = await this.getIdsForFilter(filter)
 
     if (filteredIds.length === 0) {
       return []
     }
 
-    const idValuePairs: Array<{ id: string, value: any }> = []
-    for (const id of filteredIds) {
-      const value = await this.getFieldValueForEntity(id, orderKey)
-      idValuePairs.push({ id, value })
+    if (
+      filteredIds.length > 500 &&
+      !MetadataIndexManager.announcedFallbackSorts.has(orderKey)
+    ) {
+      MetadataIndexManager.announcedFallbackSorts.add(orderKey)
+      prodLog.warn(
+        `[brainy] ordered read on '${orderKey}' has no column index — served by the ` +
+          `batched fallback over ${filteredIds.length} rows (bounded, one batch pass; ` +
+          `announced once per field). A native column for this field makes it O(K).`
+      )
     }
 
+    const fallbackValues = await this.resolveOrderValuesBatch(filteredIds, orderAddress)
+    const idValuePairs = filteredIds.map(id => ({ id, value: fallbackValues.get(id) }))
+
     idValuePairs.sort((a, b) => this.compareAddressedValues(a.value, b.value, a.id, b.id, order))
 
     const sorted = idValuePairs.map(p => p.id)