perf(sort): ordered reads never do per-row storage round-trips — the 199-317s production scan class dies structurally

BRAINY-PROD-LATENCY-TRIAD Track A1 (David-approved plan): the sort path's
value resolution goes BATCHED — one chunked metadata-record batch pass
serves any N, replacing the serial per-row getNoun loop (62-98ms x 3,224
rows = the measured 199-317 second silent scan on self prod). The
metadata record carries every sortable value: system scalars EXACT
(bucketed-index precision loss can never force a per-row disk read
again) and the user bag via the shape-aware split, both record eras.

- resolveOrderValuesBatch: the one sanctioned value source for ordered
  reads (batch door: getNounMetadataBatch -> getMetadataBatch -> chunked
  parallel; never serial).
- Column top-K page re-sort and the no-column fallback both rewired.
- B2 down-payment: the no-column fallback ANNOUNCES itself once per
  field past 500 rows - silent degradation is illegal.
- THE CALL-SHAPE PIN (tests/unit/utils/metadataIndex-sort-callshape):
  zero vector-record reads, batch calls only, latency-blind so it holds
  on any machine - the serial loop cannot quietly return. Ordering
  contract re-pinned through the batch path (nulls last both directions,
  ties by id, never drop).

(! = perf contract change only; no API change. Gates: unit 1904/1904,
integration 758, conformance 27/27.)

src/utils/metadataIndex.ts

@@ -5,7 +5,8 @@
  */
 
 import { StorageAdapter, resolveEntityField, NounMetadata, VerbMetadata } from '../coreTypes.js'
-import { SYSTEM_ENTITY_SCALARS, parseFieldAddress, UnresolvableFieldError } from '../db/fieldAddressing.js'
+import { SYSTEM_ENTITY_SCALARS, parseFieldAddress, UnresolvableFieldError, type FieldAddress } from '../db/fieldAddressing.js'
+import { splitNounMetadataRecord } from '../types/reservedFields.js'
 import { ColumnStore } from '../indexes/columnStore/ColumnStore.js'
 import type { MetadataIndexProvider } from '../plugin.js'
 import { MetadataIndexCache, MetadataIndexCacheConfig } from './metadataIndexCache.js'
@@ -2207,6 +2208,98 @@ export class MetadataIndexManager implements MetadataIndexProvider {
    * @returns Promise<string[]> - Entity IDs sorted by specified field
    *
    */
+  /**
+   * Resolve the orderBy value for MANY entities in BATCHED metadata-record
+   * reads — the sort path's one sanctioned value source (BRAINY-PROD-LATENCY-TRIAD).
+   *
+   * THE ASYMPTOTIC LAW THIS ENFORCES: an ordered read never does per-row
+   * storage round-trips. The previous shape — `await getFieldValueForEntity`
+   * per id, each opening the VECTOR record serially — cost 62–98ms × N on a
+   * production filesystem brain: 3,224 rows took 199–317 SECONDS, silently.
+   * The metadata RECORD (smaller, cached, batch-readable) carries everything
+   * a sort can address: the ten system scalars top-level — EXACT values, no
+   * bucketing loss — and the user's bag (v2 nested or legacy flat, resolved
+   * through the shape-aware split). One batched read pass serves any N.
+   *
+   * The call-shape is pinned by tests (zero per-row reads, batch calls only)
+   * so the serial loop cannot quietly return.
+   *
+   * @param ids - Entity ids to resolve (any size; reads are chunk-batched).
+   * @param orderAddress - The parsed orderBy address (system or metadata scope).
+   * @returns id → value map; ids whose record is missing map to `undefined`
+   *   (they sort LAST per the ordering contract — never dropped).
+   */
+  private async resolveOrderValuesBatch(
+    ids: string[],
+    orderAddress: FieldAddress
+  ): Promise<Map<string, unknown>> {
+    const values = new Map<string, unknown>()
+    if (ids.length === 0) return values
+
+    // Batch door, best first: BaseStorage's getNounMetadataBatch (native
+    // batch or parallel reads inside), then the adapter-optional
+    // getMetadataBatch, then chunked-parallel single reads — NEVER serial.
+    const storage = this.storage as StorageAdapter & {
+      getNounMetadataBatch?(ids: string[]): Promise<Map<string, NounMetadata>>
+    }
+    const CHUNK = 500
+    const records = new Map<string, NounMetadata>()
+    for (let i = 0; i < ids.length; i += CHUNK) {
+      const chunk = ids.slice(i, i + CHUNK)
+      if (typeof storage.getNounMetadataBatch === 'function') {
+        const batch = await storage.getNounMetadataBatch(chunk)
+        for (const [id, rec] of batch) records.set(id, rec)
+      } else if (typeof storage.getMetadataBatch === 'function') {
+        const batch = await storage.getMetadataBatch(chunk)
+        for (const [id, rec] of batch) records.set(id, rec)
+      } else {
+        const loaded = await Promise.all(
+          chunk.map(async (id) => [id, await storage.getNounMetadata(id)] as const)
+        )
+        for (const [id, rec] of loaded) if (rec) records.set(id, rec)
+      }
+    }
+
+    for (const id of ids) {
+      const record = records.get(id)
+      if (!record) {
+        values.set(id, undefined)
+        continue
+      }
+      // Shape-aware split serves both record eras: engine scalars from the
+      // reserved half (EXACT timestamps — the bucketed index is never
+      // consulted here), user fields from the bag.
+      const { reserved, custom } = splitNounMetadataRecord(
+        record as Record<string, unknown>
+      )
+      if (orderAddress.scope === 'system') {
+        values.set(
+          id,
+          orderAddress.field === 'type'
+            ? reserved.noun
+            : (reserved as Record<string, unknown>)[orderAddress.field]
+        )
+      } else {
+        let value: unknown = custom[orderAddress.field]
+        if (value === undefined && orderAddress.field.includes('.')) {
+          // Dotted user path: traverse INSIDE the bag.
+          value = orderAddress.field
+            .split('.')
+            .reduce<unknown>(
+              (o, seg) =>
+                o && typeof o === 'object' ? (o as Record<string, unknown>)[seg] : undefined,
+              custom
+            )
+        }
+        values.set(id, value)
+      }
+    }
+    return values
+  }
+
+  /** Once-per-field flag for the fallback-degradation announcement. */
+  private static announcedFallbackSorts = new Set<string>()
+
   async getSortedIdsForFilter(
     filter: any,
     orderBy: string,
@@ -2274,12 +2367,12 @@ export class MetadataIndexManager implements MetadataIndexProvider {
       // ORDERING CONTRACT (cross-engine, sealed): rows missing the field are
       // NEVER dropped — they sort LAST in both directions — and ties break by
       // id ascending. The column only contains rows that HAVE the field, so
-      // (1) re-sort the page deterministically (value, then id) with K cheap
-      // value reads, and (2) append the filtered rows the column omitted,
-      // id-ascending, filling any remaining page budget.
-      const page = await Promise.all(
-        sortedUuids.map(async id => ({ id, value: await this.getFieldValueForEntity(id, orderKey) }))
-      )
+      // (1) re-sort the page deterministically (value, then id) via ONE
+      // batched value resolution — never per-row reads — and (2) append the
+      // filtered rows the column omitted, id-ascending, filling any
+      // remaining page budget.
+      const pageValues = await this.resolveOrderValuesBatch(sortedUuids, orderAddress)
+      const page = sortedUuids.map(id => ({ id, value: pageValues.get(id) }))
       page.sort((a, b) => this.compareAddressedValues(a.value, b.value, a.id, b.id, order))
       let result = page.map(p => p.id)
 
@@ -2293,20 +2386,32 @@ export class MetadataIndexManager implements MetadataIndexProvider {
       return topK !== undefined ? result.slice(0, topK) : result
     }
 
-    // Fallback: sparse index path (for fields not yet in column store).
-    // Requires a non-empty filter because it reads O(k) entity values from storage.
+    // Fallback: no column serves this field. BOUNDED + ANNOUNCED, never
+    // silent (the B2 no-silent-degradation law, BRAINY-PROD-LATENCY-TRIAD):
+    // O(N) in row count but served by BATCHED metadata-record reads — the
+    // serial per-row getNoun loop that turned 3,224 rows into a 199–317s
+    // scan is dead, and the call-shape pin keeps it dead.
     const filteredIds = await this.getIdsForFilter(filter)
 
     if (filteredIds.length === 0) {
       return []
     }
 
-    const idValuePairs: Array<{ id: string, value: any }> = []
-    for (const id of filteredIds) {
-      const value = await this.getFieldValueForEntity(id, orderKey)
-      idValuePairs.push({ id, value })
+    if (
+      filteredIds.length > 500 &&
+      !MetadataIndexManager.announcedFallbackSorts.has(orderKey)
+    ) {
+      MetadataIndexManager.announcedFallbackSorts.add(orderKey)
+      prodLog.warn(
+        `[brainy] ordered read on '${orderKey}' has no column index — served by the ` +
+          `batched fallback over ${filteredIds.length} rows (bounded, one batch pass; ` +
+          `announced once per field). A native column for this field makes it O(K).`
+      )
     }
 
+    const fallbackValues = await this.resolveOrderValuesBatch(filteredIds, orderAddress)
+    const idValuePairs = filteredIds.map(id => ({ id, value: fallbackValues.get(id) }))
+
     idValuePairs.sort((a, b) => this.compareAddressedValues(a.value, b.value, a.id, b.id, order))
 
     const sorted = idValuePairs.map(p => p.id)

tests/unit/utils/metadataIndex-sort-callshape.test.ts

@@ -0,0 +1,119 @@
+/**
+ * @module tests/unit/utils/metadataIndex-sort-callshape
+ * @description THE ASYMPTOTIC CALL-SHAPE PIN for ordered reads
+ * (BRAINY-PROD-LATENCY-TRIAD, David-approved plan Track A1). The defect it
+ * keeps dead: `getSortedIdsForFilter`'s value resolution did a SERIAL
+ * `storage.getNoun()` (the heavyweight VECTOR record) per filtered row —
+ * 62–98ms × 3,224 rows = the measured 199–317 SECOND production sort, with
+ * `topK` applied only after the full scan. These pins assert the SHAPE of
+ * the storage traffic, not wall-clock (latency-blind, so they hold on any
+ * machine): an ordered read performs ZERO per-row vector-record reads and
+ * resolves sort values through BATCHED metadata-record calls only.
+ */
+import { describe, it, expect, beforeAll, afterAll, vi } from 'vitest'
+import { Brainy } from '../../../src/index.js'
+import { NounType } from '../../../src/types/graphTypes.js'
+
+const ROWS = 60
+
+describe('ordered reads — the batched call-shape law (no per-row storage loops)', () => {
+  let brain: Brainy
+  let storage: {
+    getNoun: (id: string) => Promise<unknown>
+    getNounMetadata: (id: string) => Promise<unknown>
+    getNounMetadataBatch: (ids: string[]) => Promise<Map<string, unknown>>
+  }
+
+  beforeAll(async () => {
+    brain = new Brainy({ storage: { type: 'memory' }, requireSubtype: false })
+    await brain.init()
+    for (let i = 0; i < ROWS; i++) {
+      await brain.add({
+        data: `row ${i}`,
+        type: NounType.Document,
+        metadata: { rank: (i * 7) % ROWS, plain: `p${i}` }
+      })
+    }
+    storage = (brain as unknown as { storage: typeof storage }).storage
+  }, 120000)
+
+  afterAll(async () => {
+    await brain.close().catch(() => {})
+  })
+
+  it('user-field orderBy: zero vector-record reads, zero serial metadata reads — batch calls only', async () => {
+    const getNounSpy = vi.spyOn(storage, 'getNoun')
+    const singleReadSpy = vi.spyOn(storage, 'getNounMetadata')
+    const batchSpy = vi.spyOn(storage, 'getNounMetadataBatch')
+
+    const rows = await brain.find({
+      type: NounType.Document,
+      orderBy: 'rank',
+      order: 'desc',
+      limit: 10
+    })
+    expect(rows.length).toBe(10)
+    expect((rows[0].metadata as Record<string, unknown>).rank).toBe(ROWS - 1)
+
+    // THE PIN: the sort's value resolution never opens a vector record and
+    // never falls into a per-row metadata loop. (Result hydration after
+    // pagination is allowed to read; the SORT itself must be batch-only —
+    // hence the ceiling: strictly fewer single reads than sorted rows.)
+    expect(getNounSpy.mock.calls.length, 'per-row vector-record reads in an ordered read').toBe(0)
+    expect(batchSpy.mock.calls.length, 'the batch door was used').toBeGreaterThanOrEqual(1)
+    expect(
+      singleReadSpy.mock.calls.length,
+      'serial per-row metadata reads (the 199s shape)'
+    ).toBeLessThan(ROWS / 2)
+
+    vi.restoreAllMocks()
+  })
+
+  it('system.createdAt orderBy: exact values from batched records — the bucketed index is never a per-row disk excuse', async () => {
+    const getNounSpy = vi.spyOn(storage, 'getNoun')
+    const batchSpy = vi.spyOn(storage, 'getNounMetadataBatch')
+
+    const rows = await brain.find({
+      type: NounType.Document,
+      orderBy: 'system.createdAt',
+      order: 'asc',
+      limit: 15
+    })
+    expect(rows.length).toBe(15)
+
+    expect(getNounSpy.mock.calls.length, 'per-row vector-record reads').toBe(0)
+    expect(batchSpy.mock.calls.length).toBeGreaterThanOrEqual(1)
+
+    // Exactness: ascending createdAt must be non-decreasing with full
+    // millisecond precision (the old path sorted minute-BUCKETED values or
+    // paid a per-row disk read for exact ones — both are dead). Find results
+    // carry the timestamps on the nested full entity.
+    const stamps = rows.map(
+      (r) => ((r as unknown as { entity?: { createdAt?: number } }).entity?.createdAt ??
+        (r as unknown as { createdAt?: number }).createdAt) as number
+    )
+    for (let i = 1; i < stamps.length; i++) {
+      expect(stamps[i]).toBeGreaterThanOrEqual(stamps[i - 1])
+    }
+
+    vi.restoreAllMocks()
+  })
+
+  it('the ordering contract survives the batch path: missing values LAST both directions, ties by id asc, rows never dropped', async () => {
+    // Three rows lack `rank`? No — all carry it; add two rows WITHOUT it.
+    const a = await brain.add({ data: 'no-rank a', type: NounType.Document, metadata: { plain: 'x' } })
+    const b = await brain.add({ data: 'no-rank b', type: NounType.Document, metadata: { plain: 'y' } })
+
+    for (const order of ['asc', 'desc'] as const) {
+      const rows = await brain.find({
+        type: NounType.Document,
+        orderBy: 'rank',
+        order,
+        limit: ROWS + 10
+      })
+      expect(rows.length, `complete result (${order})`).toBe(ROWS + 2)
+      const lastTwo = rows.slice(-2).map((r) => r.id).sort()
+      expect(lastTwo, `missing-value rows sort LAST (${order})`).toEqual([a, b].sort())
+    }
+  })
+})