perf(8.0): bound find({ where, orderBy }) sort to the page (CTX-BR-FIND-ORDERBY)

A filtered + ordered find() materialized the FULL sorted match set before slicing
the page — passing k = filteredIds.length to the column store's filteredSortTopK.
A broad filter + orderBy returning 20 rows at billion scale produced hundreds of
millions of sorted ids to discard all but the page (O(matches) heap).

Thread the page bound (offset + limit + the hidden-tier over-fetch) into
getSortedIdsForFilter → filteredSortTopK / sortTopK / the sparse-path slice, so
the sort produces only the page. Ordering and pagination are unchanged.

From cor's 2026-06-24 co-release scaling audit (item 1).

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

RELEASES.md

@@ -73,6 +73,11 @@ no separate migration doc. **`8.0.0-rc.2` is live** — install with
 >   matches that DO pass the filter, rather than coming back empty. No API change. With the native
 >   `@soulcraft/cor` 3.0 stack the matched universe is forwarded as an opaque roaring buffer with
 >   zero id materialization in TypeScript; the pure-JS path restricts the beam walk with a string set.
+> - **`find({ where, orderBy })` bounds the sort to the page.** A broad filter + `orderBy`
+>   returning one page no longer materializes every matching sorted id (it produced the full sorted
+>   match set, then sliced) — the page bound (`offset + limit`) is threaded into the column store's
+>   top-K sort, so returning 20 rows from a million matches stays a bounded-K heap. No API change;
+>   ordering and pagination are unchanged.
 > - **`brain.graph.subgraph()` accepts a query (query→expand).** The seed selector now takes not
 >   just entity id(s) but a `find()` result or a `FindParams` query — `brain.graph.subgraph({ where:
 >   { team: 'platform' } }, { depth: 1 })` runs the query and expands the neighborhood of every

src/brainy.ts

@@ -4763,11 +4763,15 @@ export class Brainy<T = any> implements BrainyInterface<T> {
         // Apply sorting if requested, otherwise just filter
         let filteredIds: string[]
         if (params.orderBy) {
-          // Get sorted IDs using production-scale sorted filtering
+          // Get sorted IDs using production-scale sorted filtering. Bound the sort to
+          // the page (offset+limit) plus the hidden over-fetch — so a broad filter +
+          // orderBy returning 20 rows doesn't materialize every matching sorted id.
+          const sortTopK = (params.offset || 0) + (params.limit || 10) + hiddenIds.size
           filteredIds = await this.metadataIndex.getSortedIdsForFilter(
             filter,
             params.orderBy,
-            params.order || 'asc'
+            params.order || 'asc',
+            sortTopK
           )
         } else {
           // Just filter without sorting
@@ -5049,15 +5053,18 @@ export class Brainy<T = any> implements BrainyInterface<T> {
           if (!params.query && !params.connected) {
             // Apply sorting if requested for metadata-only queries
             if (params.orderBy) {
+              // Page-bounded sort (offset+limit + hidden over-fetch) so we don't
+              // materialize every matching sorted id to return one page.
+              const limit = params.limit || 10
+              const offset = params.offset || 0
               const sortedIds = await this.metadataIndex.getSortedIdsForFilter(
                 preResolvedFilter,
                 params.orderBy,
-                params.order || 'asc'
+                params.order || 'asc',
+                offset + limit + hiddenIds.size
               )
 
               // Paginate sorted IDs BEFORE loading entities (production-scale!)
-              const limit = params.limit || 10
-              const offset = params.offset || 0
               const pageIds = sortedIds.slice(offset, offset + limit)
 
               // Batch-load entities for paginated results (10x faster on GCS)

src/plugin.ts

@@ -147,7 +147,7 @@ export interface MetadataIndexProvider {
    */
   getIdSetForFilter?(filter: any): Promise<OpaqueIdSet>
   getIdsForTextQuery(query: string): Promise<Array<{ id: string; matchCount: number }>>
-  getSortedIdsForFilter(filter: any, orderBy: string, order?: 'asc' | 'desc'): Promise<string[]>
+  getSortedIdsForFilter(filter: any, orderBy: string, order?: 'asc' | 'desc', topK?: number): Promise<string[]>
   getFilterValues(field: string): Promise<string[]>
   getFilterFields(): Promise<string[]>
   getFieldValueForEntity(entityId: string, field: string): Promise<any>

src/utils/metadataIndex.ts

@@ -2121,13 +2121,19 @@ export class MetadataIndexManager implements MetadataIndexProvider {
    * @param filter - Metadata filter criteria (uses roaring bitmaps)
    * @param orderBy - Field name to sort by (e.g., 'createdAt', 'title')
    * @param order - Sort direction: 'asc' (default) or 'desc'
+   * @param topK - Optional page bound: produce only the top `K` sorted ids
+   *   (`offset + limit`) instead of the full sorted match set. A broad filter +
+   *   orderBy that returns one page no longer materializes hundreds of millions of
+   *   sorted ids at billion scale — the column store's top-K heap produces only the
+   *   page. Omit for the full sorted set.
    * @returns Promise<string[]> - Entity IDs sorted by specified field
    *
    */
   async getSortedIdsForFilter(
     filter: any,
     orderBy: string,
-    order: 'asc' | 'desc' = 'asc'
+    order: 'asc' | 'desc' = 'asc',
+    topK?: number
   ): Promise<string[]> {
     // Column store path: O(K log S) sort via k-way merge across segments.
     // No per-entity storage reads, no precision loss from bucketing.
@@ -2146,13 +2152,17 @@ export class MetadataIndexManager implements MetadataIndexProvider {
           const intId = this.idMapper.getInt(id)
           if (intId !== undefined) filterBitmap.add(intId)
         }
+        // Page-bounded: produce only the top `topK` (offset+limit), not every
+        // match, so a broad filter + orderBy returning one page stays O(matches
+        // log K) heap, not a full sort materialization.
+        const k = topK !== undefined ? Math.min(topK, filteredIds.length) : filteredIds.length
         sortedIntIds = await this.columnStore.filteredSortTopK(
-          filterBitmap, orderBy, order, filteredIds.length
+          filterBitmap, orderBy, order, k
         )
       } else {
         // Unfiltered sort — column store handles the full entity set efficiently
         sortedIntIds = await this.columnStore.sortTopK(
-          orderBy, order, this.idMapper.size
+          orderBy, order, topK !== undefined ? Math.min(topK, this.idMapper.size) : this.idMapper.size
         )
       }
 
@@ -2195,7 +2205,8 @@ export class MetadataIndexManager implements MetadataIndexProvider {
       return order === 'asc' ? comparison : -comparison
     })
 
-    return idValuePairs.map(p => p.id)
+    const sorted = idValuePairs.map(p => p.id)
+    return topK !== undefined ? sorted.slice(0, topK) : sorted
   }
 
   /**

tests/unit/brainy/find-orderby-pagek.test.ts

@@ -0,0 +1,60 @@
+/**
+ * @module tests/unit/brainy/find-orderby-pagek
+ * @description CTX-BR-FIND-ORDERBY item (1) — `find({ where, orderBy, limit })` must
+ * produce only the requested PAGE of sorted ids, not the full sorted match set. At
+ * billion scale a broad filter + orderBy returning 20 rows previously materialized
+ * every matching sorted id (O(matches) heap); the page bound (`offset+limit`) is now
+ * threaded into `getSortedIdsForFilter` → the column store's top-K heap. Ordering
+ * correctness is covered by orderby-sort-bug.test.ts; this locks the page bound.
+ */
+
+import { describe, it, expect, beforeEach, afterEach } from 'vitest'
+import { Brainy } from '../../../src/index.js'
+import { NounType } from '../../../src/types/graphTypes.js'
+import { createTestConfig } from '../../helpers/test-factory.js'
+
+describe('find({ where, orderBy }) bounds the sort to the page (CTX-BR-FIND-ORDERBY #1)', () => {
+  let brain: Brainy
+  const MATCHES = 50
+
+  beforeEach(async () => {
+    brain = new Brainy(createTestConfig())
+    await brain.init()
+    for (let i = 0; i < MATCHES; i++) {
+      await brain.add({ type: NounType.Document, data: `doc ${i}`, metadata: { bucket: 'x', seq: i } })
+    }
+  })
+
+  afterEach(async () => {
+    await brain.close()
+  })
+
+  it('threads the page bound (offset+limit) into getSortedIdsForFilter, not the full match count', async () => {
+    let capturedTopK: number | undefined
+    const real = (brain as any).metadataIndex.getSortedIdsForFilter.bind((brain as any).metadataIndex)
+    ;(brain as any).metadataIndex.getSortedIdsForFilter = async (
+      f: any,
+      ob: string,
+      o: 'asc' | 'desc',
+      topK?: number
+    ) => {
+      capturedTopK = topK
+      return real(f, ob, o, topK)
+    }
+
+    const results = await brain.find({ where: { bucket: 'x' }, orderBy: 'createdAt', order: 'desc', limit: 5 })
+
+    expect(results).toHaveLength(5)
+    // Page-bounded: ~ limit (5) + a small hidden-tier over-fetch — NOT all 50 matches.
+    expect(capturedTopK).toBeDefined()
+    expect(capturedTopK!).toBeGreaterThanOrEqual(5)
+    expect(capturedTopK!).toBeLessThan(MATCHES)
+  })
+
+  it('still returns the correct page with offset (sort + window unchanged)', async () => {
+    // seq is a numeric metadata field; orderBy seq asc → 0,1,2,… ; page [10,15).
+    const page = await brain.find({ where: { bucket: 'x' }, orderBy: 'seq', order: 'asc', limit: 5, offset: 10 })
+    expect(page).toHaveLength(5)
+    expect(page.map((r) => (r.metadata as { seq: number }).seq)).toEqual([10, 11, 12, 13, 14])
+  })
+})