perf(8.0): bound find({ where, orderBy }) sort to the page (CTX-BR-FIND-ORDERBY)

A filtered + ordered find() materialized the FULL sorted match set before slicing
the page — passing k = filteredIds.length to the column store's filteredSortTopK.
A broad filter + orderBy returning 20 rows at billion scale produced hundreds of
millions of sorted ids to discard all but the page (O(matches) heap).

Thread the page bound (offset + limit + the hidden-tier over-fetch) into
getSortedIdsForFilter → filteredSortTopK / sortTopK / the sparse-path slice, so
the sort produces only the page. Ordering and pagination are unchanged.

From cor's 2026-06-24 co-release scaling audit (item 1).

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
David Snelling 2026-06-23 15:01:49 -07:00
parent 82dde92077
commit 450084b6ce
5 changed files with 93 additions and 10 deletions

View file

@ -0,0 +1,60 @@
/**
* @module tests/unit/brainy/find-orderby-pagek
* @description CTX-BR-FIND-ORDERBY item (1) `find({ where, orderBy, limit })` must
* produce only the requested PAGE of sorted ids, not the full sorted match set. At
* billion scale a broad filter + orderBy returning 20 rows previously materialized
* every matching sorted id (O(matches) heap); the page bound (`offset+limit`) is now
* threaded into `getSortedIdsForFilter` the column store's top-K heap. Ordering
* correctness is covered by orderby-sort-bug.test.ts; this locks the page bound.
*/
import { describe, it, expect, beforeEach, afterEach } from 'vitest'
import { Brainy } from '../../../src/index.js'
import { NounType } from '../../../src/types/graphTypes.js'
import { createTestConfig } from '../../helpers/test-factory.js'
describe('find({ where, orderBy }) bounds the sort to the page (CTX-BR-FIND-ORDERBY #1)', () => {
let brain: Brainy
const MATCHES = 50
beforeEach(async () => {
brain = new Brainy(createTestConfig())
await brain.init()
for (let i = 0; i < MATCHES; i++) {
await brain.add({ type: NounType.Document, data: `doc ${i}`, metadata: { bucket: 'x', seq: i } })
}
})
afterEach(async () => {
await brain.close()
})
it('threads the page bound (offset+limit) into getSortedIdsForFilter, not the full match count', async () => {
let capturedTopK: number | undefined
const real = (brain as any).metadataIndex.getSortedIdsForFilter.bind((brain as any).metadataIndex)
;(brain as any).metadataIndex.getSortedIdsForFilter = async (
f: any,
ob: string,
o: 'asc' | 'desc',
topK?: number
) => {
capturedTopK = topK
return real(f, ob, o, topK)
}
const results = await brain.find({ where: { bucket: 'x' }, orderBy: 'createdAt', order: 'desc', limit: 5 })
expect(results).toHaveLength(5)
// Page-bounded: ~ limit (5) + a small hidden-tier over-fetch — NOT all 50 matches.
expect(capturedTopK).toBeDefined()
expect(capturedTopK!).toBeGreaterThanOrEqual(5)
expect(capturedTopK!).toBeLessThan(MATCHES)
})
it('still returns the correct page with offset (sort + window unchanged)', async () => {
// seq is a numeric metadata field; orderBy seq asc → 0,1,2,… ; page [10,15).
const page = await brain.find({ where: { bucket: 'x' }, orderBy: 'seq', order: 'asc', limit: 5, offset: 10 })
expect(page).toHaveLength(5)
expect(page.map((r) => (r.metadata as { seq: number }).seq)).toEqual([10, 11, 12, 13, 14])
})
})