perf(8.0): bound find({ where, orderBy }) sort to the page (CTX-BR-FIND-ORDERBY)
A filtered + ordered find() materialized the FULL sorted match set before slicing the page — passing k = filteredIds.length to the column store's filteredSortTopK. A broad filter + orderBy returning 20 rows at billion scale produced hundreds of millions of sorted ids to discard all but the page (O(matches) heap). Thread the page bound (offset + limit + the hidden-tier over-fetch) into getSortedIdsForFilter → filteredSortTopK / sortTopK / the sparse-path slice, so the sort produces only the page. Ordering and pagination are unchanged. From cor's 2026-06-24 co-release scaling audit (item 1). Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
parent
82dde92077
commit
450084b6ce
5 changed files with 93 additions and 10 deletions
60
tests/unit/brainy/find-orderby-pagek.test.ts
Normal file
60
tests/unit/brainy/find-orderby-pagek.test.ts
Normal file
|
|
@ -0,0 +1,60 @@
|
|||
/**
|
||||
* @module tests/unit/brainy/find-orderby-pagek
|
||||
* @description CTX-BR-FIND-ORDERBY item (1) — `find({ where, orderBy, limit })` must
|
||||
* produce only the requested PAGE of sorted ids, not the full sorted match set. At
|
||||
* billion scale a broad filter + orderBy returning 20 rows previously materialized
|
||||
* every matching sorted id (O(matches) heap); the page bound (`offset+limit`) is now
|
||||
* threaded into `getSortedIdsForFilter` → the column store's top-K heap. Ordering
|
||||
* correctness is covered by orderby-sort-bug.test.ts; this locks the page bound.
|
||||
*/
|
||||
|
||||
import { describe, it, expect, beforeEach, afterEach } from 'vitest'
|
||||
import { Brainy } from '../../../src/index.js'
|
||||
import { NounType } from '../../../src/types/graphTypes.js'
|
||||
import { createTestConfig } from '../../helpers/test-factory.js'
|
||||
|
||||
describe('find({ where, orderBy }) bounds the sort to the page (CTX-BR-FIND-ORDERBY #1)', () => {
|
||||
let brain: Brainy
|
||||
const MATCHES = 50
|
||||
|
||||
beforeEach(async () => {
|
||||
brain = new Brainy(createTestConfig())
|
||||
await brain.init()
|
||||
for (let i = 0; i < MATCHES; i++) {
|
||||
await brain.add({ type: NounType.Document, data: `doc ${i}`, metadata: { bucket: 'x', seq: i } })
|
||||
}
|
||||
})
|
||||
|
||||
afterEach(async () => {
|
||||
await brain.close()
|
||||
})
|
||||
|
||||
it('threads the page bound (offset+limit) into getSortedIdsForFilter, not the full match count', async () => {
|
||||
let capturedTopK: number | undefined
|
||||
const real = (brain as any).metadataIndex.getSortedIdsForFilter.bind((brain as any).metadataIndex)
|
||||
;(brain as any).metadataIndex.getSortedIdsForFilter = async (
|
||||
f: any,
|
||||
ob: string,
|
||||
o: 'asc' | 'desc',
|
||||
topK?: number
|
||||
) => {
|
||||
capturedTopK = topK
|
||||
return real(f, ob, o, topK)
|
||||
}
|
||||
|
||||
const results = await brain.find({ where: { bucket: 'x' }, orderBy: 'createdAt', order: 'desc', limit: 5 })
|
||||
|
||||
expect(results).toHaveLength(5)
|
||||
// Page-bounded: ~ limit (5) + a small hidden-tier over-fetch — NOT all 50 matches.
|
||||
expect(capturedTopK).toBeDefined()
|
||||
expect(capturedTopK!).toBeGreaterThanOrEqual(5)
|
||||
expect(capturedTopK!).toBeLessThan(MATCHES)
|
||||
})
|
||||
|
||||
it('still returns the correct page with offset (sort + window unchanged)', async () => {
|
||||
// seq is a numeric metadata field; orderBy seq asc → 0,1,2,… ; page [10,15).
|
||||
const page = await brain.find({ where: { bucket: 'x' }, orderBy: 'seq', order: 'asc', limit: 5, offset: 10 })
|
||||
expect(page).toHaveLength(5)
|
||||
expect(page.map((r) => (r.metadata as { seq: number }).seq)).toEqual([10, 11, 12, 13, 14])
|
||||
})
|
||||
})
|
||||
Loading…
Add table
Add a link
Reference in a new issue