David Snelling
450084b6ce
A filtered + ordered find() materialized the FULL sorted match set before slicing the page — passing k = filteredIds.length to the column store's filteredSortTopK. A broad filter + orderBy returning 20 rows at billion scale produced hundreds of millions of sorted ids to discard all but the page (O(matches) heap). Thread the page bound (offset + limit + the hidden-tier over-fetch) into getSortedIdsForFilter → filteredSortTopK / sortTopK / the sparse-path slice, so the sort produces only the page. Ordering and pagination are unchanged. From cor's 2026-06-24 co-release scaling audit (item 1). Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
60 lines
2.5 KiB
TypeScript
60 lines
2.5 KiB
TypeScript
/**
|
|
* @module tests/unit/brainy/find-orderby-pagek
|
|
* @description CTX-BR-FIND-ORDERBY item (1) — `find({ where, orderBy, limit })` must
|
|
* produce only the requested PAGE of sorted ids, not the full sorted match set. At
|
|
* billion scale a broad filter + orderBy returning 20 rows previously materialized
|
|
* every matching sorted id (O(matches) heap); the page bound (`offset+limit`) is now
|
|
* threaded into `getSortedIdsForFilter` → the column store's top-K heap. Ordering
|
|
* correctness is covered by orderby-sort-bug.test.ts; this locks the page bound.
|
|
*/
|
|
|
|
import { describe, it, expect, beforeEach, afterEach } from 'vitest'
|
|
import { Brainy } from '../../../src/index.js'
|
|
import { NounType } from '../../../src/types/graphTypes.js'
|
|
import { createTestConfig } from '../../helpers/test-factory.js'
|
|
|
|
describe('find({ where, orderBy }) bounds the sort to the page (CTX-BR-FIND-ORDERBY #1)', () => {
|
|
let brain: Brainy
|
|
const MATCHES = 50
|
|
|
|
beforeEach(async () => {
|
|
brain = new Brainy(createTestConfig())
|
|
await brain.init()
|
|
for (let i = 0; i < MATCHES; i++) {
|
|
await brain.add({ type: NounType.Document, data: `doc ${i}`, metadata: { bucket: 'x', seq: i } })
|
|
}
|
|
})
|
|
|
|
afterEach(async () => {
|
|
await brain.close()
|
|
})
|
|
|
|
it('threads the page bound (offset+limit) into getSortedIdsForFilter, not the full match count', async () => {
|
|
let capturedTopK: number | undefined
|
|
const real = (brain as any).metadataIndex.getSortedIdsForFilter.bind((brain as any).metadataIndex)
|
|
;(brain as any).metadataIndex.getSortedIdsForFilter = async (
|
|
f: any,
|
|
ob: string,
|
|
o: 'asc' | 'desc',
|
|
topK?: number
|
|
) => {
|
|
capturedTopK = topK
|
|
return real(f, ob, o, topK)
|
|
}
|
|
|
|
const results = await brain.find({ where: { bucket: 'x' }, orderBy: 'createdAt', order: 'desc', limit: 5 })
|
|
|
|
expect(results).toHaveLength(5)
|
|
// Page-bounded: ~ limit (5) + a small hidden-tier over-fetch — NOT all 50 matches.
|
|
expect(capturedTopK).toBeDefined()
|
|
expect(capturedTopK!).toBeGreaterThanOrEqual(5)
|
|
expect(capturedTopK!).toBeLessThan(MATCHES)
|
|
})
|
|
|
|
it('still returns the correct page with offset (sort + window unchanged)', async () => {
|
|
// seq is a numeric metadata field; orderBy seq asc → 0,1,2,… ; page [10,15).
|
|
const page = await brain.find({ where: { bucket: 'x' }, orderBy: 'seq', order: 'asc', limit: 5, offset: 10 })
|
|
expect(page).toHaveLength(5)
|
|
expect(page.map((r) => (r.metadata as { seq: number }).seq)).toEqual([10, 11, 12, 13, 14])
|
|
})
|
|
})
|