diff --git a/RELEASES.md b/RELEASES.md index 226d6580..06a07242 100644 --- a/RELEASES.md +++ b/RELEASES.md @@ -73,6 +73,11 @@ no separate migration doc. **`8.0.0-rc.2` is live** — install with > matches that DO pass the filter, rather than coming back empty. No API change. With the native > `@soulcraft/cor` 3.0 stack the matched universe is forwarded as an opaque roaring buffer with > zero id materialization in TypeScript; the pure-JS path restricts the beam walk with a string set. +> - **`find({ where, orderBy })` bounds the sort to the page.** A broad filter + `orderBy` +> returning one page no longer materializes every matching sorted id (it produced the full sorted +> match set, then sliced) — the page bound (`offset + limit`) is threaded into the column store's +> top-K sort, so returning 20 rows from a million matches stays a bounded-K heap. No API change; +> ordering and pagination are unchanged. > - **`brain.graph.subgraph()` accepts a query (query→expand).** The seed selector now takes not > just entity id(s) but a `find()` result or a `FindParams` query — `brain.graph.subgraph({ where: > { team: 'platform' } }, { depth: 1 })` runs the query and expands the neighborhood of every diff --git a/src/brainy.ts b/src/brainy.ts index a90e4094..d7805c93 100644 --- a/src/brainy.ts +++ b/src/brainy.ts @@ -4763,11 +4763,15 @@ export class Brainy implements BrainyInterface { // Apply sorting if requested, otherwise just filter let filteredIds: string[] if (params.orderBy) { - // Get sorted IDs using production-scale sorted filtering + // Get sorted IDs using production-scale sorted filtering. Bound the sort to + // the page (offset+limit) plus the hidden over-fetch — so a broad filter + + // orderBy returning 20 rows doesn't materialize every matching sorted id. + const sortTopK = (params.offset || 0) + (params.limit || 10) + hiddenIds.size filteredIds = await this.metadataIndex.getSortedIdsForFilter( filter, params.orderBy, - params.order || 'asc' + params.order || 'asc', + sortTopK ) } else { // Just filter without sorting @@ -5049,15 +5053,18 @@ export class Brainy implements BrainyInterface { if (!params.query && !params.connected) { // Apply sorting if requested for metadata-only queries if (params.orderBy) { + // Page-bounded sort (offset+limit + hidden over-fetch) so we don't + // materialize every matching sorted id to return one page. + const limit = params.limit || 10 + const offset = params.offset || 0 const sortedIds = await this.metadataIndex.getSortedIdsForFilter( preResolvedFilter, params.orderBy, - params.order || 'asc' + params.order || 'asc', + offset + limit + hiddenIds.size ) // Paginate sorted IDs BEFORE loading entities (production-scale!) - const limit = params.limit || 10 - const offset = params.offset || 0 const pageIds = sortedIds.slice(offset, offset + limit) // Batch-load entities for paginated results (10x faster on GCS) diff --git a/src/plugin.ts b/src/plugin.ts index ae0ed3f0..b9b81f74 100644 --- a/src/plugin.ts +++ b/src/plugin.ts @@ -147,7 +147,7 @@ export interface MetadataIndexProvider { */ getIdSetForFilter?(filter: any): Promise getIdsForTextQuery(query: string): Promise> - getSortedIdsForFilter(filter: any, orderBy: string, order?: 'asc' | 'desc'): Promise + getSortedIdsForFilter(filter: any, orderBy: string, order?: 'asc' | 'desc', topK?: number): Promise getFilterValues(field: string): Promise getFilterFields(): Promise getFieldValueForEntity(entityId: string, field: string): Promise diff --git a/src/utils/metadataIndex.ts b/src/utils/metadataIndex.ts index 94fa4906..31fca200 100644 --- a/src/utils/metadataIndex.ts +++ b/src/utils/metadataIndex.ts @@ -2121,13 +2121,19 @@ export class MetadataIndexManager implements MetadataIndexProvider { * @param filter - Metadata filter criteria (uses roaring bitmaps) * @param orderBy - Field name to sort by (e.g., 'createdAt', 'title') * @param order - Sort direction: 'asc' (default) or 'desc' + * @param topK - Optional page bound: produce only the top `K` sorted ids + * (`offset + limit`) instead of the full sorted match set. A broad filter + + * orderBy that returns one page no longer materializes hundreds of millions of + * sorted ids at billion scale — the column store's top-K heap produces only the + * page. Omit for the full sorted set. * @returns Promise - Entity IDs sorted by specified field * */ async getSortedIdsForFilter( filter: any, orderBy: string, - order: 'asc' | 'desc' = 'asc' + order: 'asc' | 'desc' = 'asc', + topK?: number ): Promise { // Column store path: O(K log S) sort via k-way merge across segments. // No per-entity storage reads, no precision loss from bucketing. @@ -2146,13 +2152,17 @@ export class MetadataIndexManager implements MetadataIndexProvider { const intId = this.idMapper.getInt(id) if (intId !== undefined) filterBitmap.add(intId) } + // Page-bounded: produce only the top `topK` (offset+limit), not every + // match, so a broad filter + orderBy returning one page stays O(matches + // log K) heap, not a full sort materialization. + const k = topK !== undefined ? Math.min(topK, filteredIds.length) : filteredIds.length sortedIntIds = await this.columnStore.filteredSortTopK( - filterBitmap, orderBy, order, filteredIds.length + filterBitmap, orderBy, order, k ) } else { // Unfiltered sort — column store handles the full entity set efficiently sortedIntIds = await this.columnStore.sortTopK( - orderBy, order, this.idMapper.size + orderBy, order, topK !== undefined ? Math.min(topK, this.idMapper.size) : this.idMapper.size ) } @@ -2195,7 +2205,8 @@ export class MetadataIndexManager implements MetadataIndexProvider { return order === 'asc' ? comparison : -comparison }) - return idValuePairs.map(p => p.id) + const sorted = idValuePairs.map(p => p.id) + return topK !== undefined ? sorted.slice(0, topK) : sorted } /** diff --git a/tests/unit/brainy/find-orderby-pagek.test.ts b/tests/unit/brainy/find-orderby-pagek.test.ts new file mode 100644 index 00000000..9a453f8d --- /dev/null +++ b/tests/unit/brainy/find-orderby-pagek.test.ts @@ -0,0 +1,60 @@ +/** + * @module tests/unit/brainy/find-orderby-pagek + * @description CTX-BR-FIND-ORDERBY item (1) — `find({ where, orderBy, limit })` must + * produce only the requested PAGE of sorted ids, not the full sorted match set. At + * billion scale a broad filter + orderBy returning 20 rows previously materialized + * every matching sorted id (O(matches) heap); the page bound (`offset+limit`) is now + * threaded into `getSortedIdsForFilter` → the column store's top-K heap. Ordering + * correctness is covered by orderby-sort-bug.test.ts; this locks the page bound. + */ + +import { describe, it, expect, beforeEach, afterEach } from 'vitest' +import { Brainy } from '../../../src/index.js' +import { NounType } from '../../../src/types/graphTypes.js' +import { createTestConfig } from '../../helpers/test-factory.js' + +describe('find({ where, orderBy }) bounds the sort to the page (CTX-BR-FIND-ORDERBY #1)', () => { + let brain: Brainy + const MATCHES = 50 + + beforeEach(async () => { + brain = new Brainy(createTestConfig()) + await brain.init() + for (let i = 0; i < MATCHES; i++) { + await brain.add({ type: NounType.Document, data: `doc ${i}`, metadata: { bucket: 'x', seq: i } }) + } + }) + + afterEach(async () => { + await brain.close() + }) + + it('threads the page bound (offset+limit) into getSortedIdsForFilter, not the full match count', async () => { + let capturedTopK: number | undefined + const real = (brain as any).metadataIndex.getSortedIdsForFilter.bind((brain as any).metadataIndex) + ;(brain as any).metadataIndex.getSortedIdsForFilter = async ( + f: any, + ob: string, + o: 'asc' | 'desc', + topK?: number + ) => { + capturedTopK = topK + return real(f, ob, o, topK) + } + + const results = await brain.find({ where: { bucket: 'x' }, orderBy: 'createdAt', order: 'desc', limit: 5 }) + + expect(results).toHaveLength(5) + // Page-bounded: ~ limit (5) + a small hidden-tier over-fetch — NOT all 50 matches. + expect(capturedTopK).toBeDefined() + expect(capturedTopK!).toBeGreaterThanOrEqual(5) + expect(capturedTopK!).toBeLessThan(MATCHES) + }) + + it('still returns the correct page with offset (sort + window unchanged)', async () => { + // seq is a numeric metadata field; orderBy seq asc → 0,1,2,… ; page [10,15). + const page = await brain.find({ where: { bucket: 'x' }, orderBy: 'seq', order: 'asc', limit: 5, offset: 10 }) + expect(page).toHaveLength(5) + expect(page.map((r) => (r.metadata as { seq: number }).seq)).toEqual([10, 11, 12, 13, 14]) + }) +})