diff --git a/src/utils/metadataIndex.ts b/src/utils/metadataIndex.ts index f010560d..26e2999a 100644 --- a/src/utils/metadataIndex.ts +++ b/src/utils/metadataIndex.ts @@ -5,7 +5,8 @@ */ import { StorageAdapter, resolveEntityField, NounMetadata, VerbMetadata } from '../coreTypes.js' -import { SYSTEM_ENTITY_SCALARS, parseFieldAddress, UnresolvableFieldError } from '../db/fieldAddressing.js' +import { SYSTEM_ENTITY_SCALARS, parseFieldAddress, UnresolvableFieldError, type FieldAddress } from '../db/fieldAddressing.js' +import { splitNounMetadataRecord } from '../types/reservedFields.js' import { ColumnStore } from '../indexes/columnStore/ColumnStore.js' import type { MetadataIndexProvider } from '../plugin.js' import { MetadataIndexCache, MetadataIndexCacheConfig } from './metadataIndexCache.js' @@ -2207,6 +2208,98 @@ export class MetadataIndexManager implements MetadataIndexProvider { * @returns Promise - Entity IDs sorted by specified field * */ + /** + * Resolve the orderBy value for MANY entities in BATCHED metadata-record + * reads — the sort path's one sanctioned value source (BRAINY-PROD-LATENCY-TRIAD). + * + * THE ASYMPTOTIC LAW THIS ENFORCES: an ordered read never does per-row + * storage round-trips. The previous shape — `await getFieldValueForEntity` + * per id, each opening the VECTOR record serially — cost 62–98ms × N on a + * production filesystem brain: 3,224 rows took 199–317 SECONDS, silently. + * The metadata RECORD (smaller, cached, batch-readable) carries everything + * a sort can address: the ten system scalars top-level — EXACT values, no + * bucketing loss — and the user's bag (v2 nested or legacy flat, resolved + * through the shape-aware split). One batched read pass serves any N. + * + * The call-shape is pinned by tests (zero per-row reads, batch calls only) + * so the serial loop cannot quietly return. + * + * @param ids - Entity ids to resolve (any size; reads are chunk-batched). + * @param orderAddress - The parsed orderBy address (system or metadata scope). + * @returns id → value map; ids whose record is missing map to `undefined` + * (they sort LAST per the ordering contract — never dropped). + */ + private async resolveOrderValuesBatch( + ids: string[], + orderAddress: FieldAddress + ): Promise> { + const values = new Map() + if (ids.length === 0) return values + + // Batch door, best first: BaseStorage's getNounMetadataBatch (native + // batch or parallel reads inside), then the adapter-optional + // getMetadataBatch, then chunked-parallel single reads — NEVER serial. + const storage = this.storage as StorageAdapter & { + getNounMetadataBatch?(ids: string[]): Promise> + } + const CHUNK = 500 + const records = new Map() + for (let i = 0; i < ids.length; i += CHUNK) { + const chunk = ids.slice(i, i + CHUNK) + if (typeof storage.getNounMetadataBatch === 'function') { + const batch = await storage.getNounMetadataBatch(chunk) + for (const [id, rec] of batch) records.set(id, rec) + } else if (typeof storage.getMetadataBatch === 'function') { + const batch = await storage.getMetadataBatch(chunk) + for (const [id, rec] of batch) records.set(id, rec) + } else { + const loaded = await Promise.all( + chunk.map(async (id) => [id, await storage.getNounMetadata(id)] as const) + ) + for (const [id, rec] of loaded) if (rec) records.set(id, rec) + } + } + + for (const id of ids) { + const record = records.get(id) + if (!record) { + values.set(id, undefined) + continue + } + // Shape-aware split serves both record eras: engine scalars from the + // reserved half (EXACT timestamps — the bucketed index is never + // consulted here), user fields from the bag. + const { reserved, custom } = splitNounMetadataRecord( + record as Record + ) + if (orderAddress.scope === 'system') { + values.set( + id, + orderAddress.field === 'type' + ? reserved.noun + : (reserved as Record)[orderAddress.field] + ) + } else { + let value: unknown = custom[orderAddress.field] + if (value === undefined && orderAddress.field.includes('.')) { + // Dotted user path: traverse INSIDE the bag. + value = orderAddress.field + .split('.') + .reduce( + (o, seg) => + o && typeof o === 'object' ? (o as Record)[seg] : undefined, + custom + ) + } + values.set(id, value) + } + } + return values + } + + /** Once-per-field flag for the fallback-degradation announcement. */ + private static announcedFallbackSorts = new Set() + async getSortedIdsForFilter( filter: any, orderBy: string, @@ -2274,12 +2367,12 @@ export class MetadataIndexManager implements MetadataIndexProvider { // ORDERING CONTRACT (cross-engine, sealed): rows missing the field are // NEVER dropped — they sort LAST in both directions — and ties break by // id ascending. The column only contains rows that HAVE the field, so - // (1) re-sort the page deterministically (value, then id) with K cheap - // value reads, and (2) append the filtered rows the column omitted, - // id-ascending, filling any remaining page budget. - const page = await Promise.all( - sortedUuids.map(async id => ({ id, value: await this.getFieldValueForEntity(id, orderKey) })) - ) + // (1) re-sort the page deterministically (value, then id) via ONE + // batched value resolution — never per-row reads — and (2) append the + // filtered rows the column omitted, id-ascending, filling any + // remaining page budget. + const pageValues = await this.resolveOrderValuesBatch(sortedUuids, orderAddress) + const page = sortedUuids.map(id => ({ id, value: pageValues.get(id) })) page.sort((a, b) => this.compareAddressedValues(a.value, b.value, a.id, b.id, order)) let result = page.map(p => p.id) @@ -2293,20 +2386,32 @@ export class MetadataIndexManager implements MetadataIndexProvider { return topK !== undefined ? result.slice(0, topK) : result } - // Fallback: sparse index path (for fields not yet in column store). - // Requires a non-empty filter because it reads O(k) entity values from storage. + // Fallback: no column serves this field. BOUNDED + ANNOUNCED, never + // silent (the B2 no-silent-degradation law, BRAINY-PROD-LATENCY-TRIAD): + // O(N) in row count but served by BATCHED metadata-record reads — the + // serial per-row getNoun loop that turned 3,224 rows into a 199–317s + // scan is dead, and the call-shape pin keeps it dead. const filteredIds = await this.getIdsForFilter(filter) if (filteredIds.length === 0) { return [] } - const idValuePairs: Array<{ id: string, value: any }> = [] - for (const id of filteredIds) { - const value = await this.getFieldValueForEntity(id, orderKey) - idValuePairs.push({ id, value }) + if ( + filteredIds.length > 500 && + !MetadataIndexManager.announcedFallbackSorts.has(orderKey) + ) { + MetadataIndexManager.announcedFallbackSorts.add(orderKey) + prodLog.warn( + `[brainy] ordered read on '${orderKey}' has no column index — served by the ` + + `batched fallback over ${filteredIds.length} rows (bounded, one batch pass; ` + + `announced once per field). A native column for this field makes it O(K).` + ) } + const fallbackValues = await this.resolveOrderValuesBatch(filteredIds, orderAddress) + const idValuePairs = filteredIds.map(id => ({ id, value: fallbackValues.get(id) })) + idValuePairs.sort((a, b) => this.compareAddressedValues(a.value, b.value, a.id, b.id, order)) const sorted = idValuePairs.map(p => p.id) diff --git a/tests/unit/utils/metadataIndex-sort-callshape.test.ts b/tests/unit/utils/metadataIndex-sort-callshape.test.ts new file mode 100644 index 00000000..ffe89566 --- /dev/null +++ b/tests/unit/utils/metadataIndex-sort-callshape.test.ts @@ -0,0 +1,119 @@ +/** + * @module tests/unit/utils/metadataIndex-sort-callshape + * @description THE ASYMPTOTIC CALL-SHAPE PIN for ordered reads + * (BRAINY-PROD-LATENCY-TRIAD, David-approved plan Track A1). The defect it + * keeps dead: `getSortedIdsForFilter`'s value resolution did a SERIAL + * `storage.getNoun()` (the heavyweight VECTOR record) per filtered row — + * 62–98ms × 3,224 rows = the measured 199–317 SECOND production sort, with + * `topK` applied only after the full scan. These pins assert the SHAPE of + * the storage traffic, not wall-clock (latency-blind, so they hold on any + * machine): an ordered read performs ZERO per-row vector-record reads and + * resolves sort values through BATCHED metadata-record calls only. + */ +import { describe, it, expect, beforeAll, afterAll, vi } from 'vitest' +import { Brainy } from '../../../src/index.js' +import { NounType } from '../../../src/types/graphTypes.js' + +const ROWS = 60 + +describe('ordered reads — the batched call-shape law (no per-row storage loops)', () => { + let brain: Brainy + let storage: { + getNoun: (id: string) => Promise + getNounMetadata: (id: string) => Promise + getNounMetadataBatch: (ids: string[]) => Promise> + } + + beforeAll(async () => { + brain = new Brainy({ storage: { type: 'memory' }, requireSubtype: false }) + await brain.init() + for (let i = 0; i < ROWS; i++) { + await brain.add({ + data: `row ${i}`, + type: NounType.Document, + metadata: { rank: (i * 7) % ROWS, plain: `p${i}` } + }) + } + storage = (brain as unknown as { storage: typeof storage }).storage + }, 120000) + + afterAll(async () => { + await brain.close().catch(() => {}) + }) + + it('user-field orderBy: zero vector-record reads, zero serial metadata reads — batch calls only', async () => { + const getNounSpy = vi.spyOn(storage, 'getNoun') + const singleReadSpy = vi.spyOn(storage, 'getNounMetadata') + const batchSpy = vi.spyOn(storage, 'getNounMetadataBatch') + + const rows = await brain.find({ + type: NounType.Document, + orderBy: 'rank', + order: 'desc', + limit: 10 + }) + expect(rows.length).toBe(10) + expect((rows[0].metadata as Record).rank).toBe(ROWS - 1) + + // THE PIN: the sort's value resolution never opens a vector record and + // never falls into a per-row metadata loop. (Result hydration after + // pagination is allowed to read; the SORT itself must be batch-only — + // hence the ceiling: strictly fewer single reads than sorted rows.) + expect(getNounSpy.mock.calls.length, 'per-row vector-record reads in an ordered read').toBe(0) + expect(batchSpy.mock.calls.length, 'the batch door was used').toBeGreaterThanOrEqual(1) + expect( + singleReadSpy.mock.calls.length, + 'serial per-row metadata reads (the 199s shape)' + ).toBeLessThan(ROWS / 2) + + vi.restoreAllMocks() + }) + + it('system.createdAt orderBy: exact values from batched records — the bucketed index is never a per-row disk excuse', async () => { + const getNounSpy = vi.spyOn(storage, 'getNoun') + const batchSpy = vi.spyOn(storage, 'getNounMetadataBatch') + + const rows = await brain.find({ + type: NounType.Document, + orderBy: 'system.createdAt', + order: 'asc', + limit: 15 + }) + expect(rows.length).toBe(15) + + expect(getNounSpy.mock.calls.length, 'per-row vector-record reads').toBe(0) + expect(batchSpy.mock.calls.length).toBeGreaterThanOrEqual(1) + + // Exactness: ascending createdAt must be non-decreasing with full + // millisecond precision (the old path sorted minute-BUCKETED values or + // paid a per-row disk read for exact ones — both are dead). Find results + // carry the timestamps on the nested full entity. + const stamps = rows.map( + (r) => ((r as unknown as { entity?: { createdAt?: number } }).entity?.createdAt ?? + (r as unknown as { createdAt?: number }).createdAt) as number + ) + for (let i = 1; i < stamps.length; i++) { + expect(stamps[i]).toBeGreaterThanOrEqual(stamps[i - 1]) + } + + vi.restoreAllMocks() + }) + + it('the ordering contract survives the batch path: missing values LAST both directions, ties by id asc, rows never dropped', async () => { + // Three rows lack `rank`? No — all carry it; add two rows WITHOUT it. + const a = await brain.add({ data: 'no-rank a', type: NounType.Document, metadata: { plain: 'x' } }) + const b = await brain.add({ data: 'no-rank b', type: NounType.Document, metadata: { plain: 'y' } }) + + for (const order of ['asc', 'desc'] as const) { + const rows = await brain.find({ + type: NounType.Document, + orderBy: 'rank', + order, + limit: ROWS + 10 + }) + expect(rows.length, `complete result (${order})`).toBe(ROWS + 2) + const lastTwo = rows.slice(-2).map((r) => r.id).sort() + expect(lastTwo, `missing-value rows sort LAST (${order})`).toEqual([a, b].sort()) + } + }) +})