David Snelling
ec5b93339a
The canonical enumeration walk (getNounsWithPagination) hydrated each entity's vector + metadata ONE-AT-A-TIME inside the shard loop — every enumeration paid N x per-op-latency serially, and every index heal enumerates canonical, so this was the dominant multiplier in the measured multi-minute heals (cortex heal-cost decomposition). - Hydration is now 16-way bounded-concurrency (matching the wave cor's native rebuild uses): heal wall-clock becomes ~2xN/16 x per-op instead of N x per-op. Order, cursor resume (skipped nouns still never read), filters, peek/hasMore and totalCount are all preserved — pages are byte-identical to the serial walk. - New getNounIdsWithPagination(): the id-only opt-out for callers that own their IO schedule (an index heal). Unfiltered = ids straight from the shard paths, ZERO per-entity reads; filtered hydrates metadata only (16-way). Same cursor/offset/nextCursor contract, so it is page-compatible with the hydrating walk. Regression: pagination-parallel-hydration.test.ts pins paged==big-page identity, ids==items order, zero-read id-only, and filter parity. 103 storage tests green.
94 lines
3.9 KiB
TypeScript
94 lines
3.9 KiB
TypeScript
/**
|
||
* @module tests/unit/storage/pagination-parallel-hydration
|
||
* @description The canonical enumeration walk (getNounsWithPagination) hydrated
|
||
* each item's vector + metadata ONE-AT-A-TIME — N×per-op-latency serially, the
|
||
* dominant term in an index heal (cortex heal-cost decomposition). It now
|
||
* hydrates 16-way, and a new getNounIdsWithPagination returns ids WITHOUT
|
||
* hydration (zero per-entity reads when unfiltered). Both must preserve the exact
|
||
* pagination contract: same order, cursor continuation, filters, totalCount.
|
||
*/
|
||
import { describe, it, expect, beforeEach, vi } from 'vitest'
|
||
import { Brainy, NounType } from '../../../src/index.js'
|
||
|
||
describe('paginated enumeration — parallel hydration + id-only (cortex heal-cost)', () => {
|
||
let brain: any
|
||
let storage: any
|
||
const N = 30
|
||
|
||
beforeEach(async () => {
|
||
process.env.BRAINY_DETERMINISTIC_EMBEDDINGS = 'true'
|
||
brain = new Brainy({ requireSubtype: false, storage: { type: 'memory' }, dimensions: 384 })
|
||
await brain.init()
|
||
for (let i = 0; i < N; i++) {
|
||
await brain.add({
|
||
data: `n${i}`,
|
||
type: i % 3 === 0 ? NounType.Task : NounType.Concept,
|
||
metadata: { i }
|
||
})
|
||
}
|
||
await brain.flush()
|
||
storage = brain.storage
|
||
})
|
||
|
||
/** Page the whole dataset through a small limit via cursor and collect ordered ids. */
|
||
const pageAll = async (fn: (opts: any) => Promise<any>, key: 'items' | 'ids') => {
|
||
const out: string[] = []
|
||
let cursor: string | undefined
|
||
for (let guard = 0; guard < 1000; guard++) {
|
||
const page = await fn({ limit: 4, cursor })
|
||
const batch = key === 'items' ? page.items.map((n: any) => n.id) : page.ids
|
||
out.push(...batch)
|
||
if (!page.hasMore) break
|
||
cursor = page.nextCursor
|
||
}
|
||
return out
|
||
}
|
||
|
||
it('parallel hydration yields the SAME ordered pages as one big page', async () => {
|
||
const big = await storage.getNounsWithPagination({ limit: 1000, offset: 0 })
|
||
const bigIds = big.items.map((n: any) => n.id)
|
||
// At least the N we added (a brain also has its VFS root entity).
|
||
expect(bigIds.length).toBeGreaterThanOrEqual(N)
|
||
expect(big.totalCount).toBe(bigIds.length)
|
||
|
||
const paged = await pageAll((o) => storage.getNounsWithPagination(o), 'items')
|
||
expect(paged).toEqual(bigIds) // identical order, no dupes, no gaps across pages
|
||
})
|
||
|
||
it('getNounIdsWithPagination returns exactly the same ids, in the same order', async () => {
|
||
const idsPaged = await pageAll((o) => storage.getNounIdsWithPagination(o), 'ids')
|
||
const itemsPaged = await pageAll((o) => storage.getNounsWithPagination(o), 'items')
|
||
expect(idsPaged).toEqual(itemsPaged)
|
||
expect(new Set(idsPaged).size).toBe(idsPaged.length) // every id exactly once
|
||
expect(idsPaged.length).toBeGreaterThanOrEqual(N)
|
||
})
|
||
|
||
it('id-only enumeration does ZERO per-entity hydration reads when unfiltered', async () => {
|
||
const readSpy = vi.spyOn(storage as any, 'readCanonicalObject')
|
||
await storage.getNounIdsWithPagination({ limit: 1000, offset: 0 })
|
||
expect(readSpy).not.toHaveBeenCalled()
|
||
readSpy.mockRestore()
|
||
|
||
// The hydrating walk, by contrast, DOES read each entity.
|
||
const readSpy2 = vi.spyOn(storage as any, 'readCanonicalObject')
|
||
await storage.getNounsWithPagination({ limit: 1000, offset: 0 })
|
||
expect(readSpy2.mock.calls.length).toBeGreaterThan(0)
|
||
readSpy2.mockRestore()
|
||
})
|
||
|
||
it('a type filter matches between the hydrating and id-only walks', async () => {
|
||
const taskItems = await storage.getNounsWithPagination({
|
||
limit: 1000,
|
||
offset: 0,
|
||
filter: { nounType: NounType.Task }
|
||
})
|
||
const taskIds = await storage.getNounIdsWithPagination({
|
||
limit: 1000,
|
||
offset: 0,
|
||
filter: { nounType: NounType.Task }
|
||
})
|
||
const expected = Math.ceil(N / 3) // every 3rd is a Task
|
||
expect(taskItems.items.length).toBe(expected)
|
||
expect(new Set(taskIds.ids)).toEqual(new Set(taskItems.items.map((n: any) => n.id)))
|
||
})
|
||
})
|