perf: parallel + id-only canonical enumeration (heal-cost dominant term)
The canonical enumeration walk (getNounsWithPagination) hydrated each entity's vector + metadata ONE-AT-A-TIME inside the shard loop — every enumeration paid N x per-op-latency serially, and every index heal enumerates canonical, so this was the dominant multiplier in the measured multi-minute heals (cortex heal-cost decomposition). - Hydration is now 16-way bounded-concurrency (matching the wave cor's native rebuild uses): heal wall-clock becomes ~2xN/16 x per-op instead of N x per-op. Order, cursor resume (skipped nouns still never read), filters, peek/hasMore and totalCount are all preserved — pages are byte-identical to the serial walk. - New getNounIdsWithPagination(): the id-only opt-out for callers that own their IO schedule (an index heal). Unfiltered = ids straight from the shard paths, ZERO per-entity reads; filtered hydrates metadata only (16-way). Same cursor/offset/nextCursor contract, so it is page-compatible with the hydrating walk. Regression: pagination-parallel-hydration.test.ts pins paged==big-page identity, ids==items order, zero-read id-only, and filter parity. 103 storage tests green.
This commit is contained in:
parent
bfa1762107
commit
ec5b93339a
2 changed files with 250 additions and 18 deletions
94
tests/unit/storage/pagination-parallel-hydration.test.ts
Normal file
94
tests/unit/storage/pagination-parallel-hydration.test.ts
Normal file
|
|
@ -0,0 +1,94 @@
|
|||
/**
|
||||
* @module tests/unit/storage/pagination-parallel-hydration
|
||||
* @description The canonical enumeration walk (getNounsWithPagination) hydrated
|
||||
* each item's vector + metadata ONE-AT-A-TIME — N×per-op-latency serially, the
|
||||
* dominant term in an index heal (cortex heal-cost decomposition). It now
|
||||
* hydrates 16-way, and a new getNounIdsWithPagination returns ids WITHOUT
|
||||
* hydration (zero per-entity reads when unfiltered). Both must preserve the exact
|
||||
* pagination contract: same order, cursor continuation, filters, totalCount.
|
||||
*/
|
||||
import { describe, it, expect, beforeEach, vi } from 'vitest'
|
||||
import { Brainy, NounType } from '../../../src/index.js'
|
||||
|
||||
describe('paginated enumeration — parallel hydration + id-only (cortex heal-cost)', () => {
|
||||
let brain: any
|
||||
let storage: any
|
||||
const N = 30
|
||||
|
||||
beforeEach(async () => {
|
||||
process.env.BRAINY_DETERMINISTIC_EMBEDDINGS = 'true'
|
||||
brain = new Brainy({ requireSubtype: false, storage: { type: 'memory' }, dimensions: 384 })
|
||||
await brain.init()
|
||||
for (let i = 0; i < N; i++) {
|
||||
await brain.add({
|
||||
data: `n${i}`,
|
||||
type: i % 3 === 0 ? NounType.Task : NounType.Concept,
|
||||
metadata: { i }
|
||||
})
|
||||
}
|
||||
await brain.flush()
|
||||
storage = brain.storage
|
||||
})
|
||||
|
||||
/** Page the whole dataset through a small limit via cursor and collect ordered ids. */
|
||||
const pageAll = async (fn: (opts: any) => Promise<any>, key: 'items' | 'ids') => {
|
||||
const out: string[] = []
|
||||
let cursor: string | undefined
|
||||
for (let guard = 0; guard < 1000; guard++) {
|
||||
const page = await fn({ limit: 4, cursor })
|
||||
const batch = key === 'items' ? page.items.map((n: any) => n.id) : page.ids
|
||||
out.push(...batch)
|
||||
if (!page.hasMore) break
|
||||
cursor = page.nextCursor
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
it('parallel hydration yields the SAME ordered pages as one big page', async () => {
|
||||
const big = await storage.getNounsWithPagination({ limit: 1000, offset: 0 })
|
||||
const bigIds = big.items.map((n: any) => n.id)
|
||||
// At least the N we added (a brain also has its VFS root entity).
|
||||
expect(bigIds.length).toBeGreaterThanOrEqual(N)
|
||||
expect(big.totalCount).toBe(bigIds.length)
|
||||
|
||||
const paged = await pageAll((o) => storage.getNounsWithPagination(o), 'items')
|
||||
expect(paged).toEqual(bigIds) // identical order, no dupes, no gaps across pages
|
||||
})
|
||||
|
||||
it('getNounIdsWithPagination returns exactly the same ids, in the same order', async () => {
|
||||
const idsPaged = await pageAll((o) => storage.getNounIdsWithPagination(o), 'ids')
|
||||
const itemsPaged = await pageAll((o) => storage.getNounsWithPagination(o), 'items')
|
||||
expect(idsPaged).toEqual(itemsPaged)
|
||||
expect(new Set(idsPaged).size).toBe(idsPaged.length) // every id exactly once
|
||||
expect(idsPaged.length).toBeGreaterThanOrEqual(N)
|
||||
})
|
||||
|
||||
it('id-only enumeration does ZERO per-entity hydration reads when unfiltered', async () => {
|
||||
const readSpy = vi.spyOn(storage as any, 'readCanonicalObject')
|
||||
await storage.getNounIdsWithPagination({ limit: 1000, offset: 0 })
|
||||
expect(readSpy).not.toHaveBeenCalled()
|
||||
readSpy.mockRestore()
|
||||
|
||||
// The hydrating walk, by contrast, DOES read each entity.
|
||||
const readSpy2 = vi.spyOn(storage as any, 'readCanonicalObject')
|
||||
await storage.getNounsWithPagination({ limit: 1000, offset: 0 })
|
||||
expect(readSpy2.mock.calls.length).toBeGreaterThan(0)
|
||||
readSpy2.mockRestore()
|
||||
})
|
||||
|
||||
it('a type filter matches between the hydrating and id-only walks', async () => {
|
||||
const taskItems = await storage.getNounsWithPagination({
|
||||
limit: 1000,
|
||||
offset: 0,
|
||||
filter: { nounType: NounType.Task }
|
||||
})
|
||||
const taskIds = await storage.getNounIdsWithPagination({
|
||||
limit: 1000,
|
||||
offset: 0,
|
||||
filter: { nounType: NounType.Task }
|
||||
})
|
||||
const expected = Math.ceil(N / 3) // every 3rd is a Task
|
||||
expect(taskItems.items.length).toBe(expected)
|
||||
expect(new Set(taskIds.ids)).toEqual(new Set(taskItems.items.map((n: any) => n.id)))
|
||||
})
|
||||
})
|
||||
Loading…
Add table
Add a link
Reference in a new issue