perf: parallel + id-only canonical enumeration (heal-cost dominant term)

The canonical enumeration walk (getNounsWithPagination) hydrated each entity's
vector + metadata ONE-AT-A-TIME inside the shard loop — every enumeration paid
N x per-op-latency serially, and every index heal enumerates canonical, so this
was the dominant multiplier in the measured multi-minute heals (cortex heal-cost
decomposition).

- Hydration is now 16-way bounded-concurrency (matching the wave cor's native
  rebuild uses): heal wall-clock becomes ~2xN/16 x per-op instead of N x per-op.
  Order, cursor resume (skipped nouns still never read), filters, peek/hasMore
  and totalCount are all preserved — pages are byte-identical to the serial walk.
- New getNounIdsWithPagination(): the id-only opt-out for callers that own their
  IO schedule (an index heal). Unfiltered = ids straight from the shard paths,
  ZERO per-entity reads; filtered hydrates metadata only (16-way). Same
  cursor/offset/nextCursor contract, so it is page-compatible with the hydrating
  walk.

Regression: pagination-parallel-hydration.test.ts pins paged==big-page identity,
ids==items order, zero-read id-only, and filter parity. 103 storage tests green.
This commit is contained in:
David Snelling 2026-07-13 15:15:01 -07:00
parent bfa1762107
commit ec5b93339a
2 changed files with 250 additions and 18 deletions

View file

@ -0,0 +1,94 @@
/**
* @module tests/unit/storage/pagination-parallel-hydration
* @description The canonical enumeration walk (getNounsWithPagination) hydrated
* each item's vector + metadata ONE-AT-A-TIME N×per-op-latency serially, the
* dominant term in an index heal (cortex heal-cost decomposition). It now
* hydrates 16-way, and a new getNounIdsWithPagination returns ids WITHOUT
* hydration (zero per-entity reads when unfiltered). Both must preserve the exact
* pagination contract: same order, cursor continuation, filters, totalCount.
*/
import { describe, it, expect, beforeEach, vi } from 'vitest'
import { Brainy, NounType } from '../../../src/index.js'
describe('paginated enumeration — parallel hydration + id-only (cortex heal-cost)', () => {
let brain: any
let storage: any
const N = 30
beforeEach(async () => {
process.env.BRAINY_DETERMINISTIC_EMBEDDINGS = 'true'
brain = new Brainy({ requireSubtype: false, storage: { type: 'memory' }, dimensions: 384 })
await brain.init()
for (let i = 0; i < N; i++) {
await brain.add({
data: `n${i}`,
type: i % 3 === 0 ? NounType.Task : NounType.Concept,
metadata: { i }
})
}
await brain.flush()
storage = brain.storage
})
/** Page the whole dataset through a small limit via cursor and collect ordered ids. */
const pageAll = async (fn: (opts: any) => Promise<any>, key: 'items' | 'ids') => {
const out: string[] = []
let cursor: string | undefined
for (let guard = 0; guard < 1000; guard++) {
const page = await fn({ limit: 4, cursor })
const batch = key === 'items' ? page.items.map((n: any) => n.id) : page.ids
out.push(...batch)
if (!page.hasMore) break
cursor = page.nextCursor
}
return out
}
it('parallel hydration yields the SAME ordered pages as one big page', async () => {
const big = await storage.getNounsWithPagination({ limit: 1000, offset: 0 })
const bigIds = big.items.map((n: any) => n.id)
// At least the N we added (a brain also has its VFS root entity).
expect(bigIds.length).toBeGreaterThanOrEqual(N)
expect(big.totalCount).toBe(bigIds.length)
const paged = await pageAll((o) => storage.getNounsWithPagination(o), 'items')
expect(paged).toEqual(bigIds) // identical order, no dupes, no gaps across pages
})
it('getNounIdsWithPagination returns exactly the same ids, in the same order', async () => {
const idsPaged = await pageAll((o) => storage.getNounIdsWithPagination(o), 'ids')
const itemsPaged = await pageAll((o) => storage.getNounsWithPagination(o), 'items')
expect(idsPaged).toEqual(itemsPaged)
expect(new Set(idsPaged).size).toBe(idsPaged.length) // every id exactly once
expect(idsPaged.length).toBeGreaterThanOrEqual(N)
})
it('id-only enumeration does ZERO per-entity hydration reads when unfiltered', async () => {
const readSpy = vi.spyOn(storage as any, 'readCanonicalObject')
await storage.getNounIdsWithPagination({ limit: 1000, offset: 0 })
expect(readSpy).not.toHaveBeenCalled()
readSpy.mockRestore()
// The hydrating walk, by contrast, DOES read each entity.
const readSpy2 = vi.spyOn(storage as any, 'readCanonicalObject')
await storage.getNounsWithPagination({ limit: 1000, offset: 0 })
expect(readSpy2.mock.calls.length).toBeGreaterThan(0)
readSpy2.mockRestore()
})
it('a type filter matches between the hydrating and id-only walks', async () => {
const taskItems = await storage.getNounsWithPagination({
limit: 1000,
offset: 0,
filter: { nounType: NounType.Task }
})
const taskIds = await storage.getNounIdsWithPagination({
limit: 1000,
offset: 0,
filter: { nounType: NounType.Task }
})
const expected = Math.ceil(N / 3) // every 3rd is a Task
expect(taskItems.items.length).toBe(expected)
expect(new Set(taskIds.ids)).toEqual(new Set(taskItems.items.map((n: any) => n.id)))
})
})