perf(8.0): cursor pagination for the verb walk — full edge pagination O(N²) → O(N)
getVerbsWithPagination was offset-only: it re-scanned from shard 0 on every page and early-terminated at offset+limit, so a full edge walk was O(N²) (a consumer measured ~19k edges → 27s paging at 900/page). It now accepts an opaque resume token (cv1:<shard>:<verbId>) that resumes the shard walk immediately AFTER the last returned verb — O(N) total at any page size, no scale cliff. - Stable within-shard order (sort by verb id); ids come from the path so verbs skipped by the cursor are never read. - Cursor supersedes offset when present; offset path preserved for back-compat, and offset callers now get a usable nextCursor so they can switch to cursor paging. Foreign/malformed tokens decode to null → offset fallback (no throw). - The relationships stream generator now uses cursor (was offset → O(N²)). - Parity with the noun side (getNounsWithPagination already cursored). Test: tests/unit/storage/verb-cursor-pagination.test.ts — a full cursored walk visits every edge exactly once (no dup, no skip), matches the offset walk's set, and a foreign cursor falls back gracefully.
This commit is contained in:
parent
a914313e3a
commit
682e786cc3
3 changed files with 221 additions and 41 deletions
103
tests/unit/storage/verb-cursor-pagination.test.ts
Normal file
103
tests/unit/storage/verb-cursor-pagination.test.ts
Normal file
|
|
@ -0,0 +1,103 @@
|
|||
/**
|
||||
* @module tests/unit/storage/verb-cursor-pagination
|
||||
* @description Graph-perf #2 (8.0): cursor pagination over the verb shard walk.
|
||||
*
|
||||
* `getVerbsWithPagination` used to be offset-only — it re-scanned from shard 0 on
|
||||
* every page and early-terminated at `offset+limit`, so a full edge walk was O(N²)
|
||||
* (a consumer measured 19k edges → 27s paging at 900/page). It now accepts an
|
||||
* opaque cursor that resumes immediately after the last returned verb, making a
|
||||
* full walk O(N) at any page size. These tests pin the correctness guarantees a
|
||||
* cursor MUST provide: every item exactly once, no duplicates, no skips, and the
|
||||
* same set as an offset walk — plus graceful fallback for a foreign token.
|
||||
*/
|
||||
|
||||
import { describe, it, expect, beforeEach, afterEach } from 'vitest'
|
||||
import { Brainy } from '../../../src/index.js'
|
||||
import { NounType, VerbType } from '../../../src/types/graphTypes.js'
|
||||
import { createTestConfig } from '../../helpers/test-factory.js'
|
||||
|
||||
describe('verb cursor pagination (graph-perf #2)', () => {
|
||||
let brain: Brainy
|
||||
// The storage layer is where the cursor lives; exercise the primitive directly.
|
||||
let storage: {
|
||||
getVerbs(opts: {
|
||||
pagination?: { offset?: number; limit?: number; cursor?: string }
|
||||
}): Promise<{ items: Array<{ id: string }>; hasMore: boolean; nextCursor?: string }>
|
||||
}
|
||||
const EDGE_COUNT = 40
|
||||
|
||||
beforeEach(async () => {
|
||||
brain = new Brainy(createTestConfig())
|
||||
await brain.init()
|
||||
const ids: string[] = []
|
||||
for (let i = 0; i <= EDGE_COUNT; i++) {
|
||||
ids.push(await brain.add({ type: NounType.Person, subtype: 'employee', data: `N${i}` }))
|
||||
}
|
||||
// Hub-and-spoke: EDGE_COUNT edges from one node, spread across id-hash shards.
|
||||
for (let i = 1; i <= EDGE_COUNT; i++) {
|
||||
await brain.relate({ from: ids[0], to: ids[i], type: VerbType.RelatedTo, subtype: 'colleague' })
|
||||
}
|
||||
storage = (brain as unknown as { storage: typeof storage }).storage
|
||||
})
|
||||
|
||||
afterEach(async () => {
|
||||
await brain.close()
|
||||
})
|
||||
|
||||
it('a full cursored walk visits every edge exactly once (no dup, no skip)', async () => {
|
||||
const all = await storage.getVerbs({ pagination: { limit: 10000 } })
|
||||
const refIds = new Set(all.items.map((v) => v.id))
|
||||
expect(refIds.size).toBe(EDGE_COUNT)
|
||||
|
||||
const seen: string[] = []
|
||||
let cursor: string | undefined
|
||||
let pages = 0
|
||||
for (;;) {
|
||||
const page = await storage.getVerbs({
|
||||
pagination: cursor ? { limit: 7, cursor } : { limit: 7 }
|
||||
})
|
||||
pages++
|
||||
for (const v of page.items) seen.push(v.id)
|
||||
if (!page.hasMore) break
|
||||
expect(page.nextCursor).toBeTruthy()
|
||||
cursor = page.nextCursor
|
||||
if (pages > 100) throw new Error('cursor walk failed to terminate')
|
||||
}
|
||||
|
||||
expect(pages).toBeGreaterThan(1) // genuinely paginated
|
||||
expect(new Set(seen).size).toBe(seen.length) // no duplicates
|
||||
expect(seen.length).toBe(EDGE_COUNT) // no skips
|
||||
expect(new Set(seen)).toEqual(refIds) // exactly the full set
|
||||
})
|
||||
|
||||
it('cursor and offset walks return the same set', async () => {
|
||||
const offsetSeen: string[] = []
|
||||
let offset = 0
|
||||
for (;;) {
|
||||
const page = await storage.getVerbs({ pagination: { limit: 9, offset } })
|
||||
for (const v of page.items) offsetSeen.push(v.id)
|
||||
if (!page.hasMore) break
|
||||
offset += page.items.length
|
||||
}
|
||||
|
||||
const cursorSeen: string[] = []
|
||||
let cursor: string | undefined
|
||||
for (;;) {
|
||||
const page = await storage.getVerbs({
|
||||
pagination: cursor ? { limit: 9, cursor } : { limit: 9 }
|
||||
})
|
||||
for (const v of page.items) cursorSeen.push(v.id)
|
||||
if (!page.hasMore) break
|
||||
cursor = page.nextCursor
|
||||
}
|
||||
|
||||
expect(cursorSeen.length).toBe(EDGE_COUNT)
|
||||
expect(new Set(cursorSeen)).toEqual(new Set(offsetSeen))
|
||||
})
|
||||
|
||||
it('a foreign/malformed cursor falls back gracefully (no throw, starts from the beginning)', async () => {
|
||||
const page = await storage.getVerbs({ pagination: { limit: 5, cursor: 'not-a-cv1-token' } })
|
||||
expect(page.items.length).toBe(5)
|
||||
expect(page.hasMore).toBe(true)
|
||||
})
|
||||
})
|
||||
Loading…
Add table
Add a link
Reference in a new issue