David Snelling
682e786cc3
getVerbsWithPagination was offset-only: it re-scanned from shard 0 on every page and early-terminated at offset+limit, so a full edge walk was O(N²) (a consumer measured ~19k edges → 27s paging at 900/page). It now accepts an opaque resume token (cv1:<shard>:<verbId>) that resumes the shard walk immediately AFTER the last returned verb — O(N) total at any page size, no scale cliff. - Stable within-shard order (sort by verb id); ids come from the path so verbs skipped by the cursor are never read. - Cursor supersedes offset when present; offset path preserved for back-compat, and offset callers now get a usable nextCursor so they can switch to cursor paging. Foreign/malformed tokens decode to null → offset fallback (no throw). - The relationships stream generator now uses cursor (was offset → O(N²)). - Parity with the noun side (getNounsWithPagination already cursored). Test: tests/unit/storage/verb-cursor-pagination.test.ts — a full cursored walk visits every edge exactly once (no dup, no skip), matches the offset walk's set, and a foreign cursor falls back gracefully.
103 lines
4 KiB
TypeScript
103 lines
4 KiB
TypeScript
/**
|
|
* @module tests/unit/storage/verb-cursor-pagination
|
|
* @description Graph-perf #2 (8.0): cursor pagination over the verb shard walk.
|
|
*
|
|
* `getVerbsWithPagination` used to be offset-only — it re-scanned from shard 0 on
|
|
* every page and early-terminated at `offset+limit`, so a full edge walk was O(N²)
|
|
* (a consumer measured 19k edges → 27s paging at 900/page). It now accepts an
|
|
* opaque cursor that resumes immediately after the last returned verb, making a
|
|
* full walk O(N) at any page size. These tests pin the correctness guarantees a
|
|
* cursor MUST provide: every item exactly once, no duplicates, no skips, and the
|
|
* same set as an offset walk — plus graceful fallback for a foreign token.
|
|
*/
|
|
|
|
import { describe, it, expect, beforeEach, afterEach } from 'vitest'
|
|
import { Brainy } from '../../../src/index.js'
|
|
import { NounType, VerbType } from '../../../src/types/graphTypes.js'
|
|
import { createTestConfig } from '../../helpers/test-factory.js'
|
|
|
|
describe('verb cursor pagination (graph-perf #2)', () => {
|
|
let brain: Brainy
|
|
// The storage layer is where the cursor lives; exercise the primitive directly.
|
|
let storage: {
|
|
getVerbs(opts: {
|
|
pagination?: { offset?: number; limit?: number; cursor?: string }
|
|
}): Promise<{ items: Array<{ id: string }>; hasMore: boolean; nextCursor?: string }>
|
|
}
|
|
const EDGE_COUNT = 40
|
|
|
|
beforeEach(async () => {
|
|
brain = new Brainy(createTestConfig())
|
|
await brain.init()
|
|
const ids: string[] = []
|
|
for (let i = 0; i <= EDGE_COUNT; i++) {
|
|
ids.push(await brain.add({ type: NounType.Person, subtype: 'employee', data: `N${i}` }))
|
|
}
|
|
// Hub-and-spoke: EDGE_COUNT edges from one node, spread across id-hash shards.
|
|
for (let i = 1; i <= EDGE_COUNT; i++) {
|
|
await brain.relate({ from: ids[0], to: ids[i], type: VerbType.RelatedTo, subtype: 'colleague' })
|
|
}
|
|
storage = (brain as unknown as { storage: typeof storage }).storage
|
|
})
|
|
|
|
afterEach(async () => {
|
|
await brain.close()
|
|
})
|
|
|
|
it('a full cursored walk visits every edge exactly once (no dup, no skip)', async () => {
|
|
const all = await storage.getVerbs({ pagination: { limit: 10000 } })
|
|
const refIds = new Set(all.items.map((v) => v.id))
|
|
expect(refIds.size).toBe(EDGE_COUNT)
|
|
|
|
const seen: string[] = []
|
|
let cursor: string | undefined
|
|
let pages = 0
|
|
for (;;) {
|
|
const page = await storage.getVerbs({
|
|
pagination: cursor ? { limit: 7, cursor } : { limit: 7 }
|
|
})
|
|
pages++
|
|
for (const v of page.items) seen.push(v.id)
|
|
if (!page.hasMore) break
|
|
expect(page.nextCursor).toBeTruthy()
|
|
cursor = page.nextCursor
|
|
if (pages > 100) throw new Error('cursor walk failed to terminate')
|
|
}
|
|
|
|
expect(pages).toBeGreaterThan(1) // genuinely paginated
|
|
expect(new Set(seen).size).toBe(seen.length) // no duplicates
|
|
expect(seen.length).toBe(EDGE_COUNT) // no skips
|
|
expect(new Set(seen)).toEqual(refIds) // exactly the full set
|
|
})
|
|
|
|
it('cursor and offset walks return the same set', async () => {
|
|
const offsetSeen: string[] = []
|
|
let offset = 0
|
|
for (;;) {
|
|
const page = await storage.getVerbs({ pagination: { limit: 9, offset } })
|
|
for (const v of page.items) offsetSeen.push(v.id)
|
|
if (!page.hasMore) break
|
|
offset += page.items.length
|
|
}
|
|
|
|
const cursorSeen: string[] = []
|
|
let cursor: string | undefined
|
|
for (;;) {
|
|
const page = await storage.getVerbs({
|
|
pagination: cursor ? { limit: 9, cursor } : { limit: 9 }
|
|
})
|
|
for (const v of page.items) cursorSeen.push(v.id)
|
|
if (!page.hasMore) break
|
|
cursor = page.nextCursor
|
|
}
|
|
|
|
expect(cursorSeen.length).toBe(EDGE_COUNT)
|
|
expect(new Set(cursorSeen)).toEqual(new Set(offsetSeen))
|
|
})
|
|
|
|
it('a foreign/malformed cursor falls back gracefully (no throw, starts from the beginning)', async () => {
|
|
const page = await storage.getVerbs({ pagination: { limit: 5, cursor: 'not-a-cv1-token' } })
|
|
expect(page.items.length).toBe(5)
|
|
expect(page.hasMore).toBe(true)
|
|
})
|
|
})
|