perf(8.0): cursor pagination for the verb walk — full edge pagination O(N²) → O(N)
getVerbsWithPagination was offset-only: it re-scanned from shard 0 on every page and early-terminated at offset+limit, so a full edge walk was O(N²) (a consumer measured ~19k edges → 27s paging at 900/page). It now accepts an opaque resume token (cv1:<shard>:<verbId>) that resumes the shard walk immediately AFTER the last returned verb — O(N) total at any page size, no scale cliff. - Stable within-shard order (sort by verb id); ids come from the path so verbs skipped by the cursor are never read. - Cursor supersedes offset when present; offset path preserved for back-compat, and offset callers now get a usable nextCursor so they can switch to cursor paging. Foreign/malformed tokens decode to null → offset fallback (no throw). - The relationships stream generator now uses cursor (was offset → O(N²)). - Parity with the noun side (getNounsWithPagination already cursored). Test: tests/unit/storage/verb-cursor-pagination.test.ts — a full cursored walk visits every edge exactly once (no dup, no skip), matches the offset walk's set, and a foreign cursor falls back gracefully.
This commit is contained in:
parent
a914313e3a
commit
682e786cc3
3 changed files with 221 additions and 41 deletions
|
|
@ -184,6 +184,20 @@ function getVerbVectorPath(id: string): string {
|
|||
return `entities/verbs/${shard}/${id}/vectors.json`
|
||||
}
|
||||
|
||||
/**
|
||||
* @description Extract the verb id embedded in a verb vector path
|
||||
* (`entities/verbs/{shard}/{id}/vectors.json`). Used by the cursored verb walk
|
||||
* to order and skip candidates by id WITHOUT reading each file, which is what
|
||||
* keeps a full cursored pagination O(N) instead of O(N²).
|
||||
* @param path - A verb vector path (full or prefix-relative; must end with `/vectors.json`).
|
||||
* @returns The verb id (the path segment immediately before `/vectors.json`).
|
||||
*/
|
||||
function verbIdFromVectorPath(path: string): string {
|
||||
const withoutSuffix = path.replace(/\/vectors\.json$/, '')
|
||||
const lastSlash = withoutSuffix.lastIndexOf('/')
|
||||
return lastSlash >= 0 ? withoutSuffix.slice(lastSlash + 1) : withoutSuffix
|
||||
}
|
||||
|
||||
/**
|
||||
* Get ID-first path for verb metadata
|
||||
* No type parameter needed - direct O(1) lookup by ID
|
||||
|
|
@ -1707,7 +1721,7 @@ export abstract class BaseStorage extends BaseStorageAdapter {
|
|||
public async getVerbsWithPagination(options: {
|
||||
limit: number
|
||||
offset: number
|
||||
cursor?: string // Currently ignored (offset-based pagination). Cursor support planned
|
||||
cursor?: string // Opaque resume token from a prior page's nextCursor; when set it supersedes offset (O(N) walk, no re-scan)
|
||||
filter?: {
|
||||
verbType?: string | string[]
|
||||
sourceId?: string | string[]
|
||||
|
|
@ -1723,13 +1737,17 @@ export abstract class BaseStorage extends BaseStorageAdapter {
|
|||
}> {
|
||||
await this.ensureInitialized()
|
||||
|
||||
const { limit, offset = 0, filter } = options // cursor intentionally not extracted (not yet implemented)
|
||||
const collectedVerbs: HNSWVerbWithMetadata[] = []
|
||||
// Same peek-one-past-the-window strategy as getNounsWithPagination — see
|
||||
// the comment there. Without the extra item, hasMore is undecidable and
|
||||
// was permanently false (silent truncation of every multi-page walk).
|
||||
const targetCount = offset + limit // Requested window end
|
||||
const peekCount = targetCount + 1 // Early termination target (window + 1 peek)
|
||||
const { limit, offset = 0, filter } = options
|
||||
// Cursor (8.0): an opaque resume token (see encodeVerbWalkCursor) carrying the
|
||||
// (shard, verbId) of the last returned verb. When present it SUPERSEDES `offset`
|
||||
// and resumes the shard walk immediately AFTER that position, so a full walk is
|
||||
// O(N) total instead of the O(N²) of offset paging (which re-scans from shard 0
|
||||
// every page). Malformed / foreign tokens decode to null → offset fallback.
|
||||
const cursor = this.decodeVerbWalkCursor(options.cursor)
|
||||
|
||||
// Each collected entry remembers its shard so nextCursor can point at the exact
|
||||
// (shard, id) resume position.
|
||||
const collected: Array<{ verb: HNSWVerbWithMetadata; shard: number }> = []
|
||||
|
||||
// Prepare filter sets for efficient lookup
|
||||
const filterVerbTypes = filter?.verbType
|
||||
|
|
@ -1761,17 +1779,34 @@ export abstract class BaseStorage extends BaseStorageAdapter {
|
|||
? new Set(excludeVisibility)
|
||||
: null
|
||||
|
||||
// Iterate by shards (0x00-0xFF) instead of types - single pass!
|
||||
for (let shard = 0; shard < 256 && collectedVerbs.length < peekCount; shard++) {
|
||||
// Peek one past the window so hasMore is decidable. Cursor mode collects exactly
|
||||
// one page (+1); offset mode keeps the full [0, offset+limit] window (+1).
|
||||
const peekCount = cursor ? limit + 1 : offset + limit + 1
|
||||
// Cursor resume skips every shard BEFORE the cursor's shard outright (the core of
|
||||
// the O(N) win); offset mode always starts at shard 0.
|
||||
const startShard = cursor ? cursor.shard : 0
|
||||
|
||||
// Iterate by shards (0x00-0xFF) — single pass, early-terminating at peekCount.
|
||||
for (let shard = startShard; shard < 256 && collected.length < peekCount; shard++) {
|
||||
const shardHex = shard.toString(16).padStart(2, '0')
|
||||
const shardDir = `entities/verbs/${shardHex}`
|
||||
|
||||
try {
|
||||
const verbFiles = await this.listCanonicalObjects(shardDir)
|
||||
|
||||
for (const verbPath of verbFiles) {
|
||||
if (collectedVerbs.length >= peekCount) break
|
||||
if (!verbPath.includes('/vectors.json')) continue
|
||||
// Stable within-shard order (by verb id) so offset windows and cursor resume
|
||||
// are deterministic and consistent across calls. Ids come from the path, so
|
||||
// verbs skipped by the cursor are never read.
|
||||
const entries = verbFiles
|
||||
.filter((p) => p.includes('/vectors.json'))
|
||||
.map((p) => ({ path: p, id: verbIdFromVectorPath(p) }))
|
||||
.sort((a, b) => (a.id < b.id ? -1 : a.id > b.id ? 1 : 0))
|
||||
|
||||
for (const { path: verbPath, id: verbId } of entries) {
|
||||
if (collected.length >= peekCount) break
|
||||
// Resume: in the cursor's own shard, skip up to AND INCLUDING the cursor id
|
||||
// (later shards are processed in full). No read for skipped verbs.
|
||||
if (cursor && shard === cursor.shard && verbId <= cursor.id) continue
|
||||
|
||||
try {
|
||||
const rawVerb = await this.readCanonicalObject(verbPath)
|
||||
|
|
@ -1816,9 +1851,8 @@ export abstract class BaseStorage extends BaseStorageAdapter {
|
|||
}
|
||||
|
||||
// Combine verb + metadata via the canonical hydration helper —
|
||||
// reserved fields top-level, ONLY custom fields in `metadata`
|
||||
// (this site previously echoed the full flat record).
|
||||
collectedVerbs.push(this.hydrateVerbWithMetadata(verb, metadata))
|
||||
// reserved fields top-level, ONLY custom fields in `metadata`.
|
||||
collected.push({ verb: this.hydrateVerbWithMetadata(verb, metadata), shard })
|
||||
} catch (error) {
|
||||
// Skip verbs that fail to load
|
||||
}
|
||||
|
|
@ -1828,35 +1862,72 @@ export abstract class BaseStorage extends BaseStorageAdapter {
|
|||
}
|
||||
}
|
||||
|
||||
// Apply pagination. The peeked extra item (if any) is dropped by the slice;
|
||||
// its existence is exactly what makes hasMore true. `>` (not `>=`) keeps the
|
||||
// exact-boundary case (total == offset+limit) from looping forever.
|
||||
const paginatedVerbs = collectedVerbs.slice(offset, offset + limit)
|
||||
const hasMore = collectedVerbs.length > targetCount
|
||||
// Window selection. Cursor mode already starts at the resume point, so its window
|
||||
// is [0, limit); offset mode slices [offset, offset+limit). The peeked extra entry
|
||||
// (if any) is dropped here — its existence is exactly what makes hasMore true.
|
||||
const windowStart = cursor ? 0 : offset
|
||||
const pagePairs = collected.slice(windowStart, windowStart + limit)
|
||||
const paginatedVerbs = pagePairs.map((p) => p.verb)
|
||||
const hasMore = collected.length > windowStart + limit
|
||||
|
||||
// totalCount must be the TRUE dataset total, not this peeked page. The verb
|
||||
// type-scan early-terminates at `peekCount = offset + limit + 1`, so
|
||||
// `collectedVerbs.length` only ever reaches the page size + 1 — returning it
|
||||
// made `getVerbs({ pagination: { limit: 1 } }).totalCount` read 1 (or 2) for
|
||||
// any non-empty brain. This is the verb mirror of the noun fix (b2005ff): for
|
||||
// the unfiltered scan the authoritative total is the O(1) `totalVerbCount`
|
||||
// counter (isNew-gated, visibility-filtered, rehydrated on init); `Math.max`
|
||||
// guards a stale counter from under-reporting. A filtered scan has no cheap
|
||||
// exact total, so it keeps the collected length (a lower bound).
|
||||
// totalCount must be the TRUE dataset total, not this peeked page. For the
|
||||
// unfiltered scan the authoritative total is the O(1) `totalVerbCount` counter
|
||||
// (isNew-gated, visibility-filtered, rehydrated on init); `Math.max` guards a
|
||||
// stale counter from under-reporting. A filtered scan has no cheap exact total,
|
||||
// so it keeps the collected length (a lower bound).
|
||||
const totalCount = filter
|
||||
? collectedVerbs.length
|
||||
: Math.max(this.totalVerbCount, collectedVerbs.length)
|
||||
? collected.length
|
||||
: Math.max(this.totalVerbCount, collected.length)
|
||||
|
||||
// nextCursor encodes the (shard, id) of the LAST RETURNED verb so the next call
|
||||
// resumes immediately after it — for both cursor and offset callers (an offset
|
||||
// caller can switch to cursor paging to escape the O(N²)).
|
||||
let nextCursor: string | undefined = undefined
|
||||
if (hasMore && pagePairs.length > 0) {
|
||||
const lastPair = pagePairs[pagePairs.length - 1]
|
||||
nextCursor = this.encodeVerbWalkCursor(lastPair.shard, lastPair.verb.id)
|
||||
}
|
||||
|
||||
return {
|
||||
items: paginatedVerbs,
|
||||
totalCount,
|
||||
hasMore,
|
||||
nextCursor: hasMore && paginatedVerbs.length > 0
|
||||
? paginatedVerbs[paginatedVerbs.length - 1].id
|
||||
: undefined
|
||||
nextCursor
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* @description Encode a verb-walk resume cursor — the `(shard, verbId)` of the
|
||||
* last returned verb — as an opaque, version-tagged token. The `cv1:` prefix
|
||||
* lets {@link decodeVerbWalkCursor} reject foreign tokens (e.g. the bare-id
|
||||
* cursors the graph-index fast paths emit). `verbId` is placed last and the
|
||||
* decoder re-joins on `:` so any id format survives the round-trip.
|
||||
* @param shard - The shard (0–255) the verb lives in.
|
||||
* @param id - The verb id.
|
||||
* @returns The opaque cursor token.
|
||||
*/
|
||||
private encodeVerbWalkCursor(shard: number, id: string): string {
|
||||
return `cv1:${shard}:${id}`
|
||||
}
|
||||
|
||||
/**
|
||||
* @description Decode a verb-walk cursor produced by {@link encodeVerbWalkCursor}.
|
||||
* Returns `null` for an absent, malformed, or foreign token so the caller falls
|
||||
* back to offset-based paging rather than mis-resuming.
|
||||
* @param cursor - The opaque cursor token, or undefined.
|
||||
* @returns `{ shard, id }` resume position, or `null`.
|
||||
*/
|
||||
private decodeVerbWalkCursor(cursor?: string): { shard: number; id: string } | null {
|
||||
if (!cursor) return null
|
||||
const parts = cursor.split(':')
|
||||
if (parts.length < 3 || parts[0] !== 'cv1') return null
|
||||
const shard = Number(parts[1])
|
||||
if (!Number.isInteger(shard) || shard < 0 || shard > 255) return null
|
||||
const id = parts.slice(2).join(':')
|
||||
if (id.length === 0) return null
|
||||
return { shard, id }
|
||||
}
|
||||
|
||||
/**
|
||||
* Get verbs with pagination and filtering
|
||||
* @param options Pagination and filtering options
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue