perf(8.0): cursor pagination for the verb walk — full edge pagination O(N²) → O(N)

getVerbsWithPagination was offset-only: it re-scanned from shard 0 on every page
and early-terminated at offset+limit, so a full edge walk was O(N²) (a consumer
measured ~19k edges → 27s paging at 900/page). It now accepts an opaque resume
token (cv1:<shard>:<verbId>) that resumes the shard walk immediately AFTER the
last returned verb — O(N) total at any page size, no scale cliff.

- Stable within-shard order (sort by verb id); ids come from the path so verbs
  skipped by the cursor are never read.
- Cursor supersedes offset when present; offset path preserved for back-compat,
  and offset callers now get a usable nextCursor so they can switch to cursor
  paging. Foreign/malformed tokens decode to null → offset fallback (no throw).
- The relationships stream generator now uses cursor (was offset → O(N²)).
- Parity with the noun side (getNounsWithPagination already cursored).

Test: tests/unit/storage/verb-cursor-pagination.test.ts — a full cursored walk
visits every edge exactly once (no dup, no skip), matches the offset walk's set,
and a foreign cursor falls back gracefully.

src/brainy.ts

@@ -8279,15 +8279,17 @@ export class Brainy<T = any> implements BrainyInterface<T> {
         }
       }.bind(this),
 
-      // Stream relationships efficiently
+      // Stream relationships efficiently. Cursor-based when the adapter supports it
+      // (resumes after the last verb — O(N) for a full walk) and falls back to offset
+      // otherwise. Offset paging here re-scanned from the start every page (O(N²)).
       relationships: async function* (this: Brainy<T>, filter?: { type?: string, sourceId?: string, targetId?: string }) {
         let offset = 0
+        let cursor: string | undefined
         const batchSize = 100
-        let hasMore = true
 
-        while (hasMore) {
+        for (;;) {
           const result = await this.storage.getVerbs({
-            pagination: { offset, limit: batchSize },
+            pagination: cursor ? { limit: batchSize, cursor } : { offset, limit: batchSize },
             filter
           })
 
@@ -8295,8 +8297,12 @@ export class Brainy<T = any> implements BrainyInterface<T> {
             yield verb
           }
 
-          hasMore = result.hasMore
+          if (!result.hasMore || result.items.length === 0) break
-          offset += batchSize
+          if (result.nextCursor) {
+            cursor = result.nextCursor
+          } else {
+            offset += result.items.length
+          }
         }
       }.bind(this),
 

src/storage/baseStorage.ts

@@ -184,6 +184,20 @@ function getVerbVectorPath(id: string): string {
   return `entities/verbs/${shard}/${id}/vectors.json`
 }
 
+/**
+ * @description Extract the verb id embedded in a verb vector path
+ * (`entities/verbs/{shard}/{id}/vectors.json`). Used by the cursored verb walk
+ * to order and skip candidates by id WITHOUT reading each file, which is what
+ * keeps a full cursored pagination O(N) instead of O(N²).
+ * @param path - A verb vector path (full or prefix-relative; must end with `/vectors.json`).
+ * @returns The verb id (the path segment immediately before `/vectors.json`).
+ */
+function verbIdFromVectorPath(path: string): string {
+  const withoutSuffix = path.replace(/\/vectors\.json$/, '')
+  const lastSlash = withoutSuffix.lastIndexOf('/')
+  return lastSlash >= 0 ? withoutSuffix.slice(lastSlash + 1) : withoutSuffix
+}
+
 /**
  * Get ID-first path for verb metadata
  * No type parameter needed - direct O(1) lookup by ID
@@ -1707,7 +1721,7 @@ export abstract class BaseStorage extends BaseStorageAdapter {
   public async getVerbsWithPagination(options: {
     limit: number
     offset: number
-    cursor?: string  // Currently ignored (offset-based pagination). Cursor support planned
+    cursor?: string  // Opaque resume token from a prior page's nextCursor; when set it supersedes offset (O(N) walk, no re-scan)
     filter?: {
       verbType?: string | string[]
       sourceId?: string | string[]
@@ -1723,13 +1737,17 @@ export abstract class BaseStorage extends BaseStorageAdapter {
   }> {
     await this.ensureInitialized()
 
-    const { limit, offset = 0, filter } = options  // cursor intentionally not extracted (not yet implemented)
+    const { limit, offset = 0, filter } = options
-    const collectedVerbs: HNSWVerbWithMetadata[] = []
+    // Cursor (8.0): an opaque resume token (see encodeVerbWalkCursor) carrying the
-    // Same peek-one-past-the-window strategy as getNounsWithPagination — see
+    // (shard, verbId) of the last returned verb. When present it SUPERSEDES `offset`
-    // the comment there. Without the extra item, hasMore is undecidable and
+    // and resumes the shard walk immediately AFTER that position, so a full walk is
-    // was permanently false (silent truncation of every multi-page walk).
+    // O(N) total instead of the O(N²) of offset paging (which re-scans from shard 0
-    const targetCount = offset + limit  // Requested window end
+    // every page). Malformed / foreign tokens decode to null → offset fallback.
-    const peekCount = targetCount + 1   // Early termination target (window + 1 peek)
+    const cursor = this.decodeVerbWalkCursor(options.cursor)
+
+    // Each collected entry remembers its shard so nextCursor can point at the exact
+    // (shard, id) resume position.
+    const collected: Array<{ verb: HNSWVerbWithMetadata; shard: number }> = []
 
     // Prepare filter sets for efficient lookup
     const filterVerbTypes = filter?.verbType
@@ -1761,17 +1779,34 @@ export abstract class BaseStorage extends BaseStorageAdapter {
       ? new Set(excludeVisibility)
       : null
 
-    // Iterate by shards (0x00-0xFF) instead of types - single pass!
+    // Peek one past the window so hasMore is decidable. Cursor mode collects exactly
-    for (let shard = 0; shard < 256 && collectedVerbs.length < peekCount; shard++) {
+    // one page (+1); offset mode keeps the full [0, offset+limit] window (+1).
+    const peekCount = cursor ? limit + 1 : offset + limit + 1
+    // Cursor resume skips every shard BEFORE the cursor's shard outright (the core of
+    // the O(N) win); offset mode always starts at shard 0.
+    const startShard = cursor ? cursor.shard : 0
+
+    // Iterate by shards (0x00-0xFF) — single pass, early-terminating at peekCount.
+    for (let shard = startShard; shard < 256 && collected.length < peekCount; shard++) {
       const shardHex = shard.toString(16).padStart(2, '0')
       const shardDir = `entities/verbs/${shardHex}`
 
       try {
         const verbFiles = await this.listCanonicalObjects(shardDir)
 
-        for (const verbPath of verbFiles) {
+        // Stable within-shard order (by verb id) so offset windows and cursor resume
-          if (collectedVerbs.length >= peekCount) break
+        // are deterministic and consistent across calls. Ids come from the path, so
-          if (!verbPath.includes('/vectors.json')) continue
+        // verbs skipped by the cursor are never read.
+        const entries = verbFiles
+          .filter((p) => p.includes('/vectors.json'))
+          .map((p) => ({ path: p, id: verbIdFromVectorPath(p) }))
+          .sort((a, b) => (a.id < b.id ? -1 : a.id > b.id ? 1 : 0))
+
+        for (const { path: verbPath, id: verbId } of entries) {
+          if (collected.length >= peekCount) break
+          // Resume: in the cursor's own shard, skip up to AND INCLUDING the cursor id
+          // (later shards are processed in full). No read for skipped verbs.
+          if (cursor && shard === cursor.shard && verbId <= cursor.id) continue
 
           try {
             const rawVerb = await this.readCanonicalObject(verbPath)
@@ -1816,9 +1851,8 @@ export abstract class BaseStorage extends BaseStorageAdapter {
             }
 
             // Combine verb + metadata via the canonical hydration helper —
-            // reserved fields top-level, ONLY custom fields in `metadata`
+            // reserved fields top-level, ONLY custom fields in `metadata`.
-            // (this site previously echoed the full flat record).
+            collected.push({ verb: this.hydrateVerbWithMetadata(verb, metadata), shard })
-            collectedVerbs.push(this.hydrateVerbWithMetadata(verb, metadata))
           } catch (error) {
             // Skip verbs that fail to load
           }
@@ -1828,35 +1862,72 @@ export abstract class BaseStorage extends BaseStorageAdapter {
       }
     }
 
-    // Apply pagination. The peeked extra item (if any) is dropped by the slice;
+    // Window selection. Cursor mode already starts at the resume point, so its window
-    // its existence is exactly what makes hasMore true. `>` (not `>=`) keeps the
+    // is [0, limit); offset mode slices [offset, offset+limit). The peeked extra entry
-    // exact-boundary case (total == offset+limit) from looping forever.
+    // (if any) is dropped here — its existence is exactly what makes hasMore true.
-    const paginatedVerbs = collectedVerbs.slice(offset, offset + limit)
+    const windowStart = cursor ? 0 : offset
-    const hasMore = collectedVerbs.length > targetCount
+    const pagePairs = collected.slice(windowStart, windowStart + limit)
+    const paginatedVerbs = pagePairs.map((p) => p.verb)
+    const hasMore = collected.length > windowStart + limit
 
-    // totalCount must be the TRUE dataset total, not this peeked page. The verb
+    // totalCount must be the TRUE dataset total, not this peeked page. For the
-    // type-scan early-terminates at `peekCount = offset + limit + 1`, so
+    // unfiltered scan the authoritative total is the O(1) `totalVerbCount` counter
-    // `collectedVerbs.length` only ever reaches the page size + 1 — returning it
+    // (isNew-gated, visibility-filtered, rehydrated on init); `Math.max` guards a
-    // made `getVerbs({ pagination: { limit: 1 } }).totalCount` read 1 (or 2) for
+    // stale counter from under-reporting. A filtered scan has no cheap exact total,
-    // any non-empty brain. This is the verb mirror of the noun fix (b2005ff): for
+    // so it keeps the collected length (a lower bound).
-    // the unfiltered scan the authoritative total is the O(1) `totalVerbCount`
-    // counter (isNew-gated, visibility-filtered, rehydrated on init); `Math.max`
-    // guards a stale counter from under-reporting. A filtered scan has no cheap
-    // exact total, so it keeps the collected length (a lower bound).
     const totalCount = filter
-      ? collectedVerbs.length
+      ? collected.length
-      : Math.max(this.totalVerbCount, collectedVerbs.length)
+      : Math.max(this.totalVerbCount, collected.length)
+
+    // nextCursor encodes the (shard, id) of the LAST RETURNED verb so the next call
+    // resumes immediately after it — for both cursor and offset callers (an offset
+    // caller can switch to cursor paging to escape the O(N²)).
+    let nextCursor: string | undefined = undefined
+    if (hasMore && pagePairs.length > 0) {
+      const lastPair = pagePairs[pagePairs.length - 1]
+      nextCursor = this.encodeVerbWalkCursor(lastPair.shard, lastPair.verb.id)
+    }
 
     return {
       items: paginatedVerbs,
       totalCount,
       hasMore,
-      nextCursor: hasMore && paginatedVerbs.length > 0
+      nextCursor
-        ? paginatedVerbs[paginatedVerbs.length - 1].id
-        : undefined
     }
   }
 
+  /**
+   * @description Encode a verb-walk resume cursor — the `(shard, verbId)` of the
+   * last returned verb — as an opaque, version-tagged token. The `cv1:` prefix
+   * lets {@link decodeVerbWalkCursor} reject foreign tokens (e.g. the bare-id
+   * cursors the graph-index fast paths emit). `verbId` is placed last and the
+   * decoder re-joins on `:` so any id format survives the round-trip.
+   * @param shard - The shard (0–255) the verb lives in.
+   * @param id - The verb id.
+   * @returns The opaque cursor token.
+   */
+  private encodeVerbWalkCursor(shard: number, id: string): string {
+    return `cv1:${shard}:${id}`
+  }
+
+  /**
+   * @description Decode a verb-walk cursor produced by {@link encodeVerbWalkCursor}.
+   * Returns `null` for an absent, malformed, or foreign token so the caller falls
+   * back to offset-based paging rather than mis-resuming.
+   * @param cursor - The opaque cursor token, or undefined.
+   * @returns `{ shard, id }` resume position, or `null`.
+   */
+  private decodeVerbWalkCursor(cursor?: string): { shard: number; id: string } | null {
+    if (!cursor) return null
+    const parts = cursor.split(':')
+    if (parts.length < 3 || parts[0] !== 'cv1') return null
+    const shard = Number(parts[1])
+    if (!Number.isInteger(shard) || shard < 0 || shard > 255) return null
+    const id = parts.slice(2).join(':')
+    if (id.length === 0) return null
+    return { shard, id }
+  }
+
   /**
    * Get verbs with pagination and filtering
    * @param options Pagination and filtering options

tests/unit/storage/verb-cursor-pagination.test.ts

@@ -0,0 +1,103 @@
+/**
+ * @module tests/unit/storage/verb-cursor-pagination
+ * @description Graph-perf #2 (8.0): cursor pagination over the verb shard walk.
+ *
+ * `getVerbsWithPagination` used to be offset-only — it re-scanned from shard 0 on
+ * every page and early-terminated at `offset+limit`, so a full edge walk was O(N²)
+ * (a consumer measured 19k edges → 27s paging at 900/page). It now accepts an
+ * opaque cursor that resumes immediately after the last returned verb, making a
+ * full walk O(N) at any page size. These tests pin the correctness guarantees a
+ * cursor MUST provide: every item exactly once, no duplicates, no skips, and the
+ * same set as an offset walk — plus graceful fallback for a foreign token.
+ */
+
+import { describe, it, expect, beforeEach, afterEach } from 'vitest'
+import { Brainy } from '../../../src/index.js'
+import { NounType, VerbType } from '../../../src/types/graphTypes.js'
+import { createTestConfig } from '../../helpers/test-factory.js'
+
+describe('verb cursor pagination (graph-perf #2)', () => {
+  let brain: Brainy
+  // The storage layer is where the cursor lives; exercise the primitive directly.
+  let storage: {
+    getVerbs(opts: {
+      pagination?: { offset?: number; limit?: number; cursor?: string }
+    }): Promise<{ items: Array<{ id: string }>; hasMore: boolean; nextCursor?: string }>
+  }
+  const EDGE_COUNT = 40
+
+  beforeEach(async () => {
+    brain = new Brainy(createTestConfig())
+    await brain.init()
+    const ids: string[] = []
+    for (let i = 0; i <= EDGE_COUNT; i++) {
+      ids.push(await brain.add({ type: NounType.Person, subtype: 'employee', data: `N${i}` }))
+    }
+    // Hub-and-spoke: EDGE_COUNT edges from one node, spread across id-hash shards.
+    for (let i = 1; i <= EDGE_COUNT; i++) {
+      await brain.relate({ from: ids[0], to: ids[i], type: VerbType.RelatedTo, subtype: 'colleague' })
+    }
+    storage = (brain as unknown as { storage: typeof storage }).storage
+  })
+
+  afterEach(async () => {
+    await brain.close()
+  })
+
+  it('a full cursored walk visits every edge exactly once (no dup, no skip)', async () => {
+    const all = await storage.getVerbs({ pagination: { limit: 10000 } })
+    const refIds = new Set(all.items.map((v) => v.id))
+    expect(refIds.size).toBe(EDGE_COUNT)
+
+    const seen: string[] = []
+    let cursor: string | undefined
+    let pages = 0
+    for (;;) {
+      const page = await storage.getVerbs({
+        pagination: cursor ? { limit: 7, cursor } : { limit: 7 }
+      })
+      pages++
+      for (const v of page.items) seen.push(v.id)
+      if (!page.hasMore) break
+      expect(page.nextCursor).toBeTruthy()
+      cursor = page.nextCursor
+      if (pages > 100) throw new Error('cursor walk failed to terminate')
+    }
+
+    expect(pages).toBeGreaterThan(1) // genuinely paginated
+    expect(new Set(seen).size).toBe(seen.length) // no duplicates
+    expect(seen.length).toBe(EDGE_COUNT) // no skips
+    expect(new Set(seen)).toEqual(refIds) // exactly the full set
+  })
+
+  it('cursor and offset walks return the same set', async () => {
+    const offsetSeen: string[] = []
+    let offset = 0
+    for (;;) {
+      const page = await storage.getVerbs({ pagination: { limit: 9, offset } })
+      for (const v of page.items) offsetSeen.push(v.id)
+      if (!page.hasMore) break
+      offset += page.items.length
+    }
+
+    const cursorSeen: string[] = []
+    let cursor: string | undefined
+    for (;;) {
+      const page = await storage.getVerbs({
+        pagination: cursor ? { limit: 9, cursor } : { limit: 9 }
+      })
+      for (const v of page.items) cursorSeen.push(v.id)
+      if (!page.hasMore) break
+      cursor = page.nextCursor
+    }
+
+    expect(cursorSeen.length).toBe(EDGE_COUNT)
+    expect(new Set(cursorSeen)).toEqual(new Set(offsetSeen))
+  })
+
+  it('a foreign/malformed cursor falls back gracefully (no throw, starts from the beginning)', async () => {
+    const page = await storage.getVerbs({ pagination: { limit: 5, cursor: 'not-a-cv1-token' } })
+    expect(page.items.length).toBe(5)
+    expect(page.hasMore).toBe(true)
+  })
+})