open-brainy/tests/unit/storage/pagination-parallel-hydration.test.ts

/**
 * @module tests/unit/storage/pagination-parallel-hydration
 * @description The canonical enumeration walk (getNounsWithPagination) hydrated
 * each item's vector + metadata ONE-AT-A-TIME — N×per-op-latency serially, the
 * dominant term in an index heal (cortex heal-cost decomposition). It now
 * hydrates 16-way, and a new getNounIdsWithPagination returns ids WITHOUT
 * hydration (zero per-entity reads when unfiltered). Both must preserve the exact
 * pagination contract: same order, cursor continuation, filters, totalCount.
 */
import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest'
import { Brainy, NounType } from '../../../src/index.js'

describe('paginated enumeration — parallel hydration + id-only (cortex heal-cost)', () => {
  let brain: any
  let storage: any
  const N = 30

  beforeEach(async () => {
    process.env.BRAINY_DETERMINISTIC_EMBEDDINGS = 'true'
    brain = new Brainy({ requireSubtype: false, storage: { type: 'memory' }, dimensions: 384 })
    await brain.init()
    for (let i = 0; i < N; i++) {
      await brain.add({
        data: `n${i}`,
        type: i % 3 === 0 ? NounType.Task : NounType.Concept,
        metadata: { i }
      })
    }
    await brain.flush()
    storage = brain.storage
  })

  afterEach(async () => {
    await brain.close()
  })

  /** Page the whole dataset through a small limit via cursor and collect ordered ids. */
  const pageAll = async (fn: (opts: any) => Promise<any>, key: 'items' | 'ids') => {
    const out: string[] = []
    let cursor: string | undefined
    for (let guard = 0; guard < 1000; guard++) {
      const page = await fn({ limit: 4, cursor })
      const batch = key === 'items' ? page.items.map((n: any) => n.id) : page.ids
      out.push(...batch)
      if (!page.hasMore) break
      cursor = page.nextCursor
    }
    return out
  }

  it('parallel hydration yields the SAME ordered pages as one big page', async () => {
    const big = await storage.getNounsWithPagination({ limit: 1000, offset: 0 })
    const bigIds = big.items.map((n: any) => n.id)
    // At least the N we added (a brain also has its VFS root entity).
    expect(bigIds.length).toBeGreaterThanOrEqual(N)
    expect(big.totalCount).toBe(bigIds.length)

    const paged = await pageAll((o) => storage.getNounsWithPagination(o), 'items')
    expect(paged).toEqual(bigIds) // identical order, no dupes, no gaps across pages
  })

  it('getNounIdsWithPagination returns exactly the same ids, in the same order', async () => {
    const idsPaged = await pageAll((o) => storage.getNounIdsWithPagination(o), 'ids')
    const itemsPaged = await pageAll((o) => storage.getNounsWithPagination(o), 'items')
    expect(idsPaged).toEqual(itemsPaged)
    expect(new Set(idsPaged).size).toBe(idsPaged.length) // every id exactly once
    expect(idsPaged.length).toBeGreaterThanOrEqual(N)
  })

  it('id-only enumeration does ZERO per-entity hydration reads when unfiltered', async () => {
    const readSpy = vi.spyOn(storage as any, 'readCanonicalObject')
    await storage.getNounIdsWithPagination({ limit: 1000, offset: 0 })
    expect(readSpy).not.toHaveBeenCalled()
    readSpy.mockRestore()

    // The hydrating walk, by contrast, DOES read each entity.
    const readSpy2 = vi.spyOn(storage as any, 'readCanonicalObject')
    await storage.getNounsWithPagination({ limit: 1000, offset: 0 })
    expect(readSpy2.mock.calls.length).toBeGreaterThan(0)
    readSpy2.mockRestore()
  })

  it('a type filter matches between the hydrating and id-only walks', async () => {
    const taskItems = await storage.getNounsWithPagination({
      limit: 1000,
      offset: 0,
      filter: { nounType: NounType.Task }
    })
    const taskIds = await storage.getNounIdsWithPagination({
      limit: 1000,
      offset: 0,
      filter: { nounType: NounType.Task }
    })
    const expected = Math.ceil(N / 3) // every 3rd is a Task
    expect(taskItems.items.length).toBe(expected)
    expect(new Set(taskIds.ids)).toEqual(new Set(taskItems.items.map((n: any) => n.id)))
  })
})