open-brainy/tests/unit/plugin/provider-generation.test.ts

/**
 * Generation threading to the metadata-index and vector-index provider write
 * surfaces — the counterpart of the graph pins in
 * tests/unit/transaction/graphIndexOperations-generation.test.ts.
 *
 * The provider contract gained an optional trailing `generation?: bigint` on
 * `MetadataIndexProvider.addToIndex`/`removeFromIndex`,
 * `VectorIndexProvider.addItem`/`removeItem` (+ the feature-detected
 * `updateItem`), and the id-mapper's `getOrAssign`/`remove`. A native provider
 * with per-record delta logs stamps its durable records with it — so the value
 * arriving MUST be the real commit generation (nonzero, monotonic), never a
 * fabricated 0 and never absent on the coordinator's write paths.
 *
 * Two layers of pins:
 *   1. End-to-end: provider doubles registered via the plugin system capture
 *      the generation argument during brain.add()/update()/remove() and it
 *      must equal the committed watermark (`brain.now().generation`).
 *   2. Operation layer: execute-time (not construction-time) resolution, and
 *      one shared generation across an op's forward + rollback halves.
 */
import { describe, it, expect, afterEach } from 'vitest'
import { Brainy, NounType } from '../../../src/index.js'
import { MetadataIndexManager } from '../../../src/utils/metadataIndex.js'
import {
  AddToVectorIndexOperation,
  RemoveFromVectorIndexOperation,
  ReplaceInVectorIndexOperation,
  AddToMetadataIndexOperation,
  RemoveFromMetadataIndexOperation
} from '../../../src/transaction/operations/IndexOperations.js'
import type { VectorIndexProvider } from '../../../src/plugin.js'

const V = () => Array.from({ length: 384 }, () => Math.random())

type Captured = { method: string; id: string; generation: bigint | undefined }

const brains: Brainy[] = []
afterEach(async () => {
  for (const b of brains.splice(0)) await b.close().catch(() => {})
})

/** Metadata manager subclass that records the generation of every write. */
function makeCapturingMetadataFactory(calls: Captured[]) {
  return (storage: any) => {
    class CapturingManager extends MetadataIndexManager {
      async addToIndex(id: string, entityOrMetadata: any, skipFlush = false, deferWrites = false, generation?: bigint): Promise<void> {
        calls.push({ method: 'addToIndex', id, generation })
        return super.addToIndex(id, entityOrMetadata, skipFlush, deferWrites, generation)
      }
      async removeFromIndex(id: string, metadata?: any, generation?: bigint): Promise<void> {
        calls.push({ method: 'removeFromIndex', id, generation })
        return super.removeFromIndex(id, metadata, generation)
      }
    }
    return new CapturingManager(storage)
  }
}

/** Minimal vector-index double capturing the generation of every write. */
function makeCapturingVectorFactory(calls: Captured[]) {
  return () => {
    const items = new Map<string, number[]>()
    const double: VectorIndexProvider & { updateItem(item: { id: string; vector: number[] }, generation?: bigint): Promise<void> } = {
      name: 'capture-double',
      async addItem(item, generation) {
        calls.push({ method: 'addItem', id: item.id, generation })
        items.set(item.id, item.vector as number[])
        return item.id
      },
      async removeItem(id, generation) {
        calls.push({ method: 'removeItem', id, generation })
        return items.delete(id)
      },
      async updateItem(item, generation) {
        calls.push({ method: 'updateItem', id: item.id, generation })
        items.set(item.id, item.vector)
      },
      async search() { return [] },
      size: () => items.size,
      clear: () => { items.clear() },
      async rebuild() {},
      async flush() { return 0 },
      getPersistMode: () => 'deferred' as const
    }
    return double
  }
}

async function makeBrain(plugin: any): Promise<Brainy> {
  const brain = new Brainy({
    storage: { type: 'memory' },
    requireSubtype: false,
    silent: true,
    plugins: []
  })
  brain.use(plugin)
  await brain.init()
  brains.push(brain)
  return brain
}

describe('Metadata-index provider — real commit generation on every write (end-to-end)', () => {
  it('add()/update()/remove() pass the nonzero, monotonic commit generation to addToIndex/removeFromIndex', async () => {
    const calls: Captured[] = []
    const brain = await makeBrain({
      name: 'capture-metadata',
      activate: async (ctx: any) => {
        ctx.registerProvider('metadataIndex', makeCapturingMetadataFactory(calls))
        return true
      }
    })

    const id = await brain.add({ data: 'one', type: NounType.Concept, metadata: { k: 'a' }, vector: V() })
    const addCall = calls.find((c) => c.method === 'addToIndex' && c.id === id)
    expect(addCall).toBeDefined()
    expect(typeof addCall!.generation).toBe('bigint')
    expect(addCall!.generation!).toBeGreaterThan(0n)
    // Committed watermark after a single-op write IS this write's generation.
    expect(addCall!.generation!).toBe(BigInt(brain.now().generation))

    calls.length = 0
    await brain.update({ id, metadata: { k: 'b' } })
    const updRemove = calls.find((c) => c.method === 'removeFromIndex' && c.id === id)
    const updAdd = calls.find((c) => c.method === 'addToIndex' && c.id === id)
    expect(updRemove?.generation).toBeDefined()
    expect(updAdd?.generation).toBeDefined()
    // One commit → the remove-old + add-new legs share one watermark.
    expect(updAdd!.generation!).toBe(updRemove!.generation!)
    expect(updAdd!.generation!).toBe(BigInt(brain.now().generation))
    const updateGen = updAdd!.generation!
    expect(updateGen).toBeGreaterThan(0n)

    calls.length = 0
    await brain.remove(id)
    const rmCall = calls.find((c) => c.method === 'removeFromIndex' && c.id === id)
    expect(rmCall?.generation).toBeDefined()
    expect(rmCall!.generation!).toBeGreaterThan(updateGen) // monotonic
    expect(rmCall!.generation!).toBe(BigInt(brain.now().generation))
  })

  it('transact() adds stamp the batch receipt generation', async () => {
    const calls: Captured[] = []
    const brain = await makeBrain({
      name: 'capture-metadata-tx',
      activate: async (ctx: any) => {
        ctx.registerProvider('metadataIndex', makeCapturingMetadataFactory(calls))
        return true
      }
    })

    // Bootstrap honesty: init-time infrastructure writes (the VFS root) are
    // applied WITHOUT a generation — the provider must receive undefined,
    // never a fabricated 0.
    for (const c of calls) expect(c.generation).toBeUndefined()
    calls.length = 0

    const db = await brain.transact([
      { op: 'add', data: 'tx-one', type: NounType.Concept, vector: V() },
      { op: 'add', data: 'tx-two', type: NounType.Concept, vector: V() }
    ] as any)

    const receiptGen = BigInt(db.receipt!.generation)
    const addGens = calls.filter((c) => c.method === 'addToIndex').map((c) => c.generation)
    expect(addGens.length).toBeGreaterThanOrEqual(2)
    for (const g of addGens) expect(g).toBe(receiptGen)
  })
})

describe('Vector-index provider — real commit generation on every write (end-to-end)', () => {
  it('add()/update()/remove() pass the nonzero commit generation to addItem/updateItem/removeItem', async () => {
    const calls: Captured[] = []
    const brain = await makeBrain({
      name: 'capture-vector',
      activate: async (ctx: any) => {
        ctx.registerProvider('vector', makeCapturingVectorFactory(calls))
        return true
      }
    })

    const id = await brain.add({ data: 'vec', type: NounType.Concept, vector: V() })
    const addCall = calls.find((c) => c.method === 'addItem' && c.id === id)
    expect(addCall).toBeDefined()
    expect(typeof addCall!.generation).toBe('bigint')
    expect(addCall!.generation!).toBeGreaterThan(0n)
    expect(addCall!.generation!).toBe(BigInt(brain.now().generation))

    calls.length = 0
    await brain.update({ id, vector: V() })
    const updCall = calls.find((c) => c.method === 'updateItem' && c.id === id)
    expect(updCall?.generation).toBeDefined()
    expect(updCall!.generation!).toBeGreaterThan(addCall!.generation!) // monotonic
    expect(updCall!.generation!).toBe(BigInt(brain.now().generation))

    calls.length = 0
    await brain.remove(id)
    const rmCall = calls.find((c) => c.method === 'removeItem' && c.id === id)
    expect(rmCall?.generation).toBeDefined()
    expect(rmCall!.generation!).toBeGreaterThan(updCall!.generation!)
    expect(rmCall!.generation!).toBe(BigInt(brain.now().generation))
  })
})

describe('Index operations — generation threading (operation layer)', () => {
  function makeVectorSpy() {
    const calls: Array<{ method: string; generation: bigint | undefined }> = []
    const index = {
      name: 'spy',
      async addItem(_item: any, generation?: bigint) { calls.push({ method: 'addItem', generation }); return 'x' },
      async removeItem(_id: string, generation?: bigint) { calls.push({ method: 'removeItem', generation }); return true },
      async updateItem(_item: any, generation?: bigint) { calls.push({ method: 'updateItem', generation }) }
    } as unknown as VectorIndexProvider
    return { index, calls }
  }

  it('vector add/remove/replace resolve the thunk at EXECUTE time and reuse one generation for rollback', async () => {
    const { index, calls } = makeVectorSpy()
    let current = 1n
    const op = new AddToVectorIndexOperation(index, 'id-1', [1, 2], () => current)
    current = 42n // assigned after construction, read at execute
    const rollback = await op.execute()
    expect(calls[0]).toEqual({ method: 'addItem', generation: 42n })
    current = 77n // rollback must NOT re-read — one watermark per round trip
    await rollback()
    expect(calls[1]).toEqual({ method: 'removeItem', generation: 42n })

    calls.length = 0
    const rm = new RemoveFromVectorIndexOperation(index, 'id-1', [1, 2], () => 7n)
    const rb2 = await rm.execute()
    await rb2()
    expect(calls).toEqual([
      { method: 'removeItem', generation: 7n },
      { method: 'addItem', generation: 7n }
    ])

    calls.length = 0
    const rep = new ReplaceInVectorIndexOperation(index, 'id-1', [1, 2], [3, 4], () => 9n)
    const rb3 = await rep.execute()
    await rb3()
    expect(calls).toEqual([
      { method: 'updateItem', generation: 9n },
      { method: 'updateItem', generation: 9n }
    ])
  })

  it('metadata add/remove pass the resolved generation through both halves', async () => {
    const calls: Array<{ method: string; generation: bigint | undefined }> = []
    const manager = {
      async addToIndex(_id: string, _e: any, _s?: boolean, _d?: boolean, generation?: bigint) {
        calls.push({ method: 'addToIndex', generation })
      },
      async removeFromIndex(_id: string, _m?: any, generation?: bigint) {
        calls.push({ method: 'removeFromIndex', generation })
      }
    } as unknown as MetadataIndexManager

    const add = new AddToMetadataIndexOperation(manager, 'id-1', { type: 'x' }, () => 11n)
    const rb = await add.execute()
    await rb()
    const rm = new RemoveFromMetadataIndexOperation(manager, 'id-1', { type: 'x' }, () => 12n)
    const rb2 = await rm.execute()
    await rb2()
    expect(calls).toEqual([
      { method: 'addToIndex', generation: 11n },
      { method: 'removeFromIndex', generation: 11n },
      { method: 'removeFromIndex', generation: 12n },
      { method: 'addToIndex', generation: 12n }
    ])
  })

  it('omitted thunk (legacy caller) → provider receives undefined, never a fabricated 0', async () => {
    const { index, calls } = makeVectorSpy()
    const op = new AddToVectorIndexOperation(index, 'id-1', [1, 2])
    await op.execute()
    expect(calls[0]).toEqual({ method: 'addItem', generation: undefined })
  })
})