/** * Generation threading to the metadata-index and vector-index provider write * surfaces — the counterpart of the graph pins in * tests/unit/transaction/graphIndexOperations-generation.test.ts. * * The provider contract gained an optional trailing `generation?: bigint` on * `MetadataIndexProvider.addToIndex`/`removeFromIndex`, * `VectorIndexProvider.addItem`/`removeItem` (+ the feature-detected * `updateItem`), and the id-mapper's `getOrAssign`/`remove`. A native provider * with per-record delta logs stamps its durable records with it — so the value * arriving MUST be the real commit generation (nonzero, monotonic), never a * fabricated 0 and never absent on the coordinator's write paths. * * Two layers of pins: * 1. End-to-end: provider doubles registered via the plugin system capture * the generation argument during brain.add()/update()/remove() and it * must equal the committed watermark (`brain.now().generation`). * 2. Operation layer: execute-time (not construction-time) resolution, and * one shared generation across an op's forward + rollback halves. */ import { describe, it, expect, afterEach } from 'vitest' import { Brainy, NounType } from '../../../src/index.js' import { MetadataIndexManager } from '../../../src/utils/metadataIndex.js' import { AddToVectorIndexOperation, RemoveFromVectorIndexOperation, ReplaceInVectorIndexOperation, AddToMetadataIndexOperation, RemoveFromMetadataIndexOperation } from '../../../src/transaction/operations/IndexOperations.js' import type { VectorIndexProvider } from '../../../src/plugin.js' const V = () => Array.from({ length: 384 }, () => Math.random()) type Captured = { method: string; id: string; generation: bigint | undefined } const brains: Brainy[] = [] afterEach(async () => { for (const b of brains.splice(0)) await b.close().catch(() => {}) }) /** Metadata manager subclass that records the generation of every write. */ function makeCapturingMetadataFactory(calls: Captured[]) { return (storage: any) => { class CapturingManager extends MetadataIndexManager { async addToIndex(id: string, entityOrMetadata: any, skipFlush = false, deferWrites = false, generation?: bigint): Promise { calls.push({ method: 'addToIndex', id, generation }) return super.addToIndex(id, entityOrMetadata, skipFlush, deferWrites, generation) } async removeFromIndex(id: string, metadata?: any, generation?: bigint): Promise { calls.push({ method: 'removeFromIndex', id, generation }) return super.removeFromIndex(id, metadata, generation) } } return new CapturingManager(storage) } } /** Minimal vector-index double capturing the generation of every write. */ function makeCapturingVectorFactory(calls: Captured[]) { return () => { const items = new Map() const double: VectorIndexProvider & { updateItem(item: { id: string; vector: number[] }, generation?: bigint): Promise } = { name: 'capture-double', async addItem(item, generation) { calls.push({ method: 'addItem', id: item.id, generation }) items.set(item.id, item.vector as number[]) return item.id }, async removeItem(id, generation) { calls.push({ method: 'removeItem', id, generation }) return items.delete(id) }, async updateItem(item, generation) { calls.push({ method: 'updateItem', id: item.id, generation }) items.set(item.id, item.vector) }, async search() { return [] }, size: () => items.size, clear: () => { items.clear() }, async rebuild() {}, async flush() { return 0 }, getPersistMode: () => 'deferred' as const } return double } } async function makeBrain(plugin: any): Promise { const brain = new Brainy({ storage: { type: 'memory' }, requireSubtype: false, silent: true, plugins: [] }) brain.use(plugin) await brain.init() brains.push(brain) return brain } describe('Metadata-index provider — real commit generation on every write (end-to-end)', () => { it('add()/update()/remove() pass the nonzero, monotonic commit generation to addToIndex/removeFromIndex', async () => { const calls: Captured[] = [] const brain = await makeBrain({ name: 'capture-metadata', activate: async (ctx: any) => { ctx.registerProvider('metadataIndex', makeCapturingMetadataFactory(calls)) return true } }) const id = await brain.add({ data: 'one', type: NounType.Concept, metadata: { k: 'a' }, vector: V() }) const addCall = calls.find((c) => c.method === 'addToIndex' && c.id === id) expect(addCall).toBeDefined() expect(typeof addCall!.generation).toBe('bigint') expect(addCall!.generation!).toBeGreaterThan(0n) // Committed watermark after a single-op write IS this write's generation. expect(addCall!.generation!).toBe(BigInt(brain.now().generation)) calls.length = 0 await brain.update({ id, metadata: { k: 'b' } }) const updRemove = calls.find((c) => c.method === 'removeFromIndex' && c.id === id) const updAdd = calls.find((c) => c.method === 'addToIndex' && c.id === id) expect(updRemove?.generation).toBeDefined() expect(updAdd?.generation).toBeDefined() // One commit → the remove-old + add-new legs share one watermark. expect(updAdd!.generation!).toBe(updRemove!.generation!) expect(updAdd!.generation!).toBe(BigInt(brain.now().generation)) const updateGen = updAdd!.generation! expect(updateGen).toBeGreaterThan(0n) calls.length = 0 await brain.remove(id) const rmCall = calls.find((c) => c.method === 'removeFromIndex' && c.id === id) expect(rmCall?.generation).toBeDefined() expect(rmCall!.generation!).toBeGreaterThan(updateGen) // monotonic expect(rmCall!.generation!).toBe(BigInt(brain.now().generation)) }) it('transact() adds stamp the batch receipt generation', async () => { const calls: Captured[] = [] const brain = await makeBrain({ name: 'capture-metadata-tx', activate: async (ctx: any) => { ctx.registerProvider('metadataIndex', makeCapturingMetadataFactory(calls)) return true } }) // Bootstrap honesty: init-time infrastructure writes (the VFS root) are // applied WITHOUT a generation — the provider must receive undefined, // never a fabricated 0. for (const c of calls) expect(c.generation).toBeUndefined() calls.length = 0 const db = await brain.transact([ { op: 'add', data: 'tx-one', type: NounType.Concept, vector: V() }, { op: 'add', data: 'tx-two', type: NounType.Concept, vector: V() } ] as any) const receiptGen = BigInt(db.receipt!.generation) const addGens = calls.filter((c) => c.method === 'addToIndex').map((c) => c.generation) expect(addGens.length).toBeGreaterThanOrEqual(2) for (const g of addGens) expect(g).toBe(receiptGen) }) }) describe('Vector-index provider — real commit generation on every write (end-to-end)', () => { it('add()/update()/remove() pass the nonzero commit generation to addItem/updateItem/removeItem', async () => { const calls: Captured[] = [] const brain = await makeBrain({ name: 'capture-vector', activate: async (ctx: any) => { ctx.registerProvider('vector', makeCapturingVectorFactory(calls)) return true } }) const id = await brain.add({ data: 'vec', type: NounType.Concept, vector: V() }) const addCall = calls.find((c) => c.method === 'addItem' && c.id === id) expect(addCall).toBeDefined() expect(typeof addCall!.generation).toBe('bigint') expect(addCall!.generation!).toBeGreaterThan(0n) expect(addCall!.generation!).toBe(BigInt(brain.now().generation)) calls.length = 0 await brain.update({ id, vector: V() }) const updCall = calls.find((c) => c.method === 'updateItem' && c.id === id) expect(updCall?.generation).toBeDefined() expect(updCall!.generation!).toBeGreaterThan(addCall!.generation!) // monotonic expect(updCall!.generation!).toBe(BigInt(brain.now().generation)) calls.length = 0 await brain.remove(id) const rmCall = calls.find((c) => c.method === 'removeItem' && c.id === id) expect(rmCall?.generation).toBeDefined() expect(rmCall!.generation!).toBeGreaterThan(updCall!.generation!) expect(rmCall!.generation!).toBe(BigInt(brain.now().generation)) }) }) describe('Index operations — generation threading (operation layer)', () => { function makeVectorSpy() { const calls: Array<{ method: string; generation: bigint | undefined }> = [] const index = { name: 'spy', async addItem(_item: any, generation?: bigint) { calls.push({ method: 'addItem', generation }); return 'x' }, async removeItem(_id: string, generation?: bigint) { calls.push({ method: 'removeItem', generation }); return true }, async updateItem(_item: any, generation?: bigint) { calls.push({ method: 'updateItem', generation }) } } as unknown as VectorIndexProvider return { index, calls } } it('vector add/remove/replace resolve the thunk at EXECUTE time and reuse one generation for rollback', async () => { const { index, calls } = makeVectorSpy() let current = 1n const op = new AddToVectorIndexOperation(index, 'id-1', [1, 2], () => current) current = 42n // assigned after construction, read at execute const rollback = await op.execute() expect(calls[0]).toEqual({ method: 'addItem', generation: 42n }) current = 77n // rollback must NOT re-read — one watermark per round trip await rollback() expect(calls[1]).toEqual({ method: 'removeItem', generation: 42n }) calls.length = 0 const rm = new RemoveFromVectorIndexOperation(index, 'id-1', [1, 2], () => 7n) const rb2 = await rm.execute() await rb2() expect(calls).toEqual([ { method: 'removeItem', generation: 7n }, { method: 'addItem', generation: 7n } ]) calls.length = 0 const rep = new ReplaceInVectorIndexOperation(index, 'id-1', [1, 2], [3, 4], () => 9n) const rb3 = await rep.execute() await rb3() expect(calls).toEqual([ { method: 'updateItem', generation: 9n }, { method: 'updateItem', generation: 9n } ]) }) it('metadata add/remove pass the resolved generation through both halves', async () => { const calls: Array<{ method: string; generation: bigint | undefined }> = [] const manager = { async addToIndex(_id: string, _e: any, _s?: boolean, _d?: boolean, generation?: bigint) { calls.push({ method: 'addToIndex', generation }) }, async removeFromIndex(_id: string, _m?: any, generation?: bigint) { calls.push({ method: 'removeFromIndex', generation }) } } as unknown as MetadataIndexManager const add = new AddToMetadataIndexOperation(manager, 'id-1', { type: 'x' }, () => 11n) const rb = await add.execute() await rb() const rm = new RemoveFromMetadataIndexOperation(manager, 'id-1', { type: 'x' }, () => 12n) const rb2 = await rm.execute() await rb2() expect(calls).toEqual([ { method: 'addToIndex', generation: 11n }, { method: 'removeFromIndex', generation: 11n }, { method: 'removeFromIndex', generation: 12n }, { method: 'addToIndex', generation: 12n } ]) }) it('omitted thunk (legacy caller) → provider receives undefined, never a fabricated 0', async () => { const { index, calls } = makeVectorSpy() const op = new AddToVectorIndexOperation(index, 'id-1', [1, 2]) await op.execute() expect(calls[0]).toEqual({ method: 'addItem', generation: undefined }) }) })