David Snelling
2d532684b4
The provider contract (metadata addToIndex/removeFromIndex, vector addItem/removeItem, id-mapper getOrAssign/remove) gains an optional trailing generation — evaluated lazily at operation execute time (the graph surface's thunk pattern, generalized), threaded from all 17 construction sites: undefined during generation-0 bootstrap, the real committed generation everywhere else. Optional = additive: no existing provider or caller breaks; native delta logs that stamped literal zero start hearing truth. JS twins accept the parameter with parity notes. Pins: provider doubles capture and assert nonzero monotonic generations across add/update/remove on both surfaces.
276 lines
12 KiB
TypeScript
276 lines
12 KiB
TypeScript
/**
|
|
* Generation threading to the metadata-index and vector-index provider write
|
|
* surfaces — the counterpart of the graph pins in
|
|
* tests/unit/transaction/graphIndexOperations-generation.test.ts.
|
|
*
|
|
* The provider contract gained an optional trailing `generation?: bigint` on
|
|
* `MetadataIndexProvider.addToIndex`/`removeFromIndex`,
|
|
* `VectorIndexProvider.addItem`/`removeItem` (+ the feature-detected
|
|
* `updateItem`), and the id-mapper's `getOrAssign`/`remove`. A native provider
|
|
* with per-record delta logs stamps its durable records with it — so the value
|
|
* arriving MUST be the real commit generation (nonzero, monotonic), never a
|
|
* fabricated 0 and never absent on the coordinator's write paths.
|
|
*
|
|
* Two layers of pins:
|
|
* 1. End-to-end: provider doubles registered via the plugin system capture
|
|
* the generation argument during brain.add()/update()/remove() and it
|
|
* must equal the committed watermark (`brain.now().generation`).
|
|
* 2. Operation layer: execute-time (not construction-time) resolution, and
|
|
* one shared generation across an op's forward + rollback halves.
|
|
*/
|
|
import { describe, it, expect, afterEach } from 'vitest'
|
|
import { Brainy, NounType } from '../../../src/index.js'
|
|
import { MetadataIndexManager } from '../../../src/utils/metadataIndex.js'
|
|
import {
|
|
AddToVectorIndexOperation,
|
|
RemoveFromVectorIndexOperation,
|
|
ReplaceInVectorIndexOperation,
|
|
AddToMetadataIndexOperation,
|
|
RemoveFromMetadataIndexOperation
|
|
} from '../../../src/transaction/operations/IndexOperations.js'
|
|
import type { VectorIndexProvider } from '../../../src/plugin.js'
|
|
|
|
const V = () => Array.from({ length: 384 }, () => Math.random())
|
|
|
|
type Captured = { method: string; id: string; generation: bigint | undefined }
|
|
|
|
const brains: Brainy[] = []
|
|
afterEach(async () => {
|
|
for (const b of brains.splice(0)) await b.close().catch(() => {})
|
|
})
|
|
|
|
/** Metadata manager subclass that records the generation of every write. */
|
|
function makeCapturingMetadataFactory(calls: Captured[]) {
|
|
return (storage: any) => {
|
|
class CapturingManager extends MetadataIndexManager {
|
|
async addToIndex(id: string, entityOrMetadata: any, skipFlush = false, deferWrites = false, generation?: bigint): Promise<void> {
|
|
calls.push({ method: 'addToIndex', id, generation })
|
|
return super.addToIndex(id, entityOrMetadata, skipFlush, deferWrites, generation)
|
|
}
|
|
async removeFromIndex(id: string, metadata?: any, generation?: bigint): Promise<void> {
|
|
calls.push({ method: 'removeFromIndex', id, generation })
|
|
return super.removeFromIndex(id, metadata, generation)
|
|
}
|
|
}
|
|
return new CapturingManager(storage)
|
|
}
|
|
}
|
|
|
|
/** Minimal vector-index double capturing the generation of every write. */
|
|
function makeCapturingVectorFactory(calls: Captured[]) {
|
|
return () => {
|
|
const items = new Map<string, number[]>()
|
|
const double: VectorIndexProvider & { updateItem(item: { id: string; vector: number[] }, generation?: bigint): Promise<void> } = {
|
|
name: 'capture-double',
|
|
async addItem(item, generation) {
|
|
calls.push({ method: 'addItem', id: item.id, generation })
|
|
items.set(item.id, item.vector as number[])
|
|
return item.id
|
|
},
|
|
async removeItem(id, generation) {
|
|
calls.push({ method: 'removeItem', id, generation })
|
|
return items.delete(id)
|
|
},
|
|
async updateItem(item, generation) {
|
|
calls.push({ method: 'updateItem', id: item.id, generation })
|
|
items.set(item.id, item.vector)
|
|
},
|
|
async search() { return [] },
|
|
size: () => items.size,
|
|
clear: () => { items.clear() },
|
|
async rebuild() {},
|
|
async flush() { return 0 },
|
|
getPersistMode: () => 'deferred' as const
|
|
}
|
|
return double
|
|
}
|
|
}
|
|
|
|
async function makeBrain(plugin: any): Promise<Brainy> {
|
|
const brain = new Brainy({
|
|
storage: { type: 'memory' },
|
|
requireSubtype: false,
|
|
silent: true,
|
|
plugins: []
|
|
})
|
|
brain.use(plugin)
|
|
await brain.init()
|
|
brains.push(brain)
|
|
return brain
|
|
}
|
|
|
|
describe('Metadata-index provider — real commit generation on every write (end-to-end)', () => {
|
|
it('add()/update()/remove() pass the nonzero, monotonic commit generation to addToIndex/removeFromIndex', async () => {
|
|
const calls: Captured[] = []
|
|
const brain = await makeBrain({
|
|
name: 'capture-metadata',
|
|
activate: async (ctx: any) => {
|
|
ctx.registerProvider('metadataIndex', makeCapturingMetadataFactory(calls))
|
|
return true
|
|
}
|
|
})
|
|
|
|
const id = await brain.add({ data: 'one', type: NounType.Concept, metadata: { k: 'a' }, vector: V() })
|
|
const addCall = calls.find((c) => c.method === 'addToIndex' && c.id === id)
|
|
expect(addCall).toBeDefined()
|
|
expect(typeof addCall!.generation).toBe('bigint')
|
|
expect(addCall!.generation!).toBeGreaterThan(0n)
|
|
// Committed watermark after a single-op write IS this write's generation.
|
|
expect(addCall!.generation!).toBe(BigInt(brain.now().generation))
|
|
|
|
calls.length = 0
|
|
await brain.update({ id, metadata: { k: 'b' } })
|
|
const updRemove = calls.find((c) => c.method === 'removeFromIndex' && c.id === id)
|
|
const updAdd = calls.find((c) => c.method === 'addToIndex' && c.id === id)
|
|
expect(updRemove?.generation).toBeDefined()
|
|
expect(updAdd?.generation).toBeDefined()
|
|
// One commit → the remove-old + add-new legs share one watermark.
|
|
expect(updAdd!.generation!).toBe(updRemove!.generation!)
|
|
expect(updAdd!.generation!).toBe(BigInt(brain.now().generation))
|
|
const updateGen = updAdd!.generation!
|
|
expect(updateGen).toBeGreaterThan(0n)
|
|
|
|
calls.length = 0
|
|
await brain.remove(id)
|
|
const rmCall = calls.find((c) => c.method === 'removeFromIndex' && c.id === id)
|
|
expect(rmCall?.generation).toBeDefined()
|
|
expect(rmCall!.generation!).toBeGreaterThan(updateGen) // monotonic
|
|
expect(rmCall!.generation!).toBe(BigInt(brain.now().generation))
|
|
})
|
|
|
|
it('transact() adds stamp the batch receipt generation', async () => {
|
|
const calls: Captured[] = []
|
|
const brain = await makeBrain({
|
|
name: 'capture-metadata-tx',
|
|
activate: async (ctx: any) => {
|
|
ctx.registerProvider('metadataIndex', makeCapturingMetadataFactory(calls))
|
|
return true
|
|
}
|
|
})
|
|
|
|
// Bootstrap honesty: init-time infrastructure writes (the VFS root) are
|
|
// applied WITHOUT a generation — the provider must receive undefined,
|
|
// never a fabricated 0.
|
|
for (const c of calls) expect(c.generation).toBeUndefined()
|
|
calls.length = 0
|
|
|
|
const db = await brain.transact([
|
|
{ op: 'add', data: 'tx-one', type: NounType.Concept, vector: V() },
|
|
{ op: 'add', data: 'tx-two', type: NounType.Concept, vector: V() }
|
|
] as any)
|
|
|
|
const receiptGen = BigInt(db.receipt!.generation)
|
|
const addGens = calls.filter((c) => c.method === 'addToIndex').map((c) => c.generation)
|
|
expect(addGens.length).toBeGreaterThanOrEqual(2)
|
|
for (const g of addGens) expect(g).toBe(receiptGen)
|
|
})
|
|
})
|
|
|
|
describe('Vector-index provider — real commit generation on every write (end-to-end)', () => {
|
|
it('add()/update()/remove() pass the nonzero commit generation to addItem/updateItem/removeItem', async () => {
|
|
const calls: Captured[] = []
|
|
const brain = await makeBrain({
|
|
name: 'capture-vector',
|
|
activate: async (ctx: any) => {
|
|
ctx.registerProvider('vector', makeCapturingVectorFactory(calls))
|
|
return true
|
|
}
|
|
})
|
|
|
|
const id = await brain.add({ data: 'vec', type: NounType.Concept, vector: V() })
|
|
const addCall = calls.find((c) => c.method === 'addItem' && c.id === id)
|
|
expect(addCall).toBeDefined()
|
|
expect(typeof addCall!.generation).toBe('bigint')
|
|
expect(addCall!.generation!).toBeGreaterThan(0n)
|
|
expect(addCall!.generation!).toBe(BigInt(brain.now().generation))
|
|
|
|
calls.length = 0
|
|
await brain.update({ id, vector: V() })
|
|
const updCall = calls.find((c) => c.method === 'updateItem' && c.id === id)
|
|
expect(updCall?.generation).toBeDefined()
|
|
expect(updCall!.generation!).toBeGreaterThan(addCall!.generation!) // monotonic
|
|
expect(updCall!.generation!).toBe(BigInt(brain.now().generation))
|
|
|
|
calls.length = 0
|
|
await brain.remove(id)
|
|
const rmCall = calls.find((c) => c.method === 'removeItem' && c.id === id)
|
|
expect(rmCall?.generation).toBeDefined()
|
|
expect(rmCall!.generation!).toBeGreaterThan(updCall!.generation!)
|
|
expect(rmCall!.generation!).toBe(BigInt(brain.now().generation))
|
|
})
|
|
})
|
|
|
|
describe('Index operations — generation threading (operation layer)', () => {
|
|
function makeVectorSpy() {
|
|
const calls: Array<{ method: string; generation: bigint | undefined }> = []
|
|
const index = {
|
|
name: 'spy',
|
|
async addItem(_item: any, generation?: bigint) { calls.push({ method: 'addItem', generation }); return 'x' },
|
|
async removeItem(_id: string, generation?: bigint) { calls.push({ method: 'removeItem', generation }); return true },
|
|
async updateItem(_item: any, generation?: bigint) { calls.push({ method: 'updateItem', generation }) }
|
|
} as unknown as VectorIndexProvider
|
|
return { index, calls }
|
|
}
|
|
|
|
it('vector add/remove/replace resolve the thunk at EXECUTE time and reuse one generation for rollback', async () => {
|
|
const { index, calls } = makeVectorSpy()
|
|
let current = 1n
|
|
const op = new AddToVectorIndexOperation(index, 'id-1', [1, 2], () => current)
|
|
current = 42n // assigned after construction, read at execute
|
|
const rollback = await op.execute()
|
|
expect(calls[0]).toEqual({ method: 'addItem', generation: 42n })
|
|
current = 77n // rollback must NOT re-read — one watermark per round trip
|
|
await rollback()
|
|
expect(calls[1]).toEqual({ method: 'removeItem', generation: 42n })
|
|
|
|
calls.length = 0
|
|
const rm = new RemoveFromVectorIndexOperation(index, 'id-1', [1, 2], () => 7n)
|
|
const rb2 = await rm.execute()
|
|
await rb2()
|
|
expect(calls).toEqual([
|
|
{ method: 'removeItem', generation: 7n },
|
|
{ method: 'addItem', generation: 7n }
|
|
])
|
|
|
|
calls.length = 0
|
|
const rep = new ReplaceInVectorIndexOperation(index, 'id-1', [1, 2], [3, 4], () => 9n)
|
|
const rb3 = await rep.execute()
|
|
await rb3()
|
|
expect(calls).toEqual([
|
|
{ method: 'updateItem', generation: 9n },
|
|
{ method: 'updateItem', generation: 9n }
|
|
])
|
|
})
|
|
|
|
it('metadata add/remove pass the resolved generation through both halves', async () => {
|
|
const calls: Array<{ method: string; generation: bigint | undefined }> = []
|
|
const manager = {
|
|
async addToIndex(_id: string, _e: any, _s?: boolean, _d?: boolean, generation?: bigint) {
|
|
calls.push({ method: 'addToIndex', generation })
|
|
},
|
|
async removeFromIndex(_id: string, _m?: any, generation?: bigint) {
|
|
calls.push({ method: 'removeFromIndex', generation })
|
|
}
|
|
} as unknown as MetadataIndexManager
|
|
|
|
const add = new AddToMetadataIndexOperation(manager, 'id-1', { type: 'x' }, () => 11n)
|
|
const rb = await add.execute()
|
|
await rb()
|
|
const rm = new RemoveFromMetadataIndexOperation(manager, 'id-1', { type: 'x' }, () => 12n)
|
|
const rb2 = await rm.execute()
|
|
await rb2()
|
|
expect(calls).toEqual([
|
|
{ method: 'addToIndex', generation: 11n },
|
|
{ method: 'removeFromIndex', generation: 11n },
|
|
{ method: 'removeFromIndex', generation: 12n },
|
|
{ method: 'addToIndex', generation: 12n }
|
|
])
|
|
})
|
|
|
|
it('omitted thunk (legacy caller) → provider receives undefined, never a fabricated 0', async () => {
|
|
const { index, calls } = makeVectorSpy()
|
|
const op = new AddToVectorIndexOperation(index, 'id-1', [1, 2])
|
|
await op.execute()
|
|
expect(calls[0]).toEqual({ method: 'addItem', generation: undefined })
|
|
})
|
|
})
|