feat(plugin): every provider write surface carries the real committed generation
The provider contract (metadata addToIndex/removeFromIndex, vector addItem/removeItem, id-mapper getOrAssign/remove) gains an optional trailing generation — evaluated lazily at operation execute time (the graph surface's thunk pattern, generalized), threaded from all 17 construction sites: undefined during generation-0 bootstrap, the real committed generation everywhere else. Optional = additive: no existing provider or caller breaks; native delta logs that stamped literal zero start hearing truth. JS twins accept the parameter with parity notes. Pins: provider doubles capture and assert nonzero monotonic generations across add/update/remove on both surfaces.
This commit is contained in:
parent
3484107462
commit
2d532684b4
7 changed files with 583 additions and 63 deletions
276
tests/unit/plugin/provider-generation.test.ts
Normal file
276
tests/unit/plugin/provider-generation.test.ts
Normal file
|
|
@ -0,0 +1,276 @@
|
|||
/**
|
||||
* Generation threading to the metadata-index and vector-index provider write
|
||||
* surfaces — the counterpart of the graph pins in
|
||||
* tests/unit/transaction/graphIndexOperations-generation.test.ts.
|
||||
*
|
||||
* The provider contract gained an optional trailing `generation?: bigint` on
|
||||
* `MetadataIndexProvider.addToIndex`/`removeFromIndex`,
|
||||
* `VectorIndexProvider.addItem`/`removeItem` (+ the feature-detected
|
||||
* `updateItem`), and the id-mapper's `getOrAssign`/`remove`. A native provider
|
||||
* with per-record delta logs stamps its durable records with it — so the value
|
||||
* arriving MUST be the real commit generation (nonzero, monotonic), never a
|
||||
* fabricated 0 and never absent on the coordinator's write paths.
|
||||
*
|
||||
* Two layers of pins:
|
||||
* 1. End-to-end: provider doubles registered via the plugin system capture
|
||||
* the generation argument during brain.add()/update()/remove() and it
|
||||
* must equal the committed watermark (`brain.now().generation`).
|
||||
* 2. Operation layer: execute-time (not construction-time) resolution, and
|
||||
* one shared generation across an op's forward + rollback halves.
|
||||
*/
|
||||
import { describe, it, expect, afterEach } from 'vitest'
|
||||
import { Brainy, NounType } from '../../../src/index.js'
|
||||
import { MetadataIndexManager } from '../../../src/utils/metadataIndex.js'
|
||||
import {
|
||||
AddToVectorIndexOperation,
|
||||
RemoveFromVectorIndexOperation,
|
||||
ReplaceInVectorIndexOperation,
|
||||
AddToMetadataIndexOperation,
|
||||
RemoveFromMetadataIndexOperation
|
||||
} from '../../../src/transaction/operations/IndexOperations.js'
|
||||
import type { VectorIndexProvider } from '../../../src/plugin.js'
|
||||
|
||||
const V = () => Array.from({ length: 384 }, () => Math.random())
|
||||
|
||||
type Captured = { method: string; id: string; generation: bigint | undefined }
|
||||
|
||||
const brains: Brainy[] = []
|
||||
afterEach(async () => {
|
||||
for (const b of brains.splice(0)) await b.close().catch(() => {})
|
||||
})
|
||||
|
||||
/** Metadata manager subclass that records the generation of every write. */
|
||||
function makeCapturingMetadataFactory(calls: Captured[]) {
|
||||
return (storage: any) => {
|
||||
class CapturingManager extends MetadataIndexManager {
|
||||
async addToIndex(id: string, entityOrMetadata: any, skipFlush = false, deferWrites = false, generation?: bigint): Promise<void> {
|
||||
calls.push({ method: 'addToIndex', id, generation })
|
||||
return super.addToIndex(id, entityOrMetadata, skipFlush, deferWrites, generation)
|
||||
}
|
||||
async removeFromIndex(id: string, metadata?: any, generation?: bigint): Promise<void> {
|
||||
calls.push({ method: 'removeFromIndex', id, generation })
|
||||
return super.removeFromIndex(id, metadata, generation)
|
||||
}
|
||||
}
|
||||
return new CapturingManager(storage)
|
||||
}
|
||||
}
|
||||
|
||||
/** Minimal vector-index double capturing the generation of every write. */
|
||||
function makeCapturingVectorFactory(calls: Captured[]) {
|
||||
return () => {
|
||||
const items = new Map<string, number[]>()
|
||||
const double: VectorIndexProvider & { updateItem(item: { id: string; vector: number[] }, generation?: bigint): Promise<void> } = {
|
||||
name: 'capture-double',
|
||||
async addItem(item, generation) {
|
||||
calls.push({ method: 'addItem', id: item.id, generation })
|
||||
items.set(item.id, item.vector as number[])
|
||||
return item.id
|
||||
},
|
||||
async removeItem(id, generation) {
|
||||
calls.push({ method: 'removeItem', id, generation })
|
||||
return items.delete(id)
|
||||
},
|
||||
async updateItem(item, generation) {
|
||||
calls.push({ method: 'updateItem', id: item.id, generation })
|
||||
items.set(item.id, item.vector)
|
||||
},
|
||||
async search() { return [] },
|
||||
size: () => items.size,
|
||||
clear: () => { items.clear() },
|
||||
async rebuild() {},
|
||||
async flush() { return 0 },
|
||||
getPersistMode: () => 'deferred' as const
|
||||
}
|
||||
return double
|
||||
}
|
||||
}
|
||||
|
||||
async function makeBrain(plugin: any): Promise<Brainy> {
|
||||
const brain = new Brainy({
|
||||
storage: { type: 'memory' },
|
||||
requireSubtype: false,
|
||||
silent: true,
|
||||
plugins: []
|
||||
})
|
||||
brain.use(plugin)
|
||||
await brain.init()
|
||||
brains.push(brain)
|
||||
return brain
|
||||
}
|
||||
|
||||
describe('Metadata-index provider — real commit generation on every write (end-to-end)', () => {
|
||||
it('add()/update()/remove() pass the nonzero, monotonic commit generation to addToIndex/removeFromIndex', async () => {
|
||||
const calls: Captured[] = []
|
||||
const brain = await makeBrain({
|
||||
name: 'capture-metadata',
|
||||
activate: async (ctx: any) => {
|
||||
ctx.registerProvider('metadataIndex', makeCapturingMetadataFactory(calls))
|
||||
return true
|
||||
}
|
||||
})
|
||||
|
||||
const id = await brain.add({ data: 'one', type: NounType.Concept, metadata: { k: 'a' }, vector: V() })
|
||||
const addCall = calls.find((c) => c.method === 'addToIndex' && c.id === id)
|
||||
expect(addCall).toBeDefined()
|
||||
expect(typeof addCall!.generation).toBe('bigint')
|
||||
expect(addCall!.generation!).toBeGreaterThan(0n)
|
||||
// Committed watermark after a single-op write IS this write's generation.
|
||||
expect(addCall!.generation!).toBe(BigInt(brain.now().generation))
|
||||
|
||||
calls.length = 0
|
||||
await brain.update({ id, metadata: { k: 'b' } })
|
||||
const updRemove = calls.find((c) => c.method === 'removeFromIndex' && c.id === id)
|
||||
const updAdd = calls.find((c) => c.method === 'addToIndex' && c.id === id)
|
||||
expect(updRemove?.generation).toBeDefined()
|
||||
expect(updAdd?.generation).toBeDefined()
|
||||
// One commit → the remove-old + add-new legs share one watermark.
|
||||
expect(updAdd!.generation!).toBe(updRemove!.generation!)
|
||||
expect(updAdd!.generation!).toBe(BigInt(brain.now().generation))
|
||||
const updateGen = updAdd!.generation!
|
||||
expect(updateGen).toBeGreaterThan(0n)
|
||||
|
||||
calls.length = 0
|
||||
await brain.remove(id)
|
||||
const rmCall = calls.find((c) => c.method === 'removeFromIndex' && c.id === id)
|
||||
expect(rmCall?.generation).toBeDefined()
|
||||
expect(rmCall!.generation!).toBeGreaterThan(updateGen) // monotonic
|
||||
expect(rmCall!.generation!).toBe(BigInt(brain.now().generation))
|
||||
})
|
||||
|
||||
it('transact() adds stamp the batch receipt generation', async () => {
|
||||
const calls: Captured[] = []
|
||||
const brain = await makeBrain({
|
||||
name: 'capture-metadata-tx',
|
||||
activate: async (ctx: any) => {
|
||||
ctx.registerProvider('metadataIndex', makeCapturingMetadataFactory(calls))
|
||||
return true
|
||||
}
|
||||
})
|
||||
|
||||
// Bootstrap honesty: init-time infrastructure writes (the VFS root) are
|
||||
// applied WITHOUT a generation — the provider must receive undefined,
|
||||
// never a fabricated 0.
|
||||
for (const c of calls) expect(c.generation).toBeUndefined()
|
||||
calls.length = 0
|
||||
|
||||
const db = await brain.transact([
|
||||
{ op: 'add', data: 'tx-one', type: NounType.Concept, vector: V() },
|
||||
{ op: 'add', data: 'tx-two', type: NounType.Concept, vector: V() }
|
||||
] as any)
|
||||
|
||||
const receiptGen = BigInt(db.receipt!.generation)
|
||||
const addGens = calls.filter((c) => c.method === 'addToIndex').map((c) => c.generation)
|
||||
expect(addGens.length).toBeGreaterThanOrEqual(2)
|
||||
for (const g of addGens) expect(g).toBe(receiptGen)
|
||||
})
|
||||
})
|
||||
|
||||
describe('Vector-index provider — real commit generation on every write (end-to-end)', () => {
|
||||
it('add()/update()/remove() pass the nonzero commit generation to addItem/updateItem/removeItem', async () => {
|
||||
const calls: Captured[] = []
|
||||
const brain = await makeBrain({
|
||||
name: 'capture-vector',
|
||||
activate: async (ctx: any) => {
|
||||
ctx.registerProvider('vector', makeCapturingVectorFactory(calls))
|
||||
return true
|
||||
}
|
||||
})
|
||||
|
||||
const id = await brain.add({ data: 'vec', type: NounType.Concept, vector: V() })
|
||||
const addCall = calls.find((c) => c.method === 'addItem' && c.id === id)
|
||||
expect(addCall).toBeDefined()
|
||||
expect(typeof addCall!.generation).toBe('bigint')
|
||||
expect(addCall!.generation!).toBeGreaterThan(0n)
|
||||
expect(addCall!.generation!).toBe(BigInt(brain.now().generation))
|
||||
|
||||
calls.length = 0
|
||||
await brain.update({ id, vector: V() })
|
||||
const updCall = calls.find((c) => c.method === 'updateItem' && c.id === id)
|
||||
expect(updCall?.generation).toBeDefined()
|
||||
expect(updCall!.generation!).toBeGreaterThan(addCall!.generation!) // monotonic
|
||||
expect(updCall!.generation!).toBe(BigInt(brain.now().generation))
|
||||
|
||||
calls.length = 0
|
||||
await brain.remove(id)
|
||||
const rmCall = calls.find((c) => c.method === 'removeItem' && c.id === id)
|
||||
expect(rmCall?.generation).toBeDefined()
|
||||
expect(rmCall!.generation!).toBeGreaterThan(updCall!.generation!)
|
||||
expect(rmCall!.generation!).toBe(BigInt(brain.now().generation))
|
||||
})
|
||||
})
|
||||
|
||||
describe('Index operations — generation threading (operation layer)', () => {
|
||||
function makeVectorSpy() {
|
||||
const calls: Array<{ method: string; generation: bigint | undefined }> = []
|
||||
const index = {
|
||||
name: 'spy',
|
||||
async addItem(_item: any, generation?: bigint) { calls.push({ method: 'addItem', generation }); return 'x' },
|
||||
async removeItem(_id: string, generation?: bigint) { calls.push({ method: 'removeItem', generation }); return true },
|
||||
async updateItem(_item: any, generation?: bigint) { calls.push({ method: 'updateItem', generation }) }
|
||||
} as unknown as VectorIndexProvider
|
||||
return { index, calls }
|
||||
}
|
||||
|
||||
it('vector add/remove/replace resolve the thunk at EXECUTE time and reuse one generation for rollback', async () => {
|
||||
const { index, calls } = makeVectorSpy()
|
||||
let current = 1n
|
||||
const op = new AddToVectorIndexOperation(index, 'id-1', [1, 2], () => current)
|
||||
current = 42n // assigned after construction, read at execute
|
||||
const rollback = await op.execute()
|
||||
expect(calls[0]).toEqual({ method: 'addItem', generation: 42n })
|
||||
current = 77n // rollback must NOT re-read — one watermark per round trip
|
||||
await rollback()
|
||||
expect(calls[1]).toEqual({ method: 'removeItem', generation: 42n })
|
||||
|
||||
calls.length = 0
|
||||
const rm = new RemoveFromVectorIndexOperation(index, 'id-1', [1, 2], () => 7n)
|
||||
const rb2 = await rm.execute()
|
||||
await rb2()
|
||||
expect(calls).toEqual([
|
||||
{ method: 'removeItem', generation: 7n },
|
||||
{ method: 'addItem', generation: 7n }
|
||||
])
|
||||
|
||||
calls.length = 0
|
||||
const rep = new ReplaceInVectorIndexOperation(index, 'id-1', [1, 2], [3, 4], () => 9n)
|
||||
const rb3 = await rep.execute()
|
||||
await rb3()
|
||||
expect(calls).toEqual([
|
||||
{ method: 'updateItem', generation: 9n },
|
||||
{ method: 'updateItem', generation: 9n }
|
||||
])
|
||||
})
|
||||
|
||||
it('metadata add/remove pass the resolved generation through both halves', async () => {
|
||||
const calls: Array<{ method: string; generation: bigint | undefined }> = []
|
||||
const manager = {
|
||||
async addToIndex(_id: string, _e: any, _s?: boolean, _d?: boolean, generation?: bigint) {
|
||||
calls.push({ method: 'addToIndex', generation })
|
||||
},
|
||||
async removeFromIndex(_id: string, _m?: any, generation?: bigint) {
|
||||
calls.push({ method: 'removeFromIndex', generation })
|
||||
}
|
||||
} as unknown as MetadataIndexManager
|
||||
|
||||
const add = new AddToMetadataIndexOperation(manager, 'id-1', { type: 'x' }, () => 11n)
|
||||
const rb = await add.execute()
|
||||
await rb()
|
||||
const rm = new RemoveFromMetadataIndexOperation(manager, 'id-1', { type: 'x' }, () => 12n)
|
||||
const rb2 = await rm.execute()
|
||||
await rb2()
|
||||
expect(calls).toEqual([
|
||||
{ method: 'addToIndex', generation: 11n },
|
||||
{ method: 'removeFromIndex', generation: 11n },
|
||||
{ method: 'removeFromIndex', generation: 12n },
|
||||
{ method: 'addToIndex', generation: 12n }
|
||||
])
|
||||
})
|
||||
|
||||
it('omitted thunk (legacy caller) → provider receives undefined, never a fabricated 0', async () => {
|
||||
const { index, calls } = makeVectorSpy()
|
||||
const op = new AddToVectorIndexOperation(index, 'id-1', [1, 2])
|
||||
await op.execute()
|
||||
expect(calls[0]).toEqual({ method: 'addItem', generation: undefined })
|
||||
})
|
||||
})
|
||||
Loading…
Add table
Add a link
Reference in a new issue