soulcraftlabs/open-brainy
2
0
Fork 0
Code Issues 1 Pull requests Projects Releases 342 Packages Wiki Activity Actions 5

feat(plugin): every provider write surface carries the real committed generation

Browse source 
The provider contract (metadata addToIndex/removeFromIndex, vector
addItem/removeItem, id-mapper getOrAssign/remove) gains an optional
trailing generation — evaluated lazily at operation execute time (the
graph surface's thunk pattern, generalized), threaded from all 17
construction sites: undefined during generation-0 bootstrap, the real
committed generation everywhere else. Optional = additive: no existing
provider or caller breaks; native delta logs that stamped literal zero
start hearing truth. JS twins accept the parameter with parity notes.
Pins: provider doubles capture and assert nonzero monotonic generations
across add/update/remove on both surfaces.
This commit is contained in:
David Snelling 2026-08-10 09:29:06 -07:00
parent 3484107462
commit 2d532684b4
7 changed files with 583 additions and 63 deletions
Download patch file Download diff file Expand all files Collapse all files

148
src/transaction/operations/IndexOperations.ts
View file

@ -56,16 +56,33 @@ function resolveVectorProviderId(index: VectorIndexProvider): string {
* or timing trace see which engine actually ran, never a fossil name from
* whichever engine happened to be active when this op class was written.
*
* Generation: `generationFn` is resolved at execute time (not construction) so
* the write is stamped at the transaction's in-flight commit generation —
* which the generation store only assigns once the batch begins executing.
* The same generation is reused for the rollback removal, so an add and its
* undo reference one watermark in a provider's per-record delta log (the
* exact pattern the graph operations established).
*
* Rollback strategy:
* - Remove item from index
*/
export class AddToVectorIndexOperation implements Operation {
readonly name: string
/**
* @param index - The vector-index provider (JS HNSW or native).
* @param id - The entity's UUID.
* @param vector - The vector to index.
* @param generationFn - OPTIONAL: resolves the commit generation to stamp
* this write at, evaluated when the operation executes (see class note).
* Absent -> the provider receives no generation (undefined), never a
* fabricated 0.
*/
constructor(
private readonly index: VectorIndexProvider,
private readonly id: string,
private readonly vector: number[]
private readonly vector: number[],
private readonly generationFn?: () => bigint | undefined
) {
this.name = `AddToVectorIndex(${resolveVectorProviderId(index)})`
}
@ -74,14 +91,18 @@ export class AddToVectorIndexOperation implements Operation {
// Check if item already exists (for rollback decision)
const existed = await this.itemExists(this.id)
// Stamp this write at the in-flight commit generation; reuse it for the
// rollback so add + undo reference the same watermark.
const generation = this.generationFn?.()
// Add to index
await this.index.addItem({ id: this.id, vector: this.vector })
await this.index.addItem({ id: this.id, vector: this.vector }, generation)
// Return rollback action
return async () => {
if (!existed) {
// Remove newly added item
await this.index.removeItem(this.id)
await this.index.removeItem(this.id, generation)
}
// If item existed before, we don't rollback (update is OK)
// This prevents index corruption from removing pre-existing items
@ -131,22 +152,34 @@ export class AddToVectorIndexOperation implements Operation {
export class RemoveFromVectorIndexOperation implements Operation {
readonly name: string
/**
* @param index - The vector-index provider (JS HNSW or native).
* @param id - The entity's UUID.
* @param vector - The removed vector (required for rollback re-add).
* @param generationFn - Resolves the commit generation for this removal,
* evaluated when the operation executes; reused for the rollback re-add
* so the round trip references one watermark.
*/
constructor(
private readonly index: VectorIndexProvider,
private readonly id: string,
private readonly vector: number[] // Required for rollback
private readonly vector: number[], // Required for rollback
private readonly generationFn?: () => bigint | undefined
) {
this.name = `RemoveFromVectorIndex(${resolveVectorProviderId(index)})`
}
async execute(): Promise<RollbackAction> {
// Resolve the removal generation once; reuse it for the rollback re-add.
const generation = this.generationFn?.()
// Remove from index
await this.index.removeItem(this.id)
await this.index.removeItem(this.id, generation)
// Return rollback action
return async () => {
// Re-add item with original vector
await this.index.addItem({ id: this.id, vector: this.vector })
await this.index.addItem({ id: this.id, vector: this.vector }, generation)
}
}
}
@ -198,11 +231,22 @@ export class RemoveFromVectorIndexOperation implements Operation {
export class ReplaceInVectorIndexOperation implements Operation {
readonly name: string
/**
* @param index - The vector-index provider (JS HNSW or native).
* @param id - The entity's UUID.
* @param oldVector - The pre-update vector (required for rollback).
* @param newVector - The replacement vector.
* @param generationFn - Resolves the commit generation to stamp this write
* at, evaluated when the operation executes and reused across both
* execute branches AND the rollback — one watermark for the whole
* replace round trip.
*/
constructor(
private readonly index: VectorIndexProvider,
private readonly id: string,
private readonly oldVector: number[], // Required for rollback
private readonly newVector: number[]
private readonly newVector: number[],
private readonly generationFn?: () => bigint | undefined
) {
this.name = `ReplaceInVectorIndex(${resolveVectorProviderId(index)})`
}
@ -210,32 +254,36 @@ export class ReplaceInVectorIndexOperation implements Operation {
async execute(): Promise<RollbackAction> {
// Feature-detect the in-place capability — optional on the provider
// contract, like `getItem`/`setPersistMode` (Brainy's JS HNSW index
// ships it; a native provider may not have yet).
// ships it; a native provider may not have yet). The capability carries
// the same optional trailing generation as the required write surface.
const index = this.index as VectorIndexProvider & {
updateItem?: (item: { id: string; vector: number[] }) => Promise<void>
updateItem?: (item: { id: string; vector: number[] }, generation?: bigint) => Promise<void>
}
// One commit generation for the whole replace (both branches + rollback).
const generation = this.generationFn?.()
if (typeof index.updateItem === 'function') {
// Atomic path: one in-place call, the row never leaves the index.
await index.updateItem({ id: this.id, vector: this.newVector })
await index.updateItem({ id: this.id, vector: this.newVector }, generation)
return async () => {
// Restore the declared before-state in place (see class JSDoc for
// the item-did-not-exist posture).
await index.updateItem!({ id: this.id, vector: this.oldVector })
await index.updateItem!({ id: this.id, vector: this.oldVector }, generation)
}
}
// Fallback seam: remove+add ADJACENT within this single op — no other
// transaction operation can interleave between them (see class JSDoc).
await this.index.removeItem(this.id)
await this.index.addItem({ id: this.id, vector: this.newVector })
await this.index.removeItem(this.id, generation)
await this.index.addItem({ id: this.id, vector: this.newVector }, generation)
return async () => {
// updateItem-style restore via the same adjacent pair, back to the
// declared before-state.
await this.index.removeItem(this.id)
await this.index.addItem({ id: this.id, vector: this.oldVector })
await this.index.removeItem(this.id, generation)
await this.index.addItem({ id: this.id, vector: this.oldVector }, generation)
}
}
}
@ -243,26 +291,43 @@ export class ReplaceInVectorIndexOperation implements Operation {
/**
* Add to metadata index with rollback support
*
* Generation: `generationFn` is resolved at execute time (not construction) —
* see {@link AddToVectorIndexOperation}'s class note; the same generation is
* reused for the rollback removal so add + undo reference one watermark in a
* provider's per-record delta log.
*
* Rollback strategy:
* - Remove item from index
*/
export class AddToMetadataIndexOperation implements Operation {
readonly name = 'AddToMetadataIndex'
/**
* @param index - The metadata-index manager (JS baseline or a registered provider).
* @param id - The entity's UUID.
* @param entity - Entity or metadata structure to index.
* @param generationFn - Resolves the commit generation to stamp this write
* at, evaluated when the operation executes.
*/
constructor(
private readonly index: MetadataIndexManager,
private readonly id: string,
private readonly entity: any // Entity or metadata structure
private readonly entity: any, // Entity or metadata structure
private readonly generationFn?: () => bigint | undefined
) {}
async execute(): Promise<RollbackAction> {
// Stamp this write at the in-flight commit generation; reuse it for the
// rollback so add + undo reference the same watermark.
const generation = this.generationFn?.()
// Add to metadata index (skipFlush=true for transaction atomicity)
await this.index.addToIndex(this.id, this.entity, true)
await this.index.addToIndex(this.id, this.entity, true, false, generation)
// Return rollback action
return async () => {
// Remove from metadata index
await this.index.removeFromIndex(this.id, this.entity)
await this.index.removeFromIndex(this.id, this.entity, generation)
}
}
}
@ -270,26 +335,41 @@ export class AddToMetadataIndexOperation implements Operation {
/**
* Remove from metadata index with rollback support
*
* Generation: resolved at execute time and reused for the rollback re-add —
* one watermark for the removal round trip (see
* {@link AddToMetadataIndexOperation}).
*
* Rollback strategy:
* - Re-add item to index with original metadata
*/
export class RemoveFromMetadataIndexOperation implements Operation {
readonly name = 'RemoveFromMetadataIndex'
/**
* @param index - The metadata-index manager (JS baseline or a registered provider).
* @param id - The entity's UUID.
* @param entity - The entity/metadata being removed (required for rollback).
* @param generationFn - Resolves the commit generation for this removal,
* evaluated when the operation executes.
*/
constructor(
private readonly index: MetadataIndexManager,
private readonly id: string,
private readonly entity: any // Required for rollback
private readonly entity: any, // Required for rollback
private readonly generationFn?: () => bigint | undefined
) {}
async execute(): Promise<RollbackAction> {
// Resolve the removal generation once; reuse it for the rollback re-add.
const generation = this.generationFn?.()
// Remove from metadata index
await this.index.removeFromIndex(this.id, this.entity)
await this.index.removeFromIndex(this.id, this.entity, generation)
// Return rollback action
return async () => {
// Re-add with original metadata (skipFlush=true)
await this.index.addToIndex(this.id, this.entity, true)
await this.index.addToIndex(this.id, this.entity, true, false, generation)
}
}
}
@ -358,7 +438,7 @@ export class AddToGraphIndexOperation implements Operation {
// Stamp this edge at the in-flight commit generation; reuse it for the
// rollback so add + undo reference the same watermark. Endpoint ints
// resolve HERE — after any same-batch adds have applied.
const generation = this.generationFn()
const generation = this.generationFn?.()
const { sourceInt, targetInt } = resolveEndpointInts(this.endpointInts)
const verbInt = await this.index.addVerb(this.verb, sourceInt, targetInt, generation)
this.onVerbInt?.(verbInt)
@ -407,7 +487,7 @@ export class RemoveFromGraphIndexOperation implements Operation {
// Resolve the removal generation once; reuse it for the rollback re-add.
// Endpoint ints resolve HERE (after any same-batch adds applied) and are
// captured for the rollback, whose re-add must use the same mappings.
const generation = this.generationFn()
const generation = this.generationFn?.()
const { sourceInt, targetInt } = resolveEndpointInts(this.endpointInts)
await this.index.removeVerb(this.verb.id, generation)
@ -431,13 +511,20 @@ export class BatchAddToVectorIndexOperation implements Operation {
private operations: AddToVectorIndexOperation[]
/**
* @param index - The vector-index provider (JS HNSW or native).
* @param items - The vectors to index.
* @param generationFn - Resolves the commit generation shared by every item
* in the batch, evaluated when the operations execute.
*/
constructor(
index: VectorIndexProvider,
items: Array<{ id: string; vector: number[] }>
items: Array<{ id: string; vector: number[] }>,
generationFn?: () => bigint | undefined
) {
this.name = `BatchAddToVectorIndex(${resolveVectorProviderId(index)})`
this.operations = items.map(
item => new AddToVectorIndexOperation(index, item.id, item.vector)
item => new AddToVectorIndexOperation(index, item.id, item.vector, generationFn)
)
}
@ -472,12 +559,19 @@ export class BatchAddToMetadataIndexOperation implements Operation {
private operations: AddToMetadataIndexOperation[]
/**
* @param index - The metadata-index manager (JS baseline or a registered provider).
* @param items - The entities to index.
* @param generationFn - Resolves the commit generation shared by every item
* in the batch, evaluated when the operations execute.
*/
constructor(
index: MetadataIndexManager,
items: Array<{ id: string; entity: any }>
items: Array<{ id: string; entity: any }>,
generationFn?: () => bigint | undefined
) {
this.operations = items.map(
item => new AddToMetadataIndexOperation(index, item.id, item.entity)
item => new AddToMetadataIndexOperation(index, item.id, item.entity, generationFn)
)
}