Persisted entity totals inflated permanently: a delete's count decrement was sourced from re-reading the record being removed, so a null read (replace race, or a ghost left by a pre-8.3.1 partial delete) silently skipped the decrement while the paired add had counted — and Math.max(persistedTotal, scanned) meant no disk cleanup could ever lower the reported total. Reported by a production deployment with a write→delete→re-create repro minting drift within hours. - The caller's pre-delete read now rides through the entire delete path (remove()/removeMany()/plan → DeleteNoun/DeleteVerb operations → deleteNoun/deleteVerb → deleteNounMetadata/deleteVerbMetadata): a null internal read falls back to the known prior record instead of skipping the decrement. StorageAdapter.deleteNoun/deleteVerb gain an optional priorMetadata parameter (additive). - rebuildTypeCounts() rebuilt only the type-statistics arrays and computed the total just to LOG it — the persisted scalar (counts.json) survived every rebuild. One canonical walk now rebuilds every counter rollup (scalar totals + per-type maps + type statistics) and persists them; repairIndex() runs the recount unconditionally, since counters can be inflated over clean shelves.
343 lines
11 KiB
TypeScript
343 lines
11 KiB
TypeScript
/**
|
|
* Storage Operations with Rollback Support
|
|
*
|
|
* Provides transactional operations for all storage adapters.
|
|
* Each operation can be executed and rolled back atomically.
|
|
*
|
|
* Supports:
|
|
* - Both storage adapters (FileSystem, Memory)
|
|
* - Nouns (entities) and Verbs (relationships)
|
|
* - Metadata and vector data
|
|
*/
|
|
|
|
import type { StorageAdapter, HNSWNoun, HNSWVerb, NounMetadata, VerbMetadata } from '../../coreTypes.js'
|
|
import type { Operation, RollbackAction } from '../types.js'
|
|
|
|
/**
|
|
* Save noun metadata with rollback support
|
|
*
|
|
* Rollback strategy:
|
|
* - If metadata existed: Restore previous metadata
|
|
* - If metadata was new: Delete metadata
|
|
*/
|
|
export class SaveNounMetadataOperation implements Operation {
|
|
readonly name = 'SaveNounMetadata'
|
|
|
|
constructor(
|
|
private readonly storage: StorageAdapter,
|
|
private readonly id: string,
|
|
private readonly metadata: NounMetadata,
|
|
private readonly isNew: boolean = false
|
|
) {}
|
|
|
|
async execute(): Promise<RollbackAction> {
|
|
// Skip read for new entities — nothing to rollback to (saves 1 storage round-trip)
|
|
const previousMetadata = this.isNew
|
|
? null
|
|
: await this.storage.getNounMetadata(this.id)
|
|
|
|
// Save new metadata
|
|
await this.storage.saveNounMetadata(this.id, this.metadata)
|
|
|
|
// Return rollback action
|
|
return async () => {
|
|
if (previousMetadata) {
|
|
// Restore previous metadata
|
|
await this.storage.saveNounMetadata(this.id, previousMetadata)
|
|
} else {
|
|
// Delete newly created metadata
|
|
await this.storage.deleteNounMetadata(this.id)
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Save noun (vector data) with rollback support
|
|
*
|
|
* Rollback strategy:
|
|
* - If noun existed: Restore previous noun
|
|
* - If noun was new: Delete noun (if deleteNoun exists on adapter)
|
|
*
|
|
* Note: Not all adapters implement deleteNoun - this is acceptable
|
|
* because orphaned vector data without metadata is invisible to queries
|
|
*/
|
|
export class SaveNounOperation implements Operation {
|
|
readonly name = 'SaveNoun'
|
|
|
|
constructor(
|
|
private readonly storage: StorageAdapter,
|
|
private readonly noun: HNSWNoun,
|
|
private readonly isNew: boolean = false
|
|
) {}
|
|
|
|
async execute(): Promise<RollbackAction> {
|
|
// Skip read for new entities — nothing to rollback to (saves 1 storage round-trip)
|
|
const previousNoun = this.isNew
|
|
? null
|
|
: await this.storage.getNoun(this.noun.id)
|
|
|
|
// Save new noun
|
|
await this.storage.saveNoun(this.noun)
|
|
|
|
// Return rollback action
|
|
return async () => {
|
|
if (previousNoun) {
|
|
// Restore previous noun (extract just vector data)
|
|
const nounData: HNSWNoun = {
|
|
id: previousNoun.id,
|
|
vector: previousNoun.vector,
|
|
connections: previousNoun.connections || new Map(),
|
|
level: previousNoun.level || 0
|
|
}
|
|
await this.storage.saveNoun(nounData)
|
|
} else {
|
|
// Delete newly created noun (if adapter supports it)
|
|
// Note: Not all adapters implement deleteNoun
|
|
// This is acceptable - metadata deletion makes entity invisible
|
|
if ('deleteNoun' in this.storage && typeof this.storage.deleteNoun === 'function') {
|
|
await this.storage.deleteNoun(this.noun.id)
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Delete a noun — FULL canonical removal, with rollback support.
|
|
*
|
|
* Despite the historical name, this removes the WHOLE entity: both canonical
|
|
* legs (metadata + vector) AND the entity's container. Previously it deleted
|
|
* only the metadata leg via `deleteNounMetadata`, leaving the canonical
|
|
* `vectors.json` leg and the `<id>/` directory orphaned on disk — a "ghost"
|
|
* that reads as absent (getNoun needs both legs) yet inflates the enumerated
|
|
* count and can never be told apart from a damage scar. Routing through
|
|
* `storage.deleteNoun` removes both legs + the container in one place.
|
|
*
|
|
* Immutability is preserved: this cleans only the live-HEAD projection; the
|
|
* generation store retains the before-image so `asOf()` still reconstructs the
|
|
* deleted entity until retention expires.
|
|
*
|
|
* Rollback strategy:
|
|
* - Restore BOTH legs from the before-image (vector leg raw, metadata leg via
|
|
* the count-aware save so deleteNoun()'s decrement is reversed).
|
|
*/
|
|
export class DeleteNounMetadataOperation implements Operation {
|
|
readonly name = 'DeleteNoun'
|
|
|
|
constructor(
|
|
private readonly storage: StorageAdapter,
|
|
private readonly id: string,
|
|
/**
|
|
* OPTIONAL already-known metadata of the entity being removed (the caller's
|
|
* pre-delete read). Removal must never REQUIRE re-reading the thing being
|
|
* removed: if the reads here return null (replace race, or a ghost left by
|
|
* an earlier version), the count decrement downstream falls back to this
|
|
* record instead of being silently skipped — the skip minted permanent
|
|
* counter inflation (adds counted, paired removals not decremented).
|
|
*/
|
|
private readonly priorMetadata?: NounMetadata | null
|
|
) {}
|
|
|
|
async execute(): Promise<RollbackAction> {
|
|
// Capture the FULL before-image (both legs) so the undo restores the whole
|
|
// entity — a metadata-only rollback would leave the vector leg unrestored.
|
|
// A null metadata read falls back to the caller's pre-delete read.
|
|
const previousNoun = await this.storage.getNoun(this.id)
|
|
const previousMetadata = (await this.storage.getNounMetadata(this.id)) ?? this.priorMetadata ?? null
|
|
|
|
if (!previousNoun && !previousMetadata) {
|
|
// Nothing to delete - no rollback needed
|
|
return async () => {}
|
|
}
|
|
|
|
// Full removal: both canonical legs + the entity container + count decrement
|
|
// (the prior record keeps the decrement honest on a null canonical read).
|
|
await this.storage.deleteNoun(this.id, previousMetadata)
|
|
|
|
// Return rollback action
|
|
return async () => {
|
|
// Restore the vector leg, then the metadata leg through the count-aware
|
|
// save so deleteNoun()'s decrement is reversed.
|
|
if (previousNoun) {
|
|
await this.storage.saveNoun({
|
|
id: previousNoun.id,
|
|
vector: previousNoun.vector,
|
|
connections: previousNoun.connections || new Map(),
|
|
level: previousNoun.level || 0
|
|
})
|
|
}
|
|
if (previousMetadata) {
|
|
await this.storage.saveNounMetadata(this.id, previousMetadata)
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Save verb metadata with rollback support
|
|
*
|
|
* Rollback strategy:
|
|
* - If metadata existed: Restore previous metadata
|
|
* - If metadata was new: Delete metadata
|
|
*/
|
|
export class SaveVerbMetadataOperation implements Operation {
|
|
readonly name = 'SaveVerbMetadata'
|
|
|
|
constructor(
|
|
private readonly storage: StorageAdapter,
|
|
private readonly id: string,
|
|
private readonly metadata: VerbMetadata
|
|
) {}
|
|
|
|
async execute(): Promise<RollbackAction> {
|
|
// Get existing metadata (for rollback)
|
|
const previousMetadata = await this.storage.getVerbMetadata(this.id)
|
|
|
|
// Save new metadata
|
|
await this.storage.saveVerbMetadata(this.id, this.metadata)
|
|
|
|
// Return rollback action
|
|
return async () => {
|
|
if (previousMetadata) {
|
|
// Restore previous metadata
|
|
await this.storage.saveVerbMetadata(this.id, previousMetadata)
|
|
} else {
|
|
// Delete newly created verb (metadata + vector)
|
|
// Note: StorageAdapter has deleteVerb but not deleteVerbMetadata
|
|
await this.storage.deleteVerb(this.id)
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Save verb (vector data) with rollback support
|
|
*
|
|
* Rollback strategy:
|
|
* - If verb existed: Restore previous verb
|
|
* - If verb was new: Delete verb (if deleteVerb exists on adapter)
|
|
*/
|
|
export class SaveVerbOperation implements Operation {
|
|
readonly name = 'SaveVerb'
|
|
|
|
constructor(
|
|
private readonly storage: StorageAdapter,
|
|
private readonly verb: HNSWVerb
|
|
) {}
|
|
|
|
async execute(): Promise<RollbackAction> {
|
|
// Get existing verb (for rollback)
|
|
const previousVerb = await this.storage.getVerb(this.verb.id)
|
|
|
|
// Save new verb
|
|
await this.storage.saveVerb(this.verb)
|
|
|
|
// Return rollback action
|
|
return async () => {
|
|
if (previousVerb) {
|
|
// Restore previous verb (extract just vector data)
|
|
const verbData: HNSWVerb = {
|
|
id: previousVerb.id,
|
|
sourceId: previousVerb.sourceId,
|
|
targetId: previousVerb.targetId,
|
|
verb: previousVerb.verb,
|
|
vector: previousVerb.vector,
|
|
connections: previousVerb.connections || new Map()
|
|
}
|
|
await this.storage.saveVerb(verbData)
|
|
} else {
|
|
// Delete newly created verb (if adapter supports it)
|
|
if ('deleteVerb' in this.storage && typeof this.storage.deleteVerb === 'function') {
|
|
await this.storage.deleteVerb(this.verb.id)
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Delete verb metadata with rollback support
|
|
*
|
|
* Rollback strategy:
|
|
* - Restore deleted metadata
|
|
*/
|
|
export class DeleteVerbMetadataOperation implements Operation {
|
|
readonly name = 'DeleteVerbMetadata'
|
|
|
|
constructor(
|
|
private readonly storage: StorageAdapter,
|
|
private readonly id: string
|
|
) {}
|
|
|
|
async execute(): Promise<RollbackAction> {
|
|
// Get metadata before deletion (for rollback)
|
|
const previousMetadata = await this.storage.getVerbMetadata(this.id)
|
|
|
|
if (!previousMetadata) {
|
|
// Nothing to delete - no rollback needed
|
|
return async () => {}
|
|
}
|
|
|
|
// Delete verb (metadata + vector). The pre-read rides along so the count
|
|
// decrement never depends on re-reading the record being removed.
|
|
await this.storage.deleteVerb(this.id, previousMetadata)
|
|
|
|
// Return rollback action
|
|
return async () => {
|
|
// Restore deleted metadata
|
|
await this.storage.saveVerbMetadata(this.id, previousMetadata)
|
|
}
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Update noun metadata with rollback support
|
|
*
|
|
* Rollback strategy:
|
|
* - Restore previous metadata
|
|
*
|
|
* Note: This is a convenience operation that wraps SaveNounMetadataOperation
|
|
* with explicit "update" semantics
|
|
*/
|
|
export class UpdateNounMetadataOperation implements Operation {
|
|
readonly name = 'UpdateNounMetadata'
|
|
|
|
private saveOperation: SaveNounMetadataOperation
|
|
|
|
constructor(
|
|
storage: StorageAdapter,
|
|
id: string,
|
|
metadata: NounMetadata
|
|
) {
|
|
this.saveOperation = new SaveNounMetadataOperation(storage, id, metadata)
|
|
}
|
|
|
|
async execute(): Promise<RollbackAction> {
|
|
return await this.saveOperation.execute()
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Update verb metadata with rollback support
|
|
*
|
|
* Rollback strategy:
|
|
* - Restore previous metadata
|
|
*/
|
|
export class UpdateVerbMetadataOperation implements Operation {
|
|
readonly name = 'UpdateVerbMetadata'
|
|
|
|
private saveOperation: SaveVerbMetadataOperation
|
|
|
|
constructor(
|
|
storage: StorageAdapter,
|
|
id: string,
|
|
metadata: VerbMetadata
|
|
) {
|
|
this.saveOperation = new SaveVerbMetadataOperation(storage, id, metadata)
|
|
}
|
|
|
|
async execute(): Promise<RollbackAction> {
|
|
return await this.saveOperation.execute()
|
|
}
|
|
}
|