brainy/src/transaction/operations/StorageOperations.ts
David Snelling 2e2ba9c9aa fix: honest counters — removal never re-reads the removed record + repairIndex recounts and persists all rollups
Persisted entity totals inflated permanently: a delete's count decrement
was sourced from re-reading the record being removed, so a null read
(replace race, or a ghost left by a pre-8.3.1 partial delete) silently
skipped the decrement while the paired add had counted — and
Math.max(persistedTotal, scanned) meant no disk cleanup could ever
lower the reported total. Reported by a production deployment with a
write→delete→re-create repro minting drift within hours.

- The caller's pre-delete read now rides through the entire delete path
  (remove()/removeMany()/plan → DeleteNoun/DeleteVerb operations →
  deleteNoun/deleteVerb → deleteNounMetadata/deleteVerbMetadata): a
  null internal read falls back to the known prior record instead of
  skipping the decrement. StorageAdapter.deleteNoun/deleteVerb gain an
  optional priorMetadata parameter (additive).
- rebuildTypeCounts() rebuilt only the type-statistics arrays and
  computed the total just to LOG it — the persisted scalar
  (counts.json) survived every rebuild. One canonical walk now rebuilds
  every counter rollup (scalar totals + per-type maps + type
  statistics) and persists them; repairIndex() runs the recount
  unconditionally, since counters can be inflated over clean shelves.
2026-07-14 11:51:44 -07:00

343 lines
11 KiB
TypeScript

/**
* Storage Operations with Rollback Support
*
* Provides transactional operations for all storage adapters.
* Each operation can be executed and rolled back atomically.
*
* Supports:
* - Both storage adapters (FileSystem, Memory)
* - Nouns (entities) and Verbs (relationships)
* - Metadata and vector data
*/
import type { StorageAdapter, HNSWNoun, HNSWVerb, NounMetadata, VerbMetadata } from '../../coreTypes.js'
import type { Operation, RollbackAction } from '../types.js'
/**
* Save noun metadata with rollback support
*
* Rollback strategy:
* - If metadata existed: Restore previous metadata
* - If metadata was new: Delete metadata
*/
export class SaveNounMetadataOperation implements Operation {
readonly name = 'SaveNounMetadata'
constructor(
private readonly storage: StorageAdapter,
private readonly id: string,
private readonly metadata: NounMetadata,
private readonly isNew: boolean = false
) {}
async execute(): Promise<RollbackAction> {
// Skip read for new entities — nothing to rollback to (saves 1 storage round-trip)
const previousMetadata = this.isNew
? null
: await this.storage.getNounMetadata(this.id)
// Save new metadata
await this.storage.saveNounMetadata(this.id, this.metadata)
// Return rollback action
return async () => {
if (previousMetadata) {
// Restore previous metadata
await this.storage.saveNounMetadata(this.id, previousMetadata)
} else {
// Delete newly created metadata
await this.storage.deleteNounMetadata(this.id)
}
}
}
}
/**
* Save noun (vector data) with rollback support
*
* Rollback strategy:
* - If noun existed: Restore previous noun
* - If noun was new: Delete noun (if deleteNoun exists on adapter)
*
* Note: Not all adapters implement deleteNoun - this is acceptable
* because orphaned vector data without metadata is invisible to queries
*/
export class SaveNounOperation implements Operation {
readonly name = 'SaveNoun'
constructor(
private readonly storage: StorageAdapter,
private readonly noun: HNSWNoun,
private readonly isNew: boolean = false
) {}
async execute(): Promise<RollbackAction> {
// Skip read for new entities — nothing to rollback to (saves 1 storage round-trip)
const previousNoun = this.isNew
? null
: await this.storage.getNoun(this.noun.id)
// Save new noun
await this.storage.saveNoun(this.noun)
// Return rollback action
return async () => {
if (previousNoun) {
// Restore previous noun (extract just vector data)
const nounData: HNSWNoun = {
id: previousNoun.id,
vector: previousNoun.vector,
connections: previousNoun.connections || new Map(),
level: previousNoun.level || 0
}
await this.storage.saveNoun(nounData)
} else {
// Delete newly created noun (if adapter supports it)
// Note: Not all adapters implement deleteNoun
// This is acceptable - metadata deletion makes entity invisible
if ('deleteNoun' in this.storage && typeof this.storage.deleteNoun === 'function') {
await this.storage.deleteNoun(this.noun.id)
}
}
}
}
}
/**
* Delete a noun — FULL canonical removal, with rollback support.
*
* Despite the historical name, this removes the WHOLE entity: both canonical
* legs (metadata + vector) AND the entity's container. Previously it deleted
* only the metadata leg via `deleteNounMetadata`, leaving the canonical
* `vectors.json` leg and the `<id>/` directory orphaned on disk — a "ghost"
* that reads as absent (getNoun needs both legs) yet inflates the enumerated
* count and can never be told apart from a damage scar. Routing through
* `storage.deleteNoun` removes both legs + the container in one place.
*
* Immutability is preserved: this cleans only the live-HEAD projection; the
* generation store retains the before-image so `asOf()` still reconstructs the
* deleted entity until retention expires.
*
* Rollback strategy:
* - Restore BOTH legs from the before-image (vector leg raw, metadata leg via
* the count-aware save so deleteNoun()'s decrement is reversed).
*/
export class DeleteNounMetadataOperation implements Operation {
readonly name = 'DeleteNoun'
constructor(
private readonly storage: StorageAdapter,
private readonly id: string,
/**
* OPTIONAL already-known metadata of the entity being removed (the caller's
* pre-delete read). Removal must never REQUIRE re-reading the thing being
* removed: if the reads here return null (replace race, or a ghost left by
* an earlier version), the count decrement downstream falls back to this
* record instead of being silently skipped — the skip minted permanent
* counter inflation (adds counted, paired removals not decremented).
*/
private readonly priorMetadata?: NounMetadata | null
) {}
async execute(): Promise<RollbackAction> {
// Capture the FULL before-image (both legs) so the undo restores the whole
// entity — a metadata-only rollback would leave the vector leg unrestored.
// A null metadata read falls back to the caller's pre-delete read.
const previousNoun = await this.storage.getNoun(this.id)
const previousMetadata = (await this.storage.getNounMetadata(this.id)) ?? this.priorMetadata ?? null
if (!previousNoun && !previousMetadata) {
// Nothing to delete - no rollback needed
return async () => {}
}
// Full removal: both canonical legs + the entity container + count decrement
// (the prior record keeps the decrement honest on a null canonical read).
await this.storage.deleteNoun(this.id, previousMetadata)
// Return rollback action
return async () => {
// Restore the vector leg, then the metadata leg through the count-aware
// save so deleteNoun()'s decrement is reversed.
if (previousNoun) {
await this.storage.saveNoun({
id: previousNoun.id,
vector: previousNoun.vector,
connections: previousNoun.connections || new Map(),
level: previousNoun.level || 0
})
}
if (previousMetadata) {
await this.storage.saveNounMetadata(this.id, previousMetadata)
}
}
}
}
/**
* Save verb metadata with rollback support
*
* Rollback strategy:
* - If metadata existed: Restore previous metadata
* - If metadata was new: Delete metadata
*/
export class SaveVerbMetadataOperation implements Operation {
readonly name = 'SaveVerbMetadata'
constructor(
private readonly storage: StorageAdapter,
private readonly id: string,
private readonly metadata: VerbMetadata
) {}
async execute(): Promise<RollbackAction> {
// Get existing metadata (for rollback)
const previousMetadata = await this.storage.getVerbMetadata(this.id)
// Save new metadata
await this.storage.saveVerbMetadata(this.id, this.metadata)
// Return rollback action
return async () => {
if (previousMetadata) {
// Restore previous metadata
await this.storage.saveVerbMetadata(this.id, previousMetadata)
} else {
// Delete newly created verb (metadata + vector)
// Note: StorageAdapter has deleteVerb but not deleteVerbMetadata
await this.storage.deleteVerb(this.id)
}
}
}
}
/**
* Save verb (vector data) with rollback support
*
* Rollback strategy:
* - If verb existed: Restore previous verb
* - If verb was new: Delete verb (if deleteVerb exists on adapter)
*/
export class SaveVerbOperation implements Operation {
readonly name = 'SaveVerb'
constructor(
private readonly storage: StorageAdapter,
private readonly verb: HNSWVerb
) {}
async execute(): Promise<RollbackAction> {
// Get existing verb (for rollback)
const previousVerb = await this.storage.getVerb(this.verb.id)
// Save new verb
await this.storage.saveVerb(this.verb)
// Return rollback action
return async () => {
if (previousVerb) {
// Restore previous verb (extract just vector data)
const verbData: HNSWVerb = {
id: previousVerb.id,
sourceId: previousVerb.sourceId,
targetId: previousVerb.targetId,
verb: previousVerb.verb,
vector: previousVerb.vector,
connections: previousVerb.connections || new Map()
}
await this.storage.saveVerb(verbData)
} else {
// Delete newly created verb (if adapter supports it)
if ('deleteVerb' in this.storage && typeof this.storage.deleteVerb === 'function') {
await this.storage.deleteVerb(this.verb.id)
}
}
}
}
}
/**
* Delete verb metadata with rollback support
*
* Rollback strategy:
* - Restore deleted metadata
*/
export class DeleteVerbMetadataOperation implements Operation {
readonly name = 'DeleteVerbMetadata'
constructor(
private readonly storage: StorageAdapter,
private readonly id: string
) {}
async execute(): Promise<RollbackAction> {
// Get metadata before deletion (for rollback)
const previousMetadata = await this.storage.getVerbMetadata(this.id)
if (!previousMetadata) {
// Nothing to delete - no rollback needed
return async () => {}
}
// Delete verb (metadata + vector). The pre-read rides along so the count
// decrement never depends on re-reading the record being removed.
await this.storage.deleteVerb(this.id, previousMetadata)
// Return rollback action
return async () => {
// Restore deleted metadata
await this.storage.saveVerbMetadata(this.id, previousMetadata)
}
}
}
/**
* Update noun metadata with rollback support
*
* Rollback strategy:
* - Restore previous metadata
*
* Note: This is a convenience operation that wraps SaveNounMetadataOperation
* with explicit "update" semantics
*/
export class UpdateNounMetadataOperation implements Operation {
readonly name = 'UpdateNounMetadata'
private saveOperation: SaveNounMetadataOperation
constructor(
storage: StorageAdapter,
id: string,
metadata: NounMetadata
) {
this.saveOperation = new SaveNounMetadataOperation(storage, id, metadata)
}
async execute(): Promise<RollbackAction> {
return await this.saveOperation.execute()
}
}
/**
* Update verb metadata with rollback support
*
* Rollback strategy:
* - Restore previous metadata
*/
export class UpdateVerbMetadataOperation implements Operation {
readonly name = 'UpdateVerbMetadata'
private saveOperation: SaveVerbMetadataOperation
constructor(
storage: StorageAdapter,
id: string,
metadata: VerbMetadata
) {
this.saveOperation = new SaveVerbMetadataOperation(storage, id, metadata)
}
async execute(): Promise<RollbackAction> {
return await this.saveOperation.execute()
}
}