When a transaction failed and its rollback ALSO failed to undo a canonical write (retries exhausted), the old path logged, continued, threw TransactionRollbackError, and set state='rolled_back' — while the record it could not undo stayed durable on disk. The caller got an error implying the write was undone; a read-back showed the record. A failed rollback had no truthful response (the post-commit response lie). Give a failed rollback a two-branch honest contract, decided by observation: both commit paths already hold byte-identical before-images, so after a failed rollback the store compares current canonical state to them and classifies each touched record as reconciled, an additive orphan (present when it should be gone), or a restorative loss (gone/wrong when it should have been restored). - Adopt-forward: a single-op write whose only damage is a durably-present orphan — the record the caller asked for — is committed forward (its generation buffered) and returns success with a loud warning that the derived index may be incomplete for that id until the next rebuild/repairIndex(). No error, no double-write; the record is durable and get-able immediately. - Fail loud + quarantine: a multi-op batch, or ANY restorative loss, throws the new StoreInconsistentError naming every unreconciled record and its disposition, and puts the brain into write-quarantine (reads keep working, writes refused via assertWritable) until repairIndex() reconciles the derived indexes against canonical and lifts it. The counter is not advanced, and Transaction.rollback now reports state 'inconsistent' instead of the 'rolled_back' lie when any undo failed. New: StoreInconsistentError + UnreconciledRecord exported from the root; repairIndex() forces a rebuild and clears the quarantine. Regression tests/integration/rollback-trapdoor.test.ts injects index-add-throws + canonical-delete-undo-fails and pins adopt-forward (durable, get-able, not quarantined), fail-loud (StoreInconsistentError + quarantine + reads work + repairIndex lifts it), and the error's record naming.
256 lines
7.8 KiB
TypeScript
256 lines
7.8 KiB
TypeScript
/**
|
|
* Transaction - Atomic Unit of Work
|
|
*
|
|
* Executes operations atomically: all succeed or all rollback.
|
|
* Prevents partial failures that leave system in inconsistent state.
|
|
*
|
|
* Usage:
|
|
* ```typescript
|
|
* const tx = new Transaction()
|
|
* tx.addOperation(operation1)
|
|
* tx.addOperation(operation2)
|
|
* await tx.execute() // Both succeed or both rollback
|
|
* ```
|
|
*/
|
|
|
|
import {
|
|
Operation,
|
|
RollbackAction,
|
|
TransactionState,
|
|
TransactionContext,
|
|
TransactionOptions
|
|
} from './types.js'
|
|
import {
|
|
InvalidTransactionStateError,
|
|
TransactionExecutionError,
|
|
TransactionRollbackError,
|
|
TransactionTimeoutError
|
|
} from './errors.js'
|
|
import { prodLog } from '../utils/logger.js'
|
|
|
|
/**
|
|
* Default transaction options
|
|
*/
|
|
const DEFAULT_OPTIONS: Required<TransactionOptions> = {
|
|
timeout: 30000, // 30 seconds
|
|
logging: false,
|
|
maxRollbackRetries: 3
|
|
}
|
|
|
|
/**
|
|
* Transaction class
|
|
*/
|
|
export class Transaction implements TransactionContext {
|
|
private operations: Operation[] = []
|
|
private rollbackActions: RollbackAction[] = []
|
|
private state: TransactionState = 'pending'
|
|
private readonly options: Required<TransactionOptions>
|
|
private startTime?: number
|
|
private endTime?: number
|
|
|
|
constructor(options: TransactionOptions = {}) {
|
|
this.options = { ...DEFAULT_OPTIONS, ...options }
|
|
}
|
|
|
|
/**
|
|
* Add an operation to the transaction
|
|
*/
|
|
addOperation(operation: Operation): void {
|
|
if (this.state !== 'pending') {
|
|
throw new InvalidTransactionStateError(
|
|
this.state,
|
|
'add operation'
|
|
)
|
|
}
|
|
this.operations.push(operation)
|
|
}
|
|
|
|
/**
|
|
* Get current transaction state
|
|
*/
|
|
getState(): TransactionState {
|
|
return this.state
|
|
}
|
|
|
|
/**
|
|
* Get number of operations in transaction
|
|
*/
|
|
getOperationCount(): number {
|
|
return this.operations.length
|
|
}
|
|
|
|
/**
|
|
* Execute all operations atomically
|
|
*/
|
|
async execute(): Promise<void> {
|
|
if (this.state !== 'pending') {
|
|
throw new InvalidTransactionStateError(
|
|
this.state,
|
|
'execute'
|
|
)
|
|
}
|
|
|
|
this.state = 'executing'
|
|
this.startTime = Date.now()
|
|
|
|
if (this.options.logging) {
|
|
prodLog.info(`[Transaction] Executing ${this.operations.length} operations`)
|
|
}
|
|
|
|
try {
|
|
// Execute each operation in order. This loop is the sole rollback-guarded
|
|
// region: ANY error that escapes it — an operation failure OR a
|
|
// mid-flight timeout — is caught below and rolls back every operation
|
|
// applied so far, in reverse order. That single guarantee is the
|
|
// transaction's atomicity contract, and the generation-store commit path
|
|
// depends on it: its abort cleanup assumes a throw from here already
|
|
// restored the applied operations byte-identically (it only discards the
|
|
// uncommitted staging directory). A rollback per exit path — the previous
|
|
// design — let the timeout throw slip past rollback and strand the
|
|
// already-applied writes as torn, generation-less state in canonical
|
|
// storage.
|
|
for (let i = 0; i < this.operations.length; i++) {
|
|
// Budget check BEFORE starting the next operation. A trip here throws
|
|
// into the catch below and rolls back like any other failure — it must
|
|
// never bypass rollback.
|
|
if (Date.now() - this.startTime > this.options.timeout) {
|
|
throw new TransactionTimeoutError(this.options.timeout, i)
|
|
}
|
|
|
|
const operation = this.operations[i]
|
|
|
|
if (this.options.logging) {
|
|
prodLog.info(`[Transaction] Executing operation ${i}: ${operation.name || 'unnamed'}`)
|
|
}
|
|
|
|
let rollbackAction: RollbackAction | undefined
|
|
try {
|
|
rollbackAction = await operation.execute()
|
|
} catch (error) {
|
|
// Normalize an operation failure to a TransactionExecutionError; the
|
|
// outer catch performs the (single) rollback and surfaces it.
|
|
throw new TransactionExecutionError(
|
|
`Operation ${i} failed: ${(error as Error).message}`,
|
|
i,
|
|
operation.name,
|
|
error as Error
|
|
)
|
|
}
|
|
|
|
// Record rollback action (if the operation provided one)
|
|
if (rollbackAction) {
|
|
this.rollbackActions.push(rollbackAction)
|
|
}
|
|
}
|
|
} catch (error) {
|
|
// Single rollback point: undo everything applied so far, in reverse
|
|
// order, then surface the original error. A rollback failure supersedes
|
|
// it (rollback() throws TransactionRollbackError wrapping this error).
|
|
await this.rollback(error as Error)
|
|
throw error
|
|
}
|
|
|
|
// All operations succeeded — commit.
|
|
this.commit()
|
|
}
|
|
|
|
/**
|
|
* Commit the transaction
|
|
*/
|
|
private commit(): void {
|
|
this.state = 'committed'
|
|
this.endTime = Date.now()
|
|
|
|
if (this.options.logging) {
|
|
const duration = this.endTime - (this.startTime || this.endTime)
|
|
prodLog.info(`[Transaction] Committed successfully in ${duration}ms`)
|
|
}
|
|
|
|
// Clear rollback actions - no longer needed
|
|
this.rollbackActions = []
|
|
}
|
|
|
|
/**
|
|
* Rollback all executed operations in reverse order
|
|
*/
|
|
private async rollback(originalError: Error): Promise<void> {
|
|
this.state = 'rolling_back'
|
|
|
|
if (this.options.logging) {
|
|
prodLog.info(`[Transaction] Rolling back ${this.rollbackActions.length} operations`)
|
|
}
|
|
|
|
const rollbackErrors: Error[] = []
|
|
|
|
// Execute rollback actions in REVERSE order
|
|
for (let i = this.rollbackActions.length - 1; i >= 0; i--) {
|
|
const action = this.rollbackActions[i]
|
|
|
|
// Retry rollback with exponential backoff
|
|
let attempts = 0
|
|
let success = false
|
|
|
|
while (attempts < this.options.maxRollbackRetries && !success) {
|
|
try {
|
|
await action()
|
|
success = true
|
|
|
|
if (this.options.logging) {
|
|
prodLog.info(`[Transaction] Rolled back operation ${i}`)
|
|
}
|
|
|
|
} catch (error) {
|
|
attempts++
|
|
|
|
if (attempts >= this.options.maxRollbackRetries) {
|
|
// Max retries exceeded - log error and continue
|
|
const rollbackError = error as Error
|
|
rollbackErrors.push(rollbackError)
|
|
|
|
prodLog.error(
|
|
`[Transaction] Rollback failed for operation ${i} after ${attempts} attempts: ${rollbackError.message}`
|
|
)
|
|
} else {
|
|
// Retry with exponential backoff
|
|
const delayMs = Math.pow(2, attempts) * 100
|
|
await new Promise(resolve => setTimeout(resolve, delayMs))
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
// Honest terminal state: only claim 'rolled_back' when EVERY undo applied.
|
|
// If any undo failed, the store is not cleanly rolled back — mark
|
|
// 'inconsistent' so the commit orchestration reconciles rather than trusts
|
|
// a false 'rolled_back'. (Fixes the state half of the post-commit response
|
|
// lie: the record whose undo failed is still durable.)
|
|
this.state = rollbackErrors.length > 0 ? 'inconsistent' : 'rolled_back'
|
|
this.endTime = Date.now()
|
|
|
|
if (this.options.logging) {
|
|
const duration = this.endTime - (this.startTime || this.endTime)
|
|
prodLog.info(`[Transaction] ${this.state === 'inconsistent' ? 'Rollback INCOMPLETE' : 'Rolled back'} in ${duration}ms`)
|
|
}
|
|
|
|
// If rollback encountered errors, wrap them with original error. The
|
|
// orchestration keys on `instanceof TransactionRollbackError` to know the
|
|
// undo did not fully apply and reconciliation is required.
|
|
if (rollbackErrors.length > 0) {
|
|
throw new TransactionRollbackError(
|
|
`Transaction rollback encountered ${rollbackErrors.length} errors during cleanup`,
|
|
originalError,
|
|
rollbackErrors
|
|
)
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Get transaction execution time in milliseconds
|
|
*/
|
|
getExecutionTimeMs(): number | undefined {
|
|
if (this.startTime && this.endTime) {
|
|
return this.endTime - this.startTime
|
|
}
|
|
return undefined
|
|
}
|
|
}
|