142 lines
6.4 KiB
TypeScript
142 lines
6.4 KiB
TypeScript
|
|
/**
|
||
|
|
* @module tests/unit/transaction/timeout-never-internally-retried
|
||
|
|
* @description Regression pin for the no-hot-retry contract (8.10.1).
|
||
|
|
*
|
||
|
|
* A production incident: a native-provider op ground 38-40s inside a
|
||
|
|
* transaction, blew the ~32s budget, was rolled back, and a CONSUMER pipeline
|
||
|
|
* hot-retried the identical operation into a 6-minute 100%-CPU storm.
|
||
|
|
* Investigation established brainy itself never auto-retries a
|
||
|
|
* `TransactionTimeoutError` — the storm was entirely the consumer's hot-retry
|
||
|
|
* loop, driven by a "retryable" doc-prose claim with no machine-readable
|
||
|
|
* contract. This file pins the brainy-side half of that story so it can never
|
||
|
|
* regress silently:
|
||
|
|
*
|
||
|
|
* (i) the underlying engine (`TransactionManager.executeTransaction()` →
|
||
|
|
* `Transaction.execute()`) — the exact machinery every single-record
|
||
|
|
* write (`add`/`update`/`remove`/...) drives via
|
||
|
|
* `Brainy.persistSingleOp()` — never internally re-executes a timed-out
|
||
|
|
* operation, and the error it surfaces carries `retryable === true` and
|
||
|
|
* `hotRetryUnsafe === true` (see `src/transaction/errors.ts`).
|
||
|
|
*
|
||
|
|
* Constructed directly (mirrors the existing
|
||
|
|
* `tests/unit/transaction/timeout-rollback.test.ts` pattern) rather than
|
||
|
|
* through a real `brain.add()` call: `transactTimeoutBudget()` floors
|
||
|
|
* every single-op write's budget at `opCount * 2000`ms with NO override
|
||
|
|
* seam (`transactionBudgetFloorMs` only RAISES that floor — it cannot
|
||
|
|
* lower it below the per-op-count term), so getting a real `add()` to
|
||
|
|
* time out requires a multi-second sleep. The engine-level
|
||
|
|
* `options.timeout` override used here is the exact same
|
||
|
|
* `TransactionManager`/`Transaction` code `persistSingleOp` calls —
|
||
|
|
* pinning it here pins add()'s guarantee without paying that wall-clock
|
||
|
|
* cost.
|
||
|
|
*
|
||
|
|
* (ii) `Brainy.add()`'s upsert-race retry loop (src/brainy.ts,
|
||
|
|
* `MAX_UPSERT_ATTEMPTS = 10`) — proving the loop's `catch` treats a
|
||
|
|
* `TransactionTimeoutError` as terminal (immediate rethrow) rather than
|
||
|
|
* the `InsertPreconditionExistsSignal` it retries on, so a mid-flight
|
||
|
|
* timeout can never be silently swallowed and re-attempted up to 10
|
||
|
|
* times.
|
||
|
|
*/
|
||
|
|
import { describe, it, expect } from 'vitest'
|
||
|
|
import { TransactionManager } from '../../../src/transaction/TransactionManager.js'
|
||
|
|
import type { Operation, RollbackAction } from '../../../src/transaction/types.js'
|
||
|
|
import { TransactionTimeoutError } from '../../../src/transaction/errors.js'
|
||
|
|
import { Brainy } from '../../../src/brainy.js'
|
||
|
|
import { NounType } from '../../../src/types/graphTypes.js'
|
||
|
|
|
||
|
|
const sleep = (ms: number) => new Promise<void>((r) => setTimeout(r, ms))
|
||
|
|
|
||
|
|
// Brainy's ValidationConfig fixes vectors at exactly 384 dimensions
|
||
|
|
// (src/utils/paramValidation.ts) — match it so add() doesn't reject test data.
|
||
|
|
const DIM = 384
|
||
|
|
const V = (): number[] => Array(DIM).fill(0.1)
|
||
|
|
|
||
|
|
/** An operation that counts every `execute()` invocation — the re-drive detector. */
|
||
|
|
function countingOp(opts: { delayMs?: number; name: string }): Operation & { calls: number } {
|
||
|
|
const op = {
|
||
|
|
name: opts.name,
|
||
|
|
calls: 0,
|
||
|
|
async execute(): Promise<RollbackAction | undefined> {
|
||
|
|
op.calls++
|
||
|
|
if (opts.delayMs) await sleep(opts.delayMs)
|
||
|
|
return async () => {}
|
||
|
|
}
|
||
|
|
}
|
||
|
|
return op
|
||
|
|
}
|
||
|
|
|
||
|
|
describe('transaction timeouts are never internally re-driven (8.10.1 no-hot-retry contract)', () => {
|
||
|
|
it('(i) the single-op engine (TransactionManager.executeTransaction / Transaction.execute — what add() drives via persistSingleOp) runs the overrun operation EXACTLY once and surfaces ONE retryable+hotRetryUnsafe error', async () => {
|
||
|
|
const manager = new TransactionManager()
|
||
|
|
const op0 = countingOp({ name: 'op0-overruns-budget', delayMs: 30 })
|
||
|
|
const op1 = countingOp({ name: 'op1-must-never-start' })
|
||
|
|
|
||
|
|
let caught: unknown
|
||
|
|
try {
|
||
|
|
await manager.executeTransaction(
|
||
|
|
async (tx) => {
|
||
|
|
tx.addOperation(op0)
|
||
|
|
tx.addOperation(op1)
|
||
|
|
},
|
||
|
|
// Tiny explicit override — the same override seam `transact()`
|
||
|
|
// exposes as `options.timeoutMs`; wins outright over the
|
||
|
|
// opCount*2000 floor that gates every real single-op write
|
||
|
|
// (transactTimeoutBudget()'s override semantics).
|
||
|
|
{ timeout: 5 }
|
||
|
|
)
|
||
|
|
} catch (err) {
|
||
|
|
caught = err
|
||
|
|
}
|
||
|
|
|
||
|
|
expect(caught).toBeInstanceOf(TransactionTimeoutError)
|
||
|
|
const err = caught as TransactionTimeoutError
|
||
|
|
// The machine-readable contract callers branch on instead of parsing
|
||
|
|
// message text (src/transaction/errors.ts).
|
||
|
|
expect(err.retryable).toBe(true)
|
||
|
|
expect(err.hotRetryUnsafe).toBe(true)
|
||
|
|
|
||
|
|
// The re-drive assertion: op0 (the one that overran) executed EXACTLY
|
||
|
|
// once — nothing inside TransactionManager/Transaction looped back and
|
||
|
|
// re-ran it — and op1 never started at all (the budget gate stopped it
|
||
|
|
// before it began, per Transaction.execute()'s per-operation loop).
|
||
|
|
expect(op0.calls).toBe(1)
|
||
|
|
expect(op1.calls).toBe(0)
|
||
|
|
})
|
||
|
|
|
||
|
|
it('(ii) add()\'s upsert-race retry loop (MAX_UPSERT_ATTEMPTS=10) exits on the FIRST TransactionTimeoutError — attempt counter stays at 1, never mistaken for the lost-insert-race signal it retries on', async () => {
|
||
|
|
const brain = new Brainy({
|
||
|
|
requireSubtype: false,
|
||
|
|
storage: { type: 'memory' },
|
||
|
|
silent: true
|
||
|
|
})
|
||
|
|
await brain.init()
|
||
|
|
|
||
|
|
let persistSingleOpCalls = 0
|
||
|
|
const timeoutError = new TransactionTimeoutError(5, 1, {
|
||
|
|
elapsedMs: 6,
|
||
|
|
totalOperations: 2,
|
||
|
|
operationName: 'SaveNounMetadata'
|
||
|
|
})
|
||
|
|
// Stub the private commit seam add() drives (persistSingleOp) to throw
|
||
|
|
// the exact error type the real engine surfaces on a mid-flight timeout.
|
||
|
|
// This test pins the upsert loop's EXCEPTION-HANDLING contract (does it
|
||
|
|
// retry a TransactionTimeoutError like it retries
|
||
|
|
// InsertPreconditionExistsSignal?), not the timing mechanics of a real
|
||
|
|
// timeout — those are pinned by test (i) and by
|
||
|
|
// tests/unit/transaction/timeout-rollback.test.ts.
|
||
|
|
;(brain as any).persistSingleOp = async (): Promise<never> => {
|
||
|
|
persistSingleOpCalls++
|
||
|
|
throw timeoutError
|
||
|
|
}
|
||
|
|
|
||
|
|
await expect(
|
||
|
|
brain.add({ data: 'a', type: NounType.Thing, vector: V() })
|
||
|
|
).rejects.toBe(timeoutError)
|
||
|
|
|
||
|
|
// The loop's attempt counter: exactly one call, never retried up to
|
||
|
|
// MAX_UPSERT_ATTEMPTS.
|
||
|
|
expect(persistSingleOpCalls).toBe(1)
|
||
|
|
await brain.close()
|
||
|
|
})
|
||
|
|
})
|