fix: refuse writes when single-op history cannot be made durable
The async group-commit flush that persists single-op generation history swallowed persist failures as a bare warn: writes kept succeeding while their before-images piled up in memory, never durable and unbounded, with no signal. The generation store now accounts for every failed flush at one place (flushPendingSingleOps), tolerates a transient blip (retry with capped exponential backoff), and after PENDING_FLUSH_FAILURE_THRESHOLD consecutive failures LATCHES a durability failure and refuses further single-op and transact writes with a typed, exported PendingFlushDurabilityError — rather than promise a durability it cannot deliver. Live canonical data is untouched; only the immutable history is stuck. The latch self-heals: the moment a flush succeeds (a retry, or an explicit flush()/close()) it lifts and writes resume. Loud errors, never quiet losses.
This commit is contained in:
parent
d8301f8d08
commit
54c183668c
4 changed files with 302 additions and 9 deletions
106
tests/unit/db/pending-flush-durability.test.ts
Normal file
106
tests/unit/db/pending-flush-durability.test.ts
Normal file
|
|
@ -0,0 +1,106 @@
|
|||
/**
|
||||
* @module tests/unit/db/pending-flush-durability
|
||||
* @description Finding 8: the async group-commit flush that persists single-op
|
||||
* generation HISTORY must not swallow persist failures. Before the fix a failed
|
||||
* background flush was a bare `prodLog.warn` — writes kept succeeding while their
|
||||
* before-images silently piled up in memory, never durable and unbounded.
|
||||
*
|
||||
* The contract now:
|
||||
* - a single transient flush failure does NOT trip the alarm (tolerance), and
|
||||
* - after PENDING_FLUSH_FAILURE_THRESHOLD consecutive failures the store LATCHES
|
||||
* and REFUSES further writes with a typed PendingFlushDurabilityError rather
|
||||
* than accumulate un-durable history, and
|
||||
* - it self-heals: once a flush finally succeeds the latch lifts and writes resume.
|
||||
* Live canonical data is never touched — only the immutable history is stuck.
|
||||
*
|
||||
* Fake timers keep the background retry/coalesce timers from firing, so the exact
|
||||
* number of flush attempts (and thus the latch point) is deterministic — the test
|
||||
* drives every flush explicitly.
|
||||
*/
|
||||
import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest'
|
||||
import { MemoryStorage } from '../../../src/storage/adapters/memoryStorage.js'
|
||||
import { GenerationStore } from '../../../src/db/generationStore.js'
|
||||
import { PendingFlushDurabilityError } from '../../../src/db/errors.js'
|
||||
import { NounType } from '../../../src/types/graphTypes.js'
|
||||
|
||||
const ID = (suffix: string): string => `00000000-0000-4000-8000-0000000000${suffix}`
|
||||
const meta = (v: number): Record<string, unknown> => ({
|
||||
noun: NounType.Document,
|
||||
subtype: 'note',
|
||||
data: `payload-v${v}`,
|
||||
version: v,
|
||||
_rev: v
|
||||
})
|
||||
|
||||
describe('db/GenerationStore pending-flush durability (finding 8)', () => {
|
||||
let storage: MemoryStorage
|
||||
let store: GenerationStore
|
||||
|
||||
beforeEach(async () => {
|
||||
vi.useFakeTimers()
|
||||
storage = new MemoryStorage()
|
||||
await storage.init()
|
||||
store = new GenerationStore(storage)
|
||||
await store.open()
|
||||
// High size threshold so single-ops never auto-flush — the test owns every flush.
|
||||
;(store as any).pendingFlushThreshold = 1000
|
||||
})
|
||||
|
||||
afterEach(() => {
|
||||
vi.useRealTimers()
|
||||
})
|
||||
|
||||
/** One single-op write that buffers a pending generation. */
|
||||
const singleOp = (id: string, v: number) =>
|
||||
store.commitSingleOp({
|
||||
touched: { nouns: [id], verbs: [] },
|
||||
execute: async () => {
|
||||
await storage.saveNounMetadata(id, meta(v))
|
||||
}
|
||||
})
|
||||
|
||||
it('latches after repeated flush failures, refuses writes, then self-heals', async () => {
|
||||
// Buffer a pending generation while storage is healthy.
|
||||
await singleOp(ID('aa'), 1)
|
||||
|
||||
// Fault the HISTORY flush (raw generation-record writes) — NOT the live
|
||||
// canonical write, which uses a different path (saveNounMetadata).
|
||||
const fault = Object.assign(new Error('EIO simulated'), { code: 'EIO' })
|
||||
const spy = vi.spyOn(storage as any, 'writeRawObject').mockRejectedValue(fault)
|
||||
|
||||
// Each explicit flush fails and is accounted; the third trips the latch.
|
||||
for (let i = 0; i < 3; i++) {
|
||||
await expect(store.flushPendingSingleOps()).rejects.toThrow('EIO simulated')
|
||||
}
|
||||
|
||||
// Writes are now refused LOUDLY rather than piling up un-durable history.
|
||||
await expect(singleOp(ID('bb'), 1)).rejects.toBeInstanceOf(PendingFlushDurabilityError)
|
||||
// The live counter did not advance for the refused write (no generation consumed).
|
||||
expect(store.generation()).toBe(1)
|
||||
|
||||
// Storage recovers → an explicit flush drains the retained tier and lifts the latch.
|
||||
spy.mockRestore()
|
||||
await expect(store.flushPendingSingleOps()).resolves.toBeUndefined()
|
||||
|
||||
// Writes resume; the next single-op commits normally.
|
||||
const receipt = await singleOp(ID('cc'), 1)
|
||||
expect(receipt.generation).toBe(2)
|
||||
})
|
||||
|
||||
it('a single transient flush failure does NOT latch — writes continue', async () => {
|
||||
await singleOp(ID('aa'), 1)
|
||||
|
||||
const spy = vi.spyOn(storage as any, 'writeRawObject').mockRejectedValue(new Error('blip'))
|
||||
await expect(store.flushPendingSingleOps()).rejects.toThrow('blip') // failure #1, below threshold
|
||||
spy.mockRestore()
|
||||
|
||||
// Below the latch threshold → the store still accepts writes.
|
||||
const receipt = await singleOp(ID('bb'), 1)
|
||||
expect(receipt.generation).toBe(2)
|
||||
|
||||
// A subsequent healthy flush drains the tier and resets the failure counter.
|
||||
await expect(store.flushPendingSingleOps()).resolves.toBeUndefined()
|
||||
expect((store as any).pendingFlushFailures).toBe(0)
|
||||
expect((store as any).pendingFlushError).toBeNull()
|
||||
})
|
||||
})
|
||||
Loading…
Add table
Add a link
Reference in a new issue