David Snelling
54c183668c
The async group-commit flush that persists single-op generation history swallowed persist failures as a bare warn: writes kept succeeding while their before-images piled up in memory, never durable and unbounded, with no signal. The generation store now accounts for every failed flush at one place (flushPendingSingleOps), tolerates a transient blip (retry with capped exponential backoff), and after PENDING_FLUSH_FAILURE_THRESHOLD consecutive failures LATCHES a durability failure and refuses further single-op and transact writes with a typed, exported PendingFlushDurabilityError — rather than promise a durability it cannot deliver. Live canonical data is untouched; only the immutable history is stuck. The latch self-heals: the moment a flush succeeds (a retry, or an explicit flush()/close()) it lifts and writes resume. Loud errors, never quiet losses.
106 lines
4.4 KiB
TypeScript
106 lines
4.4 KiB
TypeScript
/**
|
|
* @module tests/unit/db/pending-flush-durability
|
|
* @description Finding 8: the async group-commit flush that persists single-op
|
|
* generation HISTORY must not swallow persist failures. Before the fix a failed
|
|
* background flush was a bare `prodLog.warn` — writes kept succeeding while their
|
|
* before-images silently piled up in memory, never durable and unbounded.
|
|
*
|
|
* The contract now:
|
|
* - a single transient flush failure does NOT trip the alarm (tolerance), and
|
|
* - after PENDING_FLUSH_FAILURE_THRESHOLD consecutive failures the store LATCHES
|
|
* and REFUSES further writes with a typed PendingFlushDurabilityError rather
|
|
* than accumulate un-durable history, and
|
|
* - it self-heals: once a flush finally succeeds the latch lifts and writes resume.
|
|
* Live canonical data is never touched — only the immutable history is stuck.
|
|
*
|
|
* Fake timers keep the background retry/coalesce timers from firing, so the exact
|
|
* number of flush attempts (and thus the latch point) is deterministic — the test
|
|
* drives every flush explicitly.
|
|
*/
|
|
import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest'
|
|
import { MemoryStorage } from '../../../src/storage/adapters/memoryStorage.js'
|
|
import { GenerationStore } from '../../../src/db/generationStore.js'
|
|
import { PendingFlushDurabilityError } from '../../../src/db/errors.js'
|
|
import { NounType } from '../../../src/types/graphTypes.js'
|
|
|
|
const ID = (suffix: string): string => `00000000-0000-4000-8000-0000000000${suffix}`
|
|
const meta = (v: number): Record<string, unknown> => ({
|
|
noun: NounType.Document,
|
|
subtype: 'note',
|
|
data: `payload-v${v}`,
|
|
version: v,
|
|
_rev: v
|
|
})
|
|
|
|
describe('db/GenerationStore pending-flush durability (finding 8)', () => {
|
|
let storage: MemoryStorage
|
|
let store: GenerationStore
|
|
|
|
beforeEach(async () => {
|
|
vi.useFakeTimers()
|
|
storage = new MemoryStorage()
|
|
await storage.init()
|
|
store = new GenerationStore(storage)
|
|
await store.open()
|
|
// High size threshold so single-ops never auto-flush — the test owns every flush.
|
|
;(store as any).pendingFlushThreshold = 1000
|
|
})
|
|
|
|
afterEach(() => {
|
|
vi.useRealTimers()
|
|
})
|
|
|
|
/** One single-op write that buffers a pending generation. */
|
|
const singleOp = (id: string, v: number) =>
|
|
store.commitSingleOp({
|
|
touched: { nouns: [id], verbs: [] },
|
|
execute: async () => {
|
|
await storage.saveNounMetadata(id, meta(v))
|
|
}
|
|
})
|
|
|
|
it('latches after repeated flush failures, refuses writes, then self-heals', async () => {
|
|
// Buffer a pending generation while storage is healthy.
|
|
await singleOp(ID('aa'), 1)
|
|
|
|
// Fault the HISTORY flush (raw generation-record writes) — NOT the live
|
|
// canonical write, which uses a different path (saveNounMetadata).
|
|
const fault = Object.assign(new Error('EIO simulated'), { code: 'EIO' })
|
|
const spy = vi.spyOn(storage as any, 'writeRawObject').mockRejectedValue(fault)
|
|
|
|
// Each explicit flush fails and is accounted; the third trips the latch.
|
|
for (let i = 0; i < 3; i++) {
|
|
await expect(store.flushPendingSingleOps()).rejects.toThrow('EIO simulated')
|
|
}
|
|
|
|
// Writes are now refused LOUDLY rather than piling up un-durable history.
|
|
await expect(singleOp(ID('bb'), 1)).rejects.toBeInstanceOf(PendingFlushDurabilityError)
|
|
// The live counter did not advance for the refused write (no generation consumed).
|
|
expect(store.generation()).toBe(1)
|
|
|
|
// Storage recovers → an explicit flush drains the retained tier and lifts the latch.
|
|
spy.mockRestore()
|
|
await expect(store.flushPendingSingleOps()).resolves.toBeUndefined()
|
|
|
|
// Writes resume; the next single-op commits normally.
|
|
const receipt = await singleOp(ID('cc'), 1)
|
|
expect(receipt.generation).toBe(2)
|
|
})
|
|
|
|
it('a single transient flush failure does NOT latch — writes continue', async () => {
|
|
await singleOp(ID('aa'), 1)
|
|
|
|
const spy = vi.spyOn(storage as any, 'writeRawObject').mockRejectedValue(new Error('blip'))
|
|
await expect(store.flushPendingSingleOps()).rejects.toThrow('blip') // failure #1, below threshold
|
|
spy.mockRestore()
|
|
|
|
// Below the latch threshold → the store still accepts writes.
|
|
const receipt = await singleOp(ID('bb'), 1)
|
|
expect(receipt.generation).toBe(2)
|
|
|
|
// A subsequent healthy flush drains the tier and resets the failure counter.
|
|
await expect(store.flushPendingSingleOps()).resolves.toBeUndefined()
|
|
expect((store as any).pendingFlushFailures).toBe(0)
|
|
expect((store as any).pendingFlushError).toBeNull()
|
|
})
|
|
})
|