232 lines
9 KiB
TypeScript
232 lines
9 KiB
TypeScript
|
|
/**
|
||
|
|
* @module tests/unit/db/fault-injection-shim
|
||
|
|
* @description The fault-injection storage wrapper proven in isolation: a
|
||
|
|
* torn write persists a decodable prefix (the crash shape durability tests
|
||
|
|
* replay), a dropped sync is observable (armed → the inner adapter never sees
|
||
|
|
* it; journaled), a failed append throws without writing a byte, knobs are
|
||
|
|
* one-shot, and unarmed operation is a transparent passthrough. The full
|
||
|
|
* commit-path fault matrix lives with the log's ack work — this file proves
|
||
|
|
* the SHIM itself.
|
||
|
|
*/
|
||
|
|
import { describe, it, expect, beforeEach } from 'vitest'
|
||
|
|
import { MemoryStorage } from '../../../src/storage/adapters/memoryStorage.js'
|
||
|
|
import {
|
||
|
|
FactLog,
|
||
|
|
storageSupportsFactLog,
|
||
|
|
type CommitFact,
|
||
|
|
type FactLogStorage
|
||
|
|
} from '../../../src/db/factLog.js'
|
||
|
|
import {
|
||
|
|
FaultInjectionStorage,
|
||
|
|
FaultInjectedError
|
||
|
|
} from '../../../src/db/faultInjectionStorage.js'
|
||
|
|
import {
|
||
|
|
encodeFactV2,
|
||
|
|
encodeSegmentHeaderV2,
|
||
|
|
decodeGroupV2,
|
||
|
|
parseSegmentHeader,
|
||
|
|
SEGMENT_HEADER_BYTES,
|
||
|
|
type CommitFactV2
|
||
|
|
} from '../../../src/db/factLogFormat.js'
|
||
|
|
|
||
|
|
const UUID = (n: number): string =>
|
||
|
|
`00000000-0000-4000-8000-${String(n).padStart(12, '0')}`
|
||
|
|
|
||
|
|
const factV2 = (generation: number): CommitFactV2 => ({
|
||
|
|
generation,
|
||
|
|
timestamp: 1_700_000_000_000 + generation,
|
||
|
|
records: [{ type: 'noun.tombstone', id: UUID(generation) }]
|
||
|
|
})
|
||
|
|
|
||
|
|
const factV1 = (generation: number): CommitFact => ({
|
||
|
|
generation,
|
||
|
|
timestamp: 1_700_000_000_000 + generation,
|
||
|
|
ops: [
|
||
|
|
{
|
||
|
|
kind: 'noun',
|
||
|
|
id: UUID(generation),
|
||
|
|
record: { metadata: { noun: 'document' }, vector: null }
|
||
|
|
}
|
||
|
|
]
|
||
|
|
})
|
||
|
|
|
||
|
|
describe('fault-injection storage wrapper', () => {
|
||
|
|
let inner: FactLogStorage & { syncRawObjects: (paths: string[]) => Promise<void> }
|
||
|
|
let shim: FaultInjectionStorage
|
||
|
|
let innerSyncCalls: string[][]
|
||
|
|
|
||
|
|
beforeEach(async () => {
|
||
|
|
const mem: any = new MemoryStorage()
|
||
|
|
await mem.init()
|
||
|
|
innerSyncCalls = []
|
||
|
|
const realSync = mem.syncRawObjects.bind(mem)
|
||
|
|
mem.syncRawObjects = async (paths: string[]) => {
|
||
|
|
innerSyncCalls.push([...paths])
|
||
|
|
return realSync(paths)
|
||
|
|
}
|
||
|
|
inner = mem
|
||
|
|
shim = new FaultInjectionStorage(inner)
|
||
|
|
})
|
||
|
|
|
||
|
|
it('satisfies the fact-log storage surface (drop-in wrapper)', () => {
|
||
|
|
expect(storageSupportsFactLog(shim)).toBe(true)
|
||
|
|
})
|
||
|
|
|
||
|
|
it('unarmed, every operation is a transparent passthrough', async () => {
|
||
|
|
await shim.writeRawBytes('seg', new Uint8Array([1, 2, 3]))
|
||
|
|
await shim.appendRawBytes('seg', new Uint8Array([4, 5]))
|
||
|
|
expect(Array.from((await shim.readRawBytes('seg'))!)).toEqual([1, 2, 3, 4, 5])
|
||
|
|
expect(await shim.rawByteSize('seg')).toBe(5)
|
||
|
|
expect(Array.from((await inner.readRawBytes('seg'))!)).toEqual([1, 2, 3, 4, 5])
|
||
|
|
|
||
|
|
await shim.writeRawObject('obj.json', { a: 1 })
|
||
|
|
expect(await shim.readRawObject('obj.json')).toEqual({ a: 1 })
|
||
|
|
await shim.deleteRawObject('obj.json')
|
||
|
|
expect(await shim.readRawObject('obj.json')).toBeNull()
|
||
|
|
|
||
|
|
await shim.syncRawObjects(['seg'])
|
||
|
|
expect(innerSyncCalls).toEqual([['seg']])
|
||
|
|
expect(shim.injectedFaults).toEqual([])
|
||
|
|
})
|
||
|
|
|
||
|
|
describe('tearWriteAtByte — a torn write produces a decodable-prefix segment', () => {
|
||
|
|
it('persists only the first N bytes of the next append; the prefix decodes intact', async () => {
|
||
|
|
const path = 'facts/seg-test.bfl'
|
||
|
|
const frame1 = encodeFactV2(factV2(1))
|
||
|
|
const frame2 = encodeFactV2(factV2(2))
|
||
|
|
|
||
|
|
await shim.appendRawBytes(path, encodeSegmentHeaderV2(1, 4096))
|
||
|
|
await shim.appendRawBytes(path, frame1)
|
||
|
|
shim.tearWriteAtByte(frame2.length - 5) // crash 5 bytes before the frame lands
|
||
|
|
await shim.appendRawBytes(path, frame2) // reports success — the tear is silent
|
||
|
|
|
||
|
|
const bytes = (await inner.readRawBytes(path))!
|
||
|
|
expect(bytes.length).toBe(SEGMENT_HEADER_BYTES + frame1.length + frame2.length - 5)
|
||
|
|
|
||
|
|
// The "crash": reopen from storage and read what actually survived.
|
||
|
|
const header = parseSegmentHeader(bytes)
|
||
|
|
expect(header).toStrictEqual({ formatVersion: 2, firstGeneration: 1, sealSize: 4096 })
|
||
|
|
const { facts, validBytes } = decodeGroupV2(bytes.subarray(SEGMENT_HEADER_BYTES))
|
||
|
|
expect(facts.map((f) => f.generation)).toEqual([1]) // fact 2's torn frame is invisible
|
||
|
|
expect(validBytes).toBe(frame1.length)
|
||
|
|
|
||
|
|
expect(shim.injectedFaults).toEqual([
|
||
|
|
{
|
||
|
|
kind: 'torn-write',
|
||
|
|
path,
|
||
|
|
requestedBytes: frame2.length,
|
||
|
|
writtenBytes: frame2.length - 5
|
||
|
|
}
|
||
|
|
])
|
||
|
|
})
|
||
|
|
|
||
|
|
it('a tear inside the frame prefix (first bytes) leaves the earlier facts intact too', async () => {
|
||
|
|
const path = 'facts/seg-prefix.bfl'
|
||
|
|
const frame1 = encodeFactV2(factV2(1))
|
||
|
|
await shim.appendRawBytes(path, encodeSegmentHeaderV2(1, 4096))
|
||
|
|
await shim.appendRawBytes(path, frame1)
|
||
|
|
shim.tearWriteAtByte(3)
|
||
|
|
await shim.appendRawBytes(path, encodeFactV2(factV2(2)))
|
||
|
|
|
||
|
|
const bytes = (await inner.readRawBytes(path))!
|
||
|
|
const { facts } = decodeGroupV2(bytes.subarray(SEGMENT_HEADER_BYTES))
|
||
|
|
expect(facts.map((f) => f.generation)).toEqual([1])
|
||
|
|
})
|
||
|
|
|
||
|
|
it('a tear at byte 0 writes nothing at all', async () => {
|
||
|
|
shim.tearWriteAtByte(0)
|
||
|
|
await shim.appendRawBytes('empty.bfl', new Uint8Array([1, 2, 3]))
|
||
|
|
expect(await inner.readRawBytes('empty.bfl')).toBeNull()
|
||
|
|
expect(shim.injectedFaults[0]).toMatchObject({ kind: 'torn-write', writtenBytes: 0 })
|
||
|
|
})
|
||
|
|
|
||
|
|
it('is one-shot: the append after the torn one lands whole', async () => {
|
||
|
|
shim.tearWriteAtByte(1)
|
||
|
|
await shim.appendRawBytes('seg', new Uint8Array([1, 2, 3, 4]))
|
||
|
|
await shim.appendRawBytes('seg', new Uint8Array([5, 6]))
|
||
|
|
expect(Array.from((await inner.readRawBytes('seg'))!)).toEqual([1, 5, 6])
|
||
|
|
})
|
||
|
|
|
||
|
|
it('refuses a negative tear offset', () => {
|
||
|
|
expect(() => shim.tearWriteAtByte(-1)).toThrow(/non-negative/)
|
||
|
|
})
|
||
|
|
})
|
||
|
|
|
||
|
|
describe('dropNextSync — a dropped sync is observable', () => {
|
||
|
|
it('the armed sync never reaches the inner adapter and is journaled', async () => {
|
||
|
|
shim.dropNextSync()
|
||
|
|
await shim.syncRawObjects(['a.bfl', 'b.bfl'])
|
||
|
|
expect(innerSyncCalls).toEqual([]) // the device never saw it
|
||
|
|
expect(shim.injectedFaults).toEqual([{ kind: 'dropped-sync', paths: ['a.bfl', 'b.bfl'] }])
|
||
|
|
})
|
||
|
|
|
||
|
|
it('is one-shot: the following sync passes through', async () => {
|
||
|
|
shim.dropNextSync()
|
||
|
|
await shim.syncRawObjects(['x'])
|
||
|
|
await shim.syncRawObjects(['y'])
|
||
|
|
expect(innerSyncCalls).toEqual([['y']])
|
||
|
|
})
|
||
|
|
})
|
||
|
|
|
||
|
|
describe('failNextAppend — a failed append throws without writing a byte', () => {
|
||
|
|
it('throws the typed error, writes nothing, and journals the fault', async () => {
|
||
|
|
await shim.appendRawBytes('seg', new Uint8Array([1]))
|
||
|
|
shim.failNextAppend()
|
||
|
|
await expect(shim.appendRawBytes('seg', new Uint8Array([2, 3]))).rejects.toThrow(
|
||
|
|
FaultInjectedError
|
||
|
|
)
|
||
|
|
expect(Array.from((await inner.readRawBytes('seg'))!)).toEqual([1]) // untouched
|
||
|
|
expect(shim.injectedFaults).toEqual([{ kind: 'failed-append', path: 'seg' }])
|
||
|
|
// one-shot: the next append succeeds
|
||
|
|
await shim.appendRawBytes('seg', new Uint8Array([4]))
|
||
|
|
expect(Array.from((await inner.readRawBytes('seg'))!)).toEqual([1, 4])
|
||
|
|
})
|
||
|
|
|
||
|
|
it('carries the operation and path for programmatic assertions', async () => {
|
||
|
|
shim.failNextAppend()
|
||
|
|
try {
|
||
|
|
await shim.appendRawBytes('some/path.bfl', new Uint8Array([1]))
|
||
|
|
expect.unreachable('append must throw')
|
||
|
|
} catch (error) {
|
||
|
|
const typed = error as FaultInjectedError
|
||
|
|
expect(typed).toBeInstanceOf(FaultInjectedError)
|
||
|
|
expect(typed.operation).toBe('append')
|
||
|
|
expect(typed.path).toBe('some/path.bfl')
|
||
|
|
}
|
||
|
|
})
|
||
|
|
|
||
|
|
it('wins over a simultaneously-armed tear; the tear stays pending for the next append', async () => {
|
||
|
|
shim.failNextAppend()
|
||
|
|
shim.tearWriteAtByte(2)
|
||
|
|
await expect(shim.appendRawBytes('seg', new Uint8Array([1, 2, 3]))).rejects.toThrow(
|
||
|
|
FaultInjectedError
|
||
|
|
)
|
||
|
|
expect(await inner.readRawBytes('seg')).toBeNull()
|
||
|
|
await shim.appendRawBytes('seg', new Uint8Array([9, 8, 7]))
|
||
|
|
expect(Array.from((await inner.readRawBytes('seg'))!)).toEqual([9, 8]) // torn at 2
|
||
|
|
expect(shim.injectedFaults.map((f) => f.kind)).toEqual(['failed-append', 'torn-write'])
|
||
|
|
})
|
||
|
|
})
|
||
|
|
|
||
|
|
describe('composed with the real fact log (v1 surface)', () => {
|
||
|
|
it('a torn append is truncated away on reopen — the log heals to the intact prefix', async () => {
|
||
|
|
const log = new FactLog(shim)
|
||
|
|
await log.open(0)
|
||
|
|
await log.append(factV1(1))
|
||
|
|
await log.sync()
|
||
|
|
|
||
|
|
shim.tearWriteAtByte(10) // fact 2's frame lands 10 bytes long — torn
|
||
|
|
await log.append(factV1(2))
|
||
|
|
await log.sync()
|
||
|
|
|
||
|
|
// The crash: abandon the instance, reopen from what storage actually holds.
|
||
|
|
const reopened = new FactLog(inner)
|
||
|
|
await reopened.open(2) // generation 2 committed elsewhere — but its fact is torn
|
||
|
|
expect(reopened.headGeneration()).toBe(1)
|
||
|
|
const all: CommitFact[] = []
|
||
|
|
for await (const batch of reopened.scanFacts().batches()) all.push(...batch.facts)
|
||
|
|
expect(all.map((f) => f.generation)).toEqual([1])
|
||
|
|
})
|
||
|
|
})
|
||
|
|
})
|