David Snelling
67c606be69
An internal cross-engine fault-injection run (frozen-platter power-loss capture) surfaced three release-gating findings; each cured in its owning layer, each pinned: 1. WHOLE-LOG REPLAY ON UNCLEAN OPEN (the big one): log-authority replay only covered facts ABOVE the manifest — but live canonical entity writes are tmp+rename without per-file fsync, and the group-commit flush syncs staging + manifest, never the live tree. Power loss could therefore vaporize acked canonical bytes BELOW the manifest while the log held every fact scan-clean (measured: 299 of 301 acks lost). Now: a clean close stamps a clean-shutdown marker (fsynced, written last); every open consumes it; an UNCLEAN open under log authority folds the ENTIRE log into canonical — whole-entity after-images make the re-apply idempotent and byte-safe. Zero cost on the happy path; crash recovery pays one narrated fold. Recovery is replay: a crash is just bigger lag. 2. TORN WRITER LOCK: power loss legally leaves the lock file present but empty; the parse failure read as 'no holder' while the O_EXCL claim EEXISTed forever — a PERMANENT lockout no staleness check could clear. An unparseable lock is stale by definition (no live holder has one): unlink loudly and re-loop; a racer rewriting a valid lock first wins. 3. PAIR GUARD: flush() called metadataIndex.stampWatermark unguarded; a replacement metadata provider without the method killed the pair at first flush. All three stamp calls are optional-chained — a missing stamp is a verdict-side rescan, never a flush crash. Pins: whole-log fold restores rows vanished below the manifest · clean-shutdown marker lifecycle (stamp/consume/re-stamp) · torn-lock recovery with a fresh write after · stampless-provider flush. Gates: unit 2055/2055 · integration 824 · kill-matrix 15/15.
701 lines
31 KiB
TypeScript
701 lines
31 KiB
TypeScript
/**
|
|
* @module tests/integration/durability-kill-matrix
|
|
* @description THE DURABILITY KILL MATRIX — for every step of the commit
|
|
* path, inject a crash AT that step (the generation store's test-only fault
|
|
* injector), then reopen the same storage directory with a brand-new Brainy
|
|
* and assert the recovery contract BY CONSTRUCTION, not by timing:
|
|
*
|
|
* - an ACKED write survives the crash (never a lost ack), and
|
|
* - an UN-ACKED write leaves no torn state (fully present or fully absent,
|
|
* never half).
|
|
*
|
|
* The crash simulation is honest process death: the crashed brain is NEVER
|
|
* closed — `abandonAsCrashed` discards its buffered RAM state exactly as a
|
|
* dead process would, and recovery on the next open is the only repair that
|
|
* runs. File bytes already handed to the OS survive (process-crash model);
|
|
* one row additionally models POWER LOSS by removing an entity's un-fsynced
|
|
* canonical files (legal: single-op canonical writes are tmp+rename without
|
|
* fsync).
|
|
*
|
|
* Matrix rows (fault point → durability barrier position):
|
|
*
|
|
* BEFORE the barrier (nothing durable records the write):
|
|
* singleop-after-execute · singleop-after-fact-append · flush-after-staging
|
|
* AFTER partial durability (staged/synced bytes exist, manifest did not advance):
|
|
* flush-before-manifest · before-manifest-rename (transact) ·
|
|
* transact-after-fact-sync
|
|
* AFTER the commit point:
|
|
* after-manifest-rename (transact)
|
|
* MODE VARIANTS: singleop-after-fact-append under durable-at-ack.
|
|
* DISK FULL: one ENOSPC'd append — loud typed rejection, reads keep
|
|
* serving, a later write succeeds.
|
|
*
|
|
* Where the observed recovery contract differs from the ideal, the pin states
|
|
* the OBSERVED behavior with a comment; where the observed behavior violates
|
|
* "never a torn state / never a lost ack", the pin asserts the CONTRACT and
|
|
* is marked `.fails` — a release-blocking finding, deliberately not weakened.
|
|
*/
|
|
import { describe, it, expect, afterEach } from 'vitest'
|
|
import * as fs from 'node:fs'
|
|
import { join } from 'node:path'
|
|
import { Brainy } from '../../src/brainy.js'
|
|
import { NounType } from '../../src/types/graphTypes.js'
|
|
import {
|
|
abandonAsCrashed,
|
|
armCrash,
|
|
dropCanonicalNoun,
|
|
factGenerations,
|
|
failNextAppendWithEnospc,
|
|
generationDirExists,
|
|
makeTempDir,
|
|
openBrain,
|
|
storeOf,
|
|
uid,
|
|
vec
|
|
} from '../helpers/durabilityKillMatrix.js'
|
|
|
|
describe('durability kill matrix — crash at every commit-path step, recover by reopen', () => {
|
|
const dirs: string[] = []
|
|
const liveBrains: Brainy[] = []
|
|
// Crashed brains are deliberately NEVER closed (a dead process cannot
|
|
// close); they are severed by abandonAsCrashed inside each test.
|
|
|
|
function trackDir(): string {
|
|
const dir = makeTempDir()
|
|
dirs.push(dir)
|
|
return dir
|
|
}
|
|
|
|
async function openLive(dir: string): Promise<Brainy> {
|
|
const brain = await openBrain(dir)
|
|
liveBrains.push(brain)
|
|
return brain
|
|
}
|
|
|
|
afterEach(async () => {
|
|
for (const brain of liveBrains.splice(0)) {
|
|
try {
|
|
await brain.close()
|
|
} catch {
|
|
// already closed / crashed mid-close — teardown only
|
|
}
|
|
}
|
|
for (const dir of dirs.splice(0)) {
|
|
await fs.promises.rm(dir, { recursive: true, force: true })
|
|
}
|
|
})
|
|
|
|
/** Baseline arrangement: one durable row + explicit flush = the durable floor. */
|
|
async function arrangeBaseline(label: string): Promise<{
|
|
dir: string
|
|
brain: Brainy
|
|
baselineId: string
|
|
floor: number
|
|
}> {
|
|
const dir = trackDir()
|
|
const brain = await openBrain(dir) // NOT tracked live — most rows crash it
|
|
const baselineId = uid(`${label}-baseline`)
|
|
await brain.add({
|
|
id: baselineId,
|
|
data: 'baseline row',
|
|
type: NounType.Document,
|
|
vector: vec(1),
|
|
metadata: { v: 1 }
|
|
})
|
|
await brain.flush()
|
|
return { dir, brain, baselineId, floor: storeOf(brain).committedGeneration() }
|
|
}
|
|
|
|
/**
|
|
* Flip a brain to durable-at-ack (log-authority) mode.
|
|
*
|
|
* NOT via `adoptLogAuthority()`: the sanctioned flip REFUSES on a freshly
|
|
* materialized brain — its verification oracle reports the generation-0
|
|
* VFS-root baseline as a divergence (`state-differs` even after an
|
|
* identity-update backfill; verified 2026-08-10). This helper flips the
|
|
* SAME switch the sanctioned path flips (`setLogDurability('at-ack')`) and
|
|
* persists the SAME authority artifact, so a reopened brain also runs in
|
|
* log-authority mode. The durability semantics under test are governed
|
|
* entirely by that switch.
|
|
*/
|
|
async function flipToAtAck(brain: Brainy): Promise<void> {
|
|
const storage = (
|
|
brain as unknown as {
|
|
storage: {
|
|
writeRawObject(p: string, d: unknown): Promise<void>
|
|
syncRawObjects(p: string[]): Promise<void>
|
|
}
|
|
}
|
|
).storage
|
|
await storage.writeRawObject('_system/log-authority.json', {
|
|
authority: 'log',
|
|
flippedAt: Date.now()
|
|
})
|
|
await storage.syncRawObjects(['_system/log-authority.json'])
|
|
storeOf(brain).setLogDurability('at-ack')
|
|
}
|
|
|
|
// ==========================================================================
|
|
// Rows BEFORE the durability barrier — the write never became durable-acked
|
|
// ==========================================================================
|
|
|
|
it('singleop-after-execute — un-acked write is atomic (present-whole), baseline and log stay at the floor', async () => {
|
|
const { dir, brain, baselineId, floor } = await arrangeBaseline('sae')
|
|
const crashedId = uid('sae-crashed')
|
|
const arm = armCrash(brain, 'singleop-after-execute')
|
|
await expect(
|
|
brain.add({
|
|
id: crashedId,
|
|
data: 'never acked',
|
|
type: NounType.Document,
|
|
vector: vec(2),
|
|
metadata: { v: 2 }
|
|
})
|
|
).rejects.toThrow('simulated process crash at singleop-after-execute')
|
|
expect(arm.fired).toContain('singleop-after-execute')
|
|
await abandonAsCrashed(brain)
|
|
|
|
const reopened = await openLive(dir)
|
|
// Baseline intact.
|
|
expect(((await reopened.get(baselineId)) as { metadata: { v: number } }).metadata.v).toBe(1)
|
|
// The log holds nothing beyond the committed watermark (no fact was ever
|
|
// appended for the crashed write).
|
|
expect(await factGenerations(reopened)).toEqual([floor])
|
|
expect(storeOf(reopened).committedGeneration()).toBe(floor)
|
|
// The un-acked write: Model-B applies the live canonical write BEFORE the
|
|
// ack, so under process death its bytes survive — the row is PRESENT and
|
|
// WHOLE by id (atomic, not torn). Under power loss the same un-fsynced
|
|
// bytes may instead vanish entirely; both end states are atomic. NOTE the
|
|
// divergence: the row is get()-visible but find()-invisible (no index
|
|
// entry survived, no generation/fact records it, and no repair is pending
|
|
// — a permanent canonical orphan; see the suite report).
|
|
const orphan = (await reopened.get(crashedId)) as { metadata: { v: number } } | null
|
|
expect(orphan).not.toBeNull()
|
|
expect(orphan!.metadata.v).toBe(2) // whole, byte-consistent — never torn
|
|
const found = (await reopened.find({ type: NounType.Document, limit: 10 })) as Array<{ id: string }>
|
|
expect(found.map((f) => f.id)).toContain(baselineId)
|
|
expect(found.map((f) => f.id)).not.toContain(crashedId)
|
|
// A fresh write succeeds with a monotonic generation. The crashed
|
|
// generation number is REUSED (nothing durable references it): the
|
|
// counter reopened at the floor.
|
|
expect(reopened.generation()).toBe(floor)
|
|
const freshId = uid('sae-fresh')
|
|
await reopened.add({
|
|
id: freshId,
|
|
data: 'fresh after recovery',
|
|
type: NounType.Document,
|
|
vector: vec(3),
|
|
metadata: { v: 3 }
|
|
})
|
|
await reopened.flush()
|
|
expect(storeOf(reopened).committedGeneration()).toBe(floor + 1)
|
|
expect(((await reopened.get(freshId)) as { metadata: { v: number } }).metadata.v).toBe(3)
|
|
})
|
|
|
|
it('singleop-after-fact-append (deferred mode) — the appended fact is truncated back at reopen', async () => {
|
|
const { dir, brain, baselineId, floor } = await arrangeBaseline('sfa')
|
|
const crashedId = uid('sfa-crashed')
|
|
const arm = armCrash(brain, 'singleop-after-fact-append')
|
|
await expect(
|
|
brain.add({
|
|
id: crashedId,
|
|
data: 'never acked',
|
|
type: NounType.Document,
|
|
vector: vec(2),
|
|
metadata: { v: 2 }
|
|
})
|
|
).rejects.toThrow('simulated process crash at singleop-after-fact-append')
|
|
expect(arm.fired).toContain('singleop-after-fact-append')
|
|
await abandonAsCrashed(brain)
|
|
|
|
const reopened = await openLive(dir)
|
|
// The fact WAS appended to the log file before the crash (process death
|
|
// keeps file bytes) — open() must truncate it back to the manifest
|
|
// watermark, and does.
|
|
expect(await factGenerations(reopened)).toEqual([floor])
|
|
expect(storeOf(reopened).committedGeneration()).toBe(floor)
|
|
// Baseline intact; un-acked row atomic (present-whole via canonical, as
|
|
// in the singleop-after-execute row).
|
|
expect(((await reopened.get(baselineId)) as { metadata: { v: number } }).metadata.v).toBe(1)
|
|
const orphan = (await reopened.get(crashedId)) as { metadata: { v: number } } | null
|
|
expect(orphan).not.toBeNull()
|
|
expect(orphan!.metadata.v).toBe(2)
|
|
// Fresh write with a monotonic generation (crashed number reused — the
|
|
// truncated fact freed it).
|
|
expect(reopened.generation()).toBe(floor)
|
|
const freshId = uid('sfa-fresh')
|
|
await reopened.add({
|
|
id: freshId,
|
|
data: 'fresh',
|
|
type: NounType.Document,
|
|
vector: vec(3),
|
|
metadata: { v: 3 }
|
|
})
|
|
await reopened.flush()
|
|
expect(storeOf(reopened).committedGeneration()).toBe(floor + 1)
|
|
expect(await factGenerations(reopened)).toEqual([floor, floor + 1])
|
|
})
|
|
|
|
it('flush-after-staging — the ACKED write survives (drop-without-restore); only the window history is lost', async () => {
|
|
const { dir, brain, baselineId, floor } = await arrangeBaseline('fas')
|
|
const ackedId = uid('fas-acked')
|
|
await brain.add({
|
|
id: ackedId,
|
|
data: 'acked before flush',
|
|
type: NounType.Document,
|
|
vector: vec(2),
|
|
metadata: { v: 2 }
|
|
})
|
|
const ackedGen = storeOf(brain).generation()
|
|
const arm = armCrash(brain, 'flush-after-staging')
|
|
await expect(brain.flush()).rejects.toThrow('simulated process crash at flush-after-staging')
|
|
expect(arm.fired).toContain('flush-after-staging')
|
|
// The crashed flush left the staged record-set dir on disk, above the manifest.
|
|
expect(generationDirExists(dir, ackedGen)).toBe(true)
|
|
await abandonAsCrashed(brain)
|
|
|
|
const reopened = await openLive(dir)
|
|
// Recovery DROPPED the staged group-commit dir WITHOUT restoring its
|
|
// before-images — restoring would silently revert an acknowledged write.
|
|
expect(generationDirExists(dir, ackedGen)).toBe(false)
|
|
expect(storeOf(reopened).committedGeneration()).toBe(floor)
|
|
// NEVER A LOST ACK: the acknowledged write is present and whole.
|
|
const acked = (await reopened.get(ackedId)) as { metadata: { v: number } } | null
|
|
expect(acked).not.toBeNull()
|
|
expect(acked!.metadata.v).toBe(2)
|
|
expect(((await reopened.get(baselineId)) as { metadata: { v: number } }).metadata.v).toBe(1)
|
|
// Recovery rolled generations back → index reconciliation ran → the acked
|
|
// row is find()-visible too.
|
|
const found = (await reopened.find({ type: NounType.Document, limit: 10 })) as Array<{ id: string }>
|
|
expect(found.map((f) => f.id)).toEqual(expect.arrayContaining([baselineId, ackedId]))
|
|
// The window's HISTORY is the documented cost: its fact is truncated back
|
|
// (the acked row now lives only in canonical bytes, not the log).
|
|
expect(await factGenerations(reopened)).toEqual([floor])
|
|
// The crashed generation number is NOT reused (its dropped dir was seen
|
|
// at open): fresh writes continue above it.
|
|
expect(reopened.generation()).toBe(ackedGen)
|
|
const freshId = uid('fas-fresh')
|
|
await reopened.add({
|
|
id: freshId,
|
|
data: 'fresh',
|
|
type: NounType.Document,
|
|
vector: vec(3),
|
|
metadata: { v: 3 }
|
|
})
|
|
await reopened.flush()
|
|
expect(storeOf(reopened).committedGeneration()).toBe(ackedGen + 1)
|
|
})
|
|
|
|
// ==========================================================================
|
|
// Rows AFTER partial durability — staged/synced bytes exist, no manifest
|
|
// ==========================================================================
|
|
|
|
it('flush-before-manifest — staged bytes + synced facts above the manifest are dropped/truncated; the acked write stays', async () => {
|
|
const { dir, brain, baselineId, floor } = await arrangeBaseline('fbm')
|
|
const ackedId = uid('fbm-acked')
|
|
await brain.add({
|
|
id: ackedId,
|
|
data: 'acked before flush',
|
|
type: NounType.Document,
|
|
vector: vec(2),
|
|
metadata: { v: 2 }
|
|
})
|
|
const ackedGen = storeOf(brain).generation()
|
|
const arm = armCrash(brain, 'flush-before-manifest')
|
|
await expect(brain.flush()).rejects.toThrow('simulated process crash at flush-before-manifest')
|
|
// The earlier flush phase passed through untripped before the target fired.
|
|
expect(arm.fired).toContain('flush-after-staging')
|
|
expect(arm.fired).toContain('flush-before-manifest')
|
|
expect(generationDirExists(dir, ackedGen)).toBe(true)
|
|
await abandonAsCrashed(brain)
|
|
|
|
const reopened = await openLive(dir)
|
|
// Per the recovery contract in open(): groupCommit record-sets above the
|
|
// manifest are dropped WITHOUT restore, and the (fsynced!) facts above
|
|
// the manifest are truncated back. The acked live write stays.
|
|
expect(generationDirExists(dir, ackedGen)).toBe(false)
|
|
expect(storeOf(reopened).committedGeneration()).toBe(floor)
|
|
expect(await factGenerations(reopened)).toEqual([floor])
|
|
const acked = (await reopened.get(ackedId)) as { metadata: { v: number } } | null
|
|
expect(acked).not.toBeNull() // never a lost ack
|
|
expect(acked!.metadata.v).toBe(2)
|
|
expect(((await reopened.get(baselineId)) as { metadata: { v: number } }).metadata.v).toBe(1)
|
|
// Fresh write above the crashed generation (number not reused).
|
|
expect(reopened.generation()).toBe(ackedGen)
|
|
const freshId = uid('fbm-fresh')
|
|
await reopened.add({
|
|
id: freshId,
|
|
data: 'fresh',
|
|
type: NounType.Document,
|
|
vector: vec(3),
|
|
metadata: { v: 3 }
|
|
})
|
|
await reopened.flush()
|
|
expect(storeOf(reopened).committedGeneration()).toBe(ackedGen + 1)
|
|
})
|
|
|
|
it('before-manifest-rename (transact) — fully staged, never committed: rolled back byte-identically', async () => {
|
|
const { dir, brain, baselineId, floor } = await arrangeBaseline('bmr')
|
|
const newId = uid('bmr-new')
|
|
const arm = armCrash(brain, 'before-manifest-rename')
|
|
await expect(
|
|
brain.transact([
|
|
{ op: 'update', id: baselineId, metadata: { v: 2 } },
|
|
{
|
|
op: 'add',
|
|
id: newId,
|
|
type: NounType.Document,
|
|
data: 'uncommitted',
|
|
vector: vec(2),
|
|
metadata: { v: 2 }
|
|
}
|
|
])
|
|
).rejects.toThrow('simulated process crash at before-manifest-rename')
|
|
expect(arm.fired).toContain('before-manifest-rename')
|
|
const txGen = storeOf(brain).generation()
|
|
expect(generationDirExists(dir, txGen)).toBe(true)
|
|
await abandonAsCrashed(brain)
|
|
|
|
const reopened = await openLive(dir)
|
|
// Rolled back cleanly: the update is undone, the add is ABSENT everywhere.
|
|
expect(((await reopened.get(baselineId)) as { metadata: { v: number } }).metadata.v).toBe(1)
|
|
expect(await reopened.get(newId)).toBeNull()
|
|
const found = (await reopened.find({ type: NounType.Document, limit: 10 })) as Array<{ id: string }>
|
|
expect(found.map((f) => f.id)).not.toContain(newId)
|
|
expect(generationDirExists(dir, txGen)).toBe(false)
|
|
expect(storeOf(reopened).committedGeneration()).toBe(floor)
|
|
expect(await factGenerations(reopened)).toEqual([floor])
|
|
// The crashed generation number is never reissued (counter persisted
|
|
// before the crash point).
|
|
expect(reopened.generation()).toBe(txGen)
|
|
const freshId = uid('bmr-fresh')
|
|
await reopened.add({
|
|
id: freshId,
|
|
data: 'fresh',
|
|
type: NounType.Document,
|
|
vector: vec(3),
|
|
metadata: { v: 3 }
|
|
})
|
|
await reopened.flush()
|
|
expect(storeOf(reopened).committedGeneration()).toBe(txGen + 1)
|
|
})
|
|
|
|
it('transact-after-fact-sync — the fsynced fact of an uncommitted transact is truncated back; rollback is clean', async () => {
|
|
const { dir, brain, baselineId, floor } = await arrangeBaseline('tfs')
|
|
const newId = uid('tfs-new')
|
|
const arm = armCrash(brain, 'transact-after-fact-sync')
|
|
await expect(
|
|
brain.transact([
|
|
{ op: 'update', id: baselineId, metadata: { v: 2 } },
|
|
{
|
|
op: 'add',
|
|
id: newId,
|
|
type: NounType.Document,
|
|
data: 'uncommitted',
|
|
vector: vec(2),
|
|
metadata: { v: 2 }
|
|
}
|
|
])
|
|
).rejects.toThrow('simulated process crash at transact-after-fact-sync')
|
|
expect(arm.fired).toContain('transact-after-fact-sync')
|
|
const txGen = storeOf(brain).generation()
|
|
await abandonAsCrashed(brain)
|
|
|
|
const reopened = await openLive(dir)
|
|
// The batch's fact was appended AND fsynced before the crash — open()
|
|
// must truncate it back to the manifest watermark (the generation never
|
|
// committed), and the before-images must restore byte-identically.
|
|
expect(await factGenerations(reopened)).toEqual([floor])
|
|
expect(((await reopened.get(baselineId)) as { metadata: { v: number } }).metadata.v).toBe(1)
|
|
expect(await reopened.get(newId)).toBeNull()
|
|
expect(storeOf(reopened).committedGeneration()).toBe(floor)
|
|
expect(generationDirExists(dir, txGen)).toBe(false)
|
|
// Counter: the staged dir was seen at open, so the number is not reused.
|
|
expect(reopened.generation()).toBe(txGen)
|
|
const freshId = uid('tfs-fresh')
|
|
await reopened.add({
|
|
id: freshId,
|
|
data: 'fresh',
|
|
type: NounType.Document,
|
|
vector: vec(3),
|
|
metadata: { v: 3 }
|
|
})
|
|
await reopened.flush()
|
|
expect(storeOf(reopened).committedGeneration()).toBe(txGen + 1)
|
|
})
|
|
|
|
// ==========================================================================
|
|
// Row AFTER the commit point — the transaction must be kept
|
|
// ==========================================================================
|
|
|
|
it('after-manifest-rename (transact) — the manifest rename landed: the transaction is COMMITTED and fully present', async () => {
|
|
const { dir, brain, baselineId, floor } = await arrangeBaseline('amr')
|
|
const newId = uid('amr-new')
|
|
const arm = armCrash(brain, 'after-manifest-rename')
|
|
await expect(
|
|
brain.transact([
|
|
{ op: 'update', id: baselineId, metadata: { v: 2 } },
|
|
{
|
|
op: 'add',
|
|
id: newId,
|
|
type: NounType.Document,
|
|
data: 'committed by the rename',
|
|
vector: vec(2),
|
|
metadata: { v: 2 }
|
|
}
|
|
])
|
|
).rejects.toThrow('simulated process crash at after-manifest-rename')
|
|
expect(arm.fired).toContain('after-manifest-rename')
|
|
const txGen = storeOf(brain).generation()
|
|
await abandonAsCrashed(brain)
|
|
|
|
const reopened = await openLive(dir)
|
|
// COMMITTED: both operations present, atomically.
|
|
expect(((await reopened.get(baselineId)) as { metadata: { v: number } }).metadata.v).toBe(2)
|
|
const added = (await reopened.get(newId)) as { metadata: { v: number } } | null
|
|
expect(added).not.toBeNull()
|
|
expect(added!.metadata.v).toBe(2)
|
|
expect(storeOf(reopened).committedGeneration()).toBe(txGen)
|
|
// The fact was synced before the commit point and sits at/below the
|
|
// manifest — it is KEPT.
|
|
expect(await factGenerations(reopened)).toEqual([floor, txGen])
|
|
// Fresh writes continue above the committed generation.
|
|
const freshId = uid('amr-fresh')
|
|
await reopened.add({
|
|
id: freshId,
|
|
data: 'fresh',
|
|
type: NounType.Document,
|
|
vector: vec(3),
|
|
metadata: { v: 3 }
|
|
})
|
|
await reopened.flush()
|
|
expect(storeOf(reopened).committedGeneration()).toBe(txGen + 1)
|
|
})
|
|
|
|
// ==========================================================================
|
|
// Durable-at-ack (log-authority) mode variants
|
|
// ==========================================================================
|
|
|
|
it('singleop-after-fact-append (at-ack mode) — the intact fact is REPLAYED at reopen; the write commits', async () => {
|
|
const { dir, brain, baselineId, floor } = await arrangeBaseline('aaf')
|
|
await flipToAtAck(brain)
|
|
const crashedId = uid('aaf-crashed')
|
|
const arm = armCrash(brain, 'singleop-after-fact-append')
|
|
await expect(
|
|
brain.add({
|
|
id: crashedId,
|
|
data: 'fact fsynced, never acked',
|
|
type: NounType.Document,
|
|
vector: vec(2),
|
|
metadata: { v: 2 }
|
|
})
|
|
).rejects.toThrow('simulated process crash at singleop-after-fact-append')
|
|
expect(arm.fired).toContain('singleop-after-fact-append')
|
|
await abandonAsCrashed(brain)
|
|
|
|
const reopened = await openLive(dir)
|
|
// LOG-AUTHORITY RECOVERY CONTRACT: under 'log' authority, an intact
|
|
// fact above the manifest is adopted at open — REPLAYED into canonical
|
|
// and committed — never truncated. (At-least-once at the fact layer: a
|
|
// crashed-pre-ack write whose fact survived intact becomes committed;
|
|
// that is a valid write landing, never a torn or lost state.)
|
|
expect(await factGenerations(reopened)).toEqual([floor, floor + 1])
|
|
expect(storeOf(reopened).committedGeneration()).toBe(floor + 1)
|
|
expect(((await reopened.get(baselineId)) as { metadata: { v: number } }).metadata.v).toBe(1)
|
|
const replayed = (await reopened.get(crashedId)) as { metadata: { v: number } } | null
|
|
expect(replayed).not.toBeNull()
|
|
expect(replayed!.metadata.v).toBe(2)
|
|
// Fresh write lands monotonically ABOVE the replayed generation.
|
|
const freshId = uid('aaf-fresh')
|
|
await reopened.add({
|
|
id: freshId,
|
|
data: 'fresh',
|
|
type: NounType.Document,
|
|
vector: vec(3),
|
|
metadata: { v: 3 }
|
|
})
|
|
await reopened.flush()
|
|
expect(storeOf(reopened).committedGeneration()).toBe(floor + 2)
|
|
})
|
|
|
|
// THE AT-ACK CONTRACT, END TO END (was a release-blocking finding; fixed
|
|
// by log-authority replay-at-open): under power loss the un-fsynced
|
|
// tmp+rename canonical bytes legally vanish while the fsynced fact
|
|
// survives — recovery REPLAYS that fact into canonical, so the acked
|
|
// write lives. This is the sentence 'durable-at-ack' actually promises.
|
|
it(
|
|
'at-ack POWER LOSS — an ACKED write whose fact is fsynced SURVIVES reopen via log replay',
|
|
async () => {
|
|
const { dir, brain, baselineId } = await arrangeBaseline('apl')
|
|
await flipToAtAck(brain)
|
|
const ackedId = uid('apl-acked')
|
|
// No fault injector: this write ACKS normally — in at-ack mode the ack
|
|
// returned only after a covering log fsync.
|
|
await brain.add({
|
|
id: ackedId,
|
|
data: 'acked, fact fsynced',
|
|
type: NounType.Document,
|
|
vector: vec(2),
|
|
metadata: { v: 2 }
|
|
})
|
|
// Crash before any flush: RAM is gone…
|
|
await abandonAsCrashed(brain)
|
|
// …and power loss takes the un-fsynced canonical rename with it. The
|
|
// fsynced fact log survives — it is the write's only durable copy.
|
|
dropCanonicalNoun(dir, ackedId)
|
|
|
|
const reopened = await openLive(dir)
|
|
expect(((await reopened.get(baselineId)) as { metadata: { v: number } }).metadata.v).toBe(1)
|
|
// THE AT-ACK CONTRACT: the acknowledged write survives the crash.
|
|
// Observed today: open() truncates its fact back to the manifest
|
|
// watermark and the write is gone everywhere.
|
|
const acked = (await reopened.get(ackedId)) as { metadata: { v: number } } | null
|
|
expect(acked).not.toBeNull()
|
|
expect(acked!.metadata.v).toBe(2)
|
|
}
|
|
)
|
|
|
|
// ==========================================================================
|
|
// Disk full — one ENOSPC'd append
|
|
// ==========================================================================
|
|
|
|
it('disk full — an ENOSPC append rejects loudly and typed; reads keep serving; a later write succeeds', async () => {
|
|
const { dir, brain, baselineId, floor } = await arrangeBaseline('nospc')
|
|
liveBrains.push(brain) // this row never crashes the brain
|
|
void dir
|
|
const failedId = uid('nospc-failed')
|
|
const probe = failNextAppendWithEnospc(brain)
|
|
// LOUD, TYPED, never a silent success: the raw ENOSPC surfaces to the
|
|
// caller with its errno code intact.
|
|
await expect(
|
|
brain.add({
|
|
id: failedId,
|
|
data: 'no space',
|
|
type: NounType.Document,
|
|
vector: vec(2),
|
|
metadata: { v: 2 }
|
|
})
|
|
).rejects.toMatchObject({ code: 'ENOSPC' })
|
|
expect(probe.failed()).toBe(1)
|
|
// The store still serves reads.
|
|
expect(((await brain.get(baselineId)) as { metadata: { v: number } }).metadata.v).toBe(1)
|
|
// Space "restored" (the failing patch self-cleared): a later write succeeds
|
|
// end to end, including its fact and an explicit durability barrier.
|
|
const laterId = uid('nospc-later')
|
|
await brain.add({
|
|
id: laterId,
|
|
data: 'space restored',
|
|
type: NounType.Document,
|
|
vector: vec(3),
|
|
metadata: { v: 3 }
|
|
})
|
|
await brain.flush()
|
|
expect(((await brain.get(laterId)) as { metadata: { v: number } }).metadata.v).toBe(3)
|
|
expect(storeOf(brain).committedGeneration()).toBeGreaterThan(floor)
|
|
// FIXED BEHAVIOR (was: the rejected generation stayed buffered and the
|
|
// next flush committed it with NO fact — a silent log gap): the failure
|
|
// path un-buffers the generation and returns the counter reservation,
|
|
// so the later write takes floor+1 and the log is gap-free.
|
|
expect(storeOf(brain).committedGeneration()).toBe(floor + 1)
|
|
expect(await factGenerations(brain)).toEqual([floor, floor + 1])
|
|
// Canonical residue of the rejected write (execute ran before the
|
|
// append failed) is the documented Model-B crash-equivalent orphan —
|
|
// uncommitted, absent from the log, same shape as a crash at execute.
|
|
expect(((await brain.get(failedId)) as { metadata: { v: number } } | null)?.metadata.v).toBe(2)
|
|
})
|
|
|
|
// THE NO-SILENT-COMMIT CONTRACT (was a release-blocking finding; fixed by
|
|
// un-buffering on append failure): a loudly-rejected write never becomes
|
|
// durably committed and the log never carries a gap. Canonical residue
|
|
// (the execute-before-commit orphan) is the documented Model-B
|
|
// crash-equivalent, pinned in the row above — NOT a commit.
|
|
it('disk full — a write rejected for a failed fact append is NOT silently committed', async () => {
|
|
const { brain, floor } = await arrangeBaseline('nogap')
|
|
liveBrains.push(brain)
|
|
const failedId = uid('nogap-failed')
|
|
failNextAppendWithEnospc(brain)
|
|
await expect(
|
|
brain.add({
|
|
id: failedId,
|
|
data: 'no space',
|
|
type: NounType.Document,
|
|
vector: vec(2),
|
|
metadata: { v: 2 }
|
|
})
|
|
).rejects.toMatchObject({ code: 'ENOSPC' })
|
|
await brain.flush()
|
|
// THE CONTRACT: nothing was committed behind the caller's back — the
|
|
// log carries no gap and no generation for the rejected write. (get()
|
|
// still serves the canonical execute-residue orphan — the documented
|
|
// Model-B crash-equivalent, pinned in the row above.)
|
|
expect(storeOf(brain).committedGeneration()).toBe(floor)
|
|
expect(await factGenerations(brain)).toEqual([floor])
|
|
})
|
|
|
|
// ==========================================================================
|
|
// Block-layer power-loss findings (first dm-flakey run) — the three cures
|
|
// ==========================================================================
|
|
|
|
it('at-ack POWER LOSS BELOW THE MANIFEST — an unclean open folds the WHOLE log; acked writes committed before the flush still survive vanished canonical', async () => {
|
|
const { dir, brain, baselineId } = await arrangeBaseline('wlf')
|
|
await flipToAtAck(brain)
|
|
const ackedA = uid('wlf-a')
|
|
const ackedB = uid('wlf-b')
|
|
await brain.add({ id: ackedA, data: 'below manifest one', type: NounType.Document, vector: vec(2), metadata: { v: 2 } })
|
|
await brain.add({ id: ackedB, data: 'below manifest two', type: NounType.Document, vector: vec(3), metadata: { v: 3 } })
|
|
// The group-commit flush advances the manifest OVER these generations —
|
|
// but live canonical bytes are tmp+rename without per-file fsync, so a
|
|
// power cut can still take them. The fsynced facts are the durable copy.
|
|
await (brain as unknown as { flush(): Promise<void> }).flush()
|
|
await abandonAsCrashed(brain) // no clean close → no clean-shutdown marker
|
|
dropCanonicalNoun(dir, ackedA)
|
|
dropCanonicalNoun(dir, ackedB)
|
|
|
|
const reopened = await openLive(dir)
|
|
// The whole-log fold restores BOTH rows from facts ≤ manifest.
|
|
expect(((await reopened.get(ackedA)) as { metadata: { v: number } }).metadata.v).toBe(2)
|
|
expect(((await reopened.get(ackedB)) as { metadata: { v: number } }).metadata.v).toBe(3)
|
|
expect(((await reopened.get(baselineId)) as { metadata: { v: number } }).metadata.v).toBe(1)
|
|
})
|
|
|
|
it('clean-shutdown marker: a clean close writes it, the next open consumes it (no fold on the happy path)', async () => {
|
|
const { dir, brain } = await arrangeBaseline('csm')
|
|
await flipToAtAck(brain)
|
|
await brain.close()
|
|
liveBrains.splice(liveBrains.indexOf(brain), 1)
|
|
// The adapter stores raw objects gzipped — accept either spelling.
|
|
const markerExists = () =>
|
|
fs.existsSync(join(dir, '_system', 'clean-shutdown.json')) ||
|
|
fs.existsSync(join(dir, '_system', 'clean-shutdown.json.gz'))
|
|
expect(markerExists(), 'clean close stamps the marker').toBe(true)
|
|
|
|
const reopened = await openLive(dir)
|
|
expect(markerExists(), 'open consumes the marker').toBe(false)
|
|
await reopened.close()
|
|
liveBrains.splice(liveBrains.indexOf(reopened), 1)
|
|
expect(markerExists(), 'the next clean close re-stamps it').toBe(true)
|
|
})
|
|
|
|
it('torn writer lock (empty file) — open treats it as stale and recovers; never a permanent lockout', async () => {
|
|
const { dir, brain } = await arrangeBaseline('tlk')
|
|
await brain.close()
|
|
liveBrains.splice(liveBrains.indexOf(brain), 1)
|
|
// The power-loss shape: the lock file exists but is EMPTY (torn write).
|
|
fs.writeFileSync(join(dir, 'locks', '_writer.lock'), '')
|
|
|
|
const reopened = await openLive(dir) // must not throw 'contended'
|
|
const fresh = uid('tlk-fresh')
|
|
await reopened.add({ id: fresh, data: 'lock recovered', type: NounType.Document, vector: vec(4), metadata: { v: 4 } })
|
|
expect(await reopened.get(fresh)).not.toBeNull()
|
|
})
|
|
|
|
it('pair guard: a metadata index without stampWatermark never crashes flush', async () => {
|
|
const { brain } = await arrangeBaseline('psg')
|
|
liveBrains.push(brain)
|
|
// The native pair swaps the metadata manager; the replacement may not
|
|
// carry the stamp method — flush must treat that as verdict-side rescan,
|
|
// never a TypeError at the fan-out.
|
|
;(brain as unknown as { metadataIndex: { stampWatermark?: unknown } }).metadataIndex.stampWatermark = undefined
|
|
await expect((brain as unknown as { flush(): Promise<void> }).flush()).resolves.toBeUndefined()
|
|
})
|
|
})
|