fix(recovery): walks are healers — the typed/tolerant boundary redrawn where block-layer fault injection proved it belonged
The quiet-loss cure regressed recovery: the new typed torn-record error was correct at identity-read time but threw inside init-time recovery walks, killing opens that previously survived. The boundary, redrawn: - IDENTITY READS (get-by-id of a specific record, CAS blob point-get): typed TornRecordError, unchanged — a caller who asked for THAT record can act on the answer. - SET-SHAPED READS AND WALKS (enumeration, pagination, batch hydration — the paths recovery rebuilds and finds page over): HEAL PAST the torn victim. The adapter's loud floor (error log + counted gauge) fires at the encounter; the walk serves the remaining rows. One crash casualty can no longer kill every query on its shard — or the open itself. - WRITES OVER TORN RECORDS ARE THE CURE: the save path's read-merge, the commit path's before-image capture, and the operations' rollback captures all treat a torn prior as the create sentinel, narrated — the incoming bytes replace the unreadable ones, and history for the id honestly restarts at that generation. Corruption can never block its own heal. - THE NaN SOURCE: torn mapper state (nextId/entries carrying garbage) discards with narration and re-derives via the existing rebuild path; the mint gains a source guard healing a non-integer counter from the live map. The reopen and first-write RangeError shapes are dead at the source, both authority branches. Pinned with the exact fault-injection scenarios: a torn entity record (including the VFS root) no longer kills the open — walks heal past it, the keeper rows serve, and the identity read of the victim itself is typed-or-healed; a torn mapper reopens and mints sanely on the first post-recovery write. Gates: tsc 0 · unit 2065/2065 · integration 828 · conformance 31/31.
This commit is contained in:
parent
214c98b4d5
commit
0e3facf4a8
6 changed files with 350 additions and 39 deletions
122
tests/integration/recovery-walk-tolerance.test.ts
Normal file
122
tests/integration/recovery-walk-tolerance.test.ts
Normal file
|
|
@ -0,0 +1,122 @@
|
|||
/**
|
||||
* @module tests/integration/recovery-walk-tolerance
|
||||
* @description The rc6-red cures — the typed/tolerant boundary redrawn where
|
||||
* block-layer fault injection proved it belonged:
|
||||
* 1. WALKS ARE HEALERS: an init-time recovery/rebuild/pagination walk that
|
||||
* meets a torn record narrates+counts (the adapter's loud floor) and
|
||||
* HEALS PAST it — the open succeeds, remaining rows serve. rc6 died
|
||||
* typed here; rc5 survived silently; the cure is loud survival.
|
||||
* 2. IDENTITY READS STAY TYPED: get-by-id of the torn record itself still
|
||||
* throws TornRecordError — a caller who asked for THAT record can act.
|
||||
* 3. TORN MAPPER STATE (the NaN→BigInt source): a mapper file carrying
|
||||
* garbage integers is discarded with narration; reopen succeeds and the
|
||||
* FIRST WRITE after recovery mints sanely — never a RangeError.
|
||||
*/
|
||||
import { describe, it, expect, afterEach } from 'vitest'
|
||||
import { mkdtempSync, rmSync, readdirSync, writeFileSync, existsSync, statSync } from 'node:fs'
|
||||
import { tmpdir } from 'node:os'
|
||||
import { join } from 'node:path'
|
||||
import { gzipSync } from 'node:zlib'
|
||||
import { Brainy, TornRecordError } from '../../src/index.js'
|
||||
import { NounType } from '../../src/types/graphTypes.js'
|
||||
|
||||
const dirs: string[] = []
|
||||
const brains: Brainy[] = []
|
||||
afterEach(async () => {
|
||||
for (const b of brains.splice(0)) await b.close().catch(() => {})
|
||||
for (const d of dirs.splice(0)) rmSync(d, { recursive: true, force: true })
|
||||
})
|
||||
|
||||
async function open(dir: string): Promise<Brainy> {
|
||||
const b = new Brainy({ storage: { type: 'filesystem', path: dir }, requireSubtype: false })
|
||||
await b.init()
|
||||
brains.push(b)
|
||||
return b
|
||||
}
|
||||
|
||||
/** Find one entity metadata file under entities/nouns and tear it. */
|
||||
function tearOneNounMetadata(dir: string, excludeId?: string): string {
|
||||
const nounsRoot = join(dir, 'entities', 'nouns')
|
||||
const walk = (d: string): string | null => {
|
||||
for (const e of readdirSync(d, { withFileTypes: true })) {
|
||||
const p = join(d, e.name)
|
||||
if (e.isDirectory()) {
|
||||
if (excludeId && e.name === excludeId) continue
|
||||
const hit = walk(p)
|
||||
if (hit) return hit
|
||||
} else if (/^metadata\.json(\.gz)?$/.test(e.name)) {
|
||||
writeFileSync(p, Buffer.from([0x1f, 0x8b, 0x00, 0xde, 0xad])) // torn gz
|
||||
return p
|
||||
}
|
||||
}
|
||||
return null
|
||||
}
|
||||
const torn = walk(nounsRoot)
|
||||
if (!torn) throw new Error('layout probe: no noun metadata file found to tear')
|
||||
// The id is the parent directory name.
|
||||
return torn.split('/').slice(-2, -1)[0]
|
||||
}
|
||||
|
||||
describe('recovery-walk tolerance (the rc6-red cures)', () => {
|
||||
it('a torn entity record does not kill the open: recovery walks heal past it, remaining rows serve, identity read throws typed', async () => {
|
||||
const dir = mkdtempSync(join(tmpdir(), 'brainy-walk-tol-'))
|
||||
dirs.push(dir)
|
||||
let brain = await open(dir)
|
||||
const keeper = await brain.add({ data: 'keeper row', type: NounType.Document, metadata: { k: 1 } })
|
||||
await brain.add({ data: 'victim row', type: NounType.Document, metadata: { k: 2 } })
|
||||
await brain.flush()
|
||||
await brain.close()
|
||||
brains.pop()
|
||||
|
||||
const tornId = tearOneNounMetadata(dir, keeper)
|
||||
|
||||
// THE PIN: the open succeeds (rc6 died right here), the keeper serves,
|
||||
// and walks (find) heal past the victim.
|
||||
brain = await open(dir)
|
||||
expect((await brain.get(keeper))!.data).toContain('keeper row')
|
||||
const rows = await brain.find({ where: {}, limit: 10 })
|
||||
expect(rows.map((r) => r.id)).toContain(keeper)
|
||||
|
||||
// Identity read of the victim itself: typed, catchable — the caller
|
||||
// asked for THAT record; under log authority the replay may have
|
||||
// already HEALED it from the fact log (also a valid outcome) — accept
|
||||
// healed-or-typed, never silent-absent-without-narration.
|
||||
try {
|
||||
const victim = await brain.get(tornId)
|
||||
// Healed by replay: the record must be real (log authority rewrote it).
|
||||
expect(victim).not.toBeNull()
|
||||
} catch (err) {
|
||||
expect(err).toBeInstanceOf(TornRecordError)
|
||||
}
|
||||
}, 120000)
|
||||
|
||||
it('a torn mapper file (NaN ints) discards with narration; reopen succeeds and the first write mints sanely', async () => {
|
||||
const dir = mkdtempSync(join(tmpdir(), 'brainy-torn-mapper-'))
|
||||
dirs.push(dir)
|
||||
let brain = await open(dir)
|
||||
await brain.add({ data: 'pre-crash row', type: NounType.Document, metadata: { k: 1 } })
|
||||
await brain.flush()
|
||||
await brain.close()
|
||||
brains.pop()
|
||||
|
||||
// The power-cut shape: the persisted mapper carries garbage integers.
|
||||
const sys = join(dir, '_system')
|
||||
const mapperPath = readdirSync(sys)
|
||||
.filter((f) => /entityIdMapper/.test(f))
|
||||
.map((f) => join(sys, f))[0]
|
||||
expect(mapperPath, 'layout probe: mapper artifact exists').toBeTruthy()
|
||||
const torn = { nextId: 'NaN-garbage', uuidToInt: { x: 'junk' }, intToUuid: { junk: 42 } }
|
||||
if (mapperPath.endsWith('.gz')) writeFileSync(mapperPath, gzipSync(JSON.stringify(torn)))
|
||||
else writeFileSync(mapperPath, JSON.stringify(torn))
|
||||
expect(statSync(mapperPath).size).toBeGreaterThan(0)
|
||||
|
||||
// Reopen MUST succeed; the first write after recovery must mint sanely
|
||||
// (rc6's fresh-write RangeError shape), and graph int resolution at
|
||||
// reopen must not throw (rc6's reopen shape).
|
||||
brain = await open(dir)
|
||||
const fresh = await brain.add({ data: 'post-recovery write', type: NounType.Document, metadata: { k: 2 } })
|
||||
expect((await brain.get(fresh))!.data).toContain('post-recovery')
|
||||
await brain.flush()
|
||||
expect(Number.isSafeInteger(brain.generation())).toBe(true)
|
||||
}, 120000)
|
||||
})
|
||||
Binary file not shown.
Reference in a new issue