open-brainy/tests/integration/recovery-walk-tolerance.test.ts
David Snelling 0e3facf4a8
All checks were successful
CI / Node 22 (push) Successful in 12m16s
CI / Node 24 (push) Successful in 12m13s
CI / Bun (latest) (push) Successful in 12m20s
fix(recovery): walks are healers — the typed/tolerant boundary redrawn where block-layer fault injection proved it belonged
The quiet-loss cure regressed recovery: the new typed torn-record error
was correct at identity-read time but threw inside init-time recovery
walks, killing opens that previously survived. The boundary, redrawn:

- IDENTITY READS (get-by-id of a specific record, CAS blob point-get):
  typed TornRecordError, unchanged — a caller who asked for THAT record
  can act on the answer.
- SET-SHAPED READS AND WALKS (enumeration, pagination, batch hydration —
  the paths recovery rebuilds and finds page over): HEAL PAST the torn
  victim. The adapter's loud floor (error log + counted gauge) fires at
  the encounter; the walk serves the remaining rows. One crash casualty
  can no longer kill every query on its shard — or the open itself.
- WRITES OVER TORN RECORDS ARE THE CURE: the save path's read-merge, the
  commit path's before-image capture, and the operations' rollback
  captures all treat a torn prior as the create sentinel, narrated — the
  incoming bytes replace the unreadable ones, and history for the id
  honestly restarts at that generation. Corruption can never block its
  own heal.
- THE NaN SOURCE: torn mapper state (nextId/entries carrying garbage)
  discards with narration and re-derives via the existing rebuild path;
  the mint gains a source guard healing a non-integer counter from the
  live map. The reopen and first-write RangeError shapes are dead at the
  source, both authority branches.

Pinned with the exact fault-injection scenarios: a torn entity record
(including the VFS root) no longer kills the open — walks heal past it,
the keeper rows serve, and the identity read of the victim itself is
typed-or-healed; a torn mapper reopens and mints sanely on the first
post-recovery write.

Gates: tsc 0 · unit 2065/2065 · integration 828 · conformance 31/31.
2026-08-11 09:20:30 -07:00

122 lines
5.4 KiB
TypeScript

/**
* @module tests/integration/recovery-walk-tolerance
* @description The rc6-red cures — the typed/tolerant boundary redrawn where
* block-layer fault injection proved it belonged:
* 1. WALKS ARE HEALERS: an init-time recovery/rebuild/pagination walk that
* meets a torn record narrates+counts (the adapter's loud floor) and
* HEALS PAST it — the open succeeds, remaining rows serve. rc6 died
* typed here; rc5 survived silently; the cure is loud survival.
* 2. IDENTITY READS STAY TYPED: get-by-id of the torn record itself still
* throws TornRecordError — a caller who asked for THAT record can act.
* 3. TORN MAPPER STATE (the NaN→BigInt source): a mapper file carrying
* garbage integers is discarded with narration; reopen succeeds and the
* FIRST WRITE after recovery mints sanely — never a RangeError.
*/
import { describe, it, expect, afterEach } from 'vitest'
import { mkdtempSync, rmSync, readdirSync, writeFileSync, existsSync, statSync } from 'node:fs'
import { tmpdir } from 'node:os'
import { join } from 'node:path'
import { gzipSync } from 'node:zlib'
import { Brainy, TornRecordError } from '../../src/index.js'
import { NounType } from '../../src/types/graphTypes.js'
const dirs: string[] = []
const brains: Brainy[] = []
afterEach(async () => {
for (const b of brains.splice(0)) await b.close().catch(() => {})
for (const d of dirs.splice(0)) rmSync(d, { recursive: true, force: true })
})
async function open(dir: string): Promise<Brainy> {
const b = new Brainy({ storage: { type: 'filesystem', path: dir }, requireSubtype: false })
await b.init()
brains.push(b)
return b
}
/** Find one entity metadata file under entities/nouns and tear it. */
function tearOneNounMetadata(dir: string, excludeId?: string): string {
const nounsRoot = join(dir, 'entities', 'nouns')
const walk = (d: string): string | null => {
for (const e of readdirSync(d, { withFileTypes: true })) {
const p = join(d, e.name)
if (e.isDirectory()) {
if (excludeId && e.name === excludeId) continue
const hit = walk(p)
if (hit) return hit
} else if (/^metadata\.json(\.gz)?$/.test(e.name)) {
writeFileSync(p, Buffer.from([0x1f, 0x8b, 0x00, 0xde, 0xad])) // torn gz
return p
}
}
return null
}
const torn = walk(nounsRoot)
if (!torn) throw new Error('layout probe: no noun metadata file found to tear')
// The id is the parent directory name.
return torn.split('/').slice(-2, -1)[0]
}
describe('recovery-walk tolerance (the rc6-red cures)', () => {
it('a torn entity record does not kill the open: recovery walks heal past it, remaining rows serve, identity read throws typed', async () => {
const dir = mkdtempSync(join(tmpdir(), 'brainy-walk-tol-'))
dirs.push(dir)
let brain = await open(dir)
const keeper = await brain.add({ data: 'keeper row', type: NounType.Document, metadata: { k: 1 } })
await brain.add({ data: 'victim row', type: NounType.Document, metadata: { k: 2 } })
await brain.flush()
await brain.close()
brains.pop()
const tornId = tearOneNounMetadata(dir, keeper)
// THE PIN: the open succeeds (rc6 died right here), the keeper serves,
// and walks (find) heal past the victim.
brain = await open(dir)
expect((await brain.get(keeper))!.data).toContain('keeper row')
const rows = await brain.find({ where: {}, limit: 10 })
expect(rows.map((r) => r.id)).toContain(keeper)
// Identity read of the victim itself: typed, catchable — the caller
// asked for THAT record; under log authority the replay may have
// already HEALED it from the fact log (also a valid outcome) — accept
// healed-or-typed, never silent-absent-without-narration.
try {
const victim = await brain.get(tornId)
// Healed by replay: the record must be real (log authority rewrote it).
expect(victim).not.toBeNull()
} catch (err) {
expect(err).toBeInstanceOf(TornRecordError)
}
}, 120000)
it('a torn mapper file (NaN ints) discards with narration; reopen succeeds and the first write mints sanely', async () => {
const dir = mkdtempSync(join(tmpdir(), 'brainy-torn-mapper-'))
dirs.push(dir)
let brain = await open(dir)
await brain.add({ data: 'pre-crash row', type: NounType.Document, metadata: { k: 1 } })
await brain.flush()
await brain.close()
brains.pop()
// The power-cut shape: the persisted mapper carries garbage integers.
const sys = join(dir, '_system')
const mapperPath = readdirSync(sys)
.filter((f) => /entityIdMapper/.test(f))
.map((f) => join(sys, f))[0]
expect(mapperPath, 'layout probe: mapper artifact exists').toBeTruthy()
const torn = { nextId: 'NaN-garbage', uuidToInt: { x: 'junk' }, intToUuid: { junk: 42 } }
if (mapperPath.endsWith('.gz')) writeFileSync(mapperPath, gzipSync(JSON.stringify(torn)))
else writeFileSync(mapperPath, JSON.stringify(torn))
expect(statSync(mapperPath).size).toBeGreaterThan(0)
// Reopen MUST succeed; the first write after recovery must mint sanely
// (rc6's fresh-write RangeError shape), and graph int resolution at
// reopen must not throw (rc6's reopen shape).
brain = await open(dir)
const fresh = await brain.add({ data: 'post-recovery write', type: NounType.Document, metadata: { k: 2 } })
expect((await brain.get(fresh))!.data).toContain('post-recovery')
await brain.flush()
expect(Number.isSafeInteger(brain.generation())).toBe(true)
}, 120000)
})