The quiet-loss cure regressed recovery: the new typed torn-record error was correct at identity-read time but threw inside init-time recovery walks, killing opens that previously survived. The boundary, redrawn: - IDENTITY READS (get-by-id of a specific record, CAS blob point-get): typed TornRecordError, unchanged — a caller who asked for THAT record can act on the answer. - SET-SHAPED READS AND WALKS (enumeration, pagination, batch hydration — the paths recovery rebuilds and finds page over): HEAL PAST the torn victim. The adapter's loud floor (error log + counted gauge) fires at the encounter; the walk serves the remaining rows. One crash casualty can no longer kill every query on its shard — or the open itself. - WRITES OVER TORN RECORDS ARE THE CURE: the save path's read-merge, the commit path's before-image capture, and the operations' rollback captures all treat a torn prior as the create sentinel, narrated — the incoming bytes replace the unreadable ones, and history for the id honestly restarts at that generation. Corruption can never block its own heal. - THE NaN SOURCE: torn mapper state (nextId/entries carrying garbage) discards with narration and re-derives via the existing rebuild path; the mint gains a source guard healing a non-integer counter from the live map. The reopen and first-write RangeError shapes are dead at the source, both authority branches. Pinned with the exact fault-injection scenarios: a torn entity record (including the VFS root) no longer kills the open — walks heal past it, the keeper rows serve, and the identity read of the victim itself is typed-or-healed; a torn mapper reopens and mints sanely on the first post-recovery write. Gates: tsc 0 · unit 2065/2065 · integration 828 · conformance 31/31.
122 lines
5.4 KiB
TypeScript
122 lines
5.4 KiB
TypeScript
/**
|
|
* @module tests/integration/recovery-walk-tolerance
|
|
* @description The rc6-red cures — the typed/tolerant boundary redrawn where
|
|
* block-layer fault injection proved it belonged:
|
|
* 1. WALKS ARE HEALERS: an init-time recovery/rebuild/pagination walk that
|
|
* meets a torn record narrates+counts (the adapter's loud floor) and
|
|
* HEALS PAST it — the open succeeds, remaining rows serve. rc6 died
|
|
* typed here; rc5 survived silently; the cure is loud survival.
|
|
* 2. IDENTITY READS STAY TYPED: get-by-id of the torn record itself still
|
|
* throws TornRecordError — a caller who asked for THAT record can act.
|
|
* 3. TORN MAPPER STATE (the NaN→BigInt source): a mapper file carrying
|
|
* garbage integers is discarded with narration; reopen succeeds and the
|
|
* FIRST WRITE after recovery mints sanely — never a RangeError.
|
|
*/
|
|
import { describe, it, expect, afterEach } from 'vitest'
|
|
import { mkdtempSync, rmSync, readdirSync, writeFileSync, existsSync, statSync } from 'node:fs'
|
|
import { tmpdir } from 'node:os'
|
|
import { join } from 'node:path'
|
|
import { gzipSync } from 'node:zlib'
|
|
import { Brainy, TornRecordError } from '../../src/index.js'
|
|
import { NounType } from '../../src/types/graphTypes.js'
|
|
|
|
const dirs: string[] = []
|
|
const brains: Brainy[] = []
|
|
afterEach(async () => {
|
|
for (const b of brains.splice(0)) await b.close().catch(() => {})
|
|
for (const d of dirs.splice(0)) rmSync(d, { recursive: true, force: true })
|
|
})
|
|
|
|
async function open(dir: string): Promise<Brainy> {
|
|
const b = new Brainy({ storage: { type: 'filesystem', path: dir }, requireSubtype: false })
|
|
await b.init()
|
|
brains.push(b)
|
|
return b
|
|
}
|
|
|
|
/** Find one entity metadata file under entities/nouns and tear it. */
|
|
function tearOneNounMetadata(dir: string, excludeId?: string): string {
|
|
const nounsRoot = join(dir, 'entities', 'nouns')
|
|
const walk = (d: string): string | null => {
|
|
for (const e of readdirSync(d, { withFileTypes: true })) {
|
|
const p = join(d, e.name)
|
|
if (e.isDirectory()) {
|
|
if (excludeId && e.name === excludeId) continue
|
|
const hit = walk(p)
|
|
if (hit) return hit
|
|
} else if (/^metadata\.json(\.gz)?$/.test(e.name)) {
|
|
writeFileSync(p, Buffer.from([0x1f, 0x8b, 0x00, 0xde, 0xad])) // torn gz
|
|
return p
|
|
}
|
|
}
|
|
return null
|
|
}
|
|
const torn = walk(nounsRoot)
|
|
if (!torn) throw new Error('layout probe: no noun metadata file found to tear')
|
|
// The id is the parent directory name.
|
|
return torn.split('/').slice(-2, -1)[0]
|
|
}
|
|
|
|
describe('recovery-walk tolerance (the rc6-red cures)', () => {
|
|
it('a torn entity record does not kill the open: recovery walks heal past it, remaining rows serve, identity read throws typed', async () => {
|
|
const dir = mkdtempSync(join(tmpdir(), 'brainy-walk-tol-'))
|
|
dirs.push(dir)
|
|
let brain = await open(dir)
|
|
const keeper = await brain.add({ data: 'keeper row', type: NounType.Document, metadata: { k: 1 } })
|
|
await brain.add({ data: 'victim row', type: NounType.Document, metadata: { k: 2 } })
|
|
await brain.flush()
|
|
await brain.close()
|
|
brains.pop()
|
|
|
|
const tornId = tearOneNounMetadata(dir, keeper)
|
|
|
|
// THE PIN: the open succeeds (rc6 died right here), the keeper serves,
|
|
// and walks (find) heal past the victim.
|
|
brain = await open(dir)
|
|
expect((await brain.get(keeper))!.data).toContain('keeper row')
|
|
const rows = await brain.find({ where: {}, limit: 10 })
|
|
expect(rows.map((r) => r.id)).toContain(keeper)
|
|
|
|
// Identity read of the victim itself: typed, catchable — the caller
|
|
// asked for THAT record; under log authority the replay may have
|
|
// already HEALED it from the fact log (also a valid outcome) — accept
|
|
// healed-or-typed, never silent-absent-without-narration.
|
|
try {
|
|
const victim = await brain.get(tornId)
|
|
// Healed by replay: the record must be real (log authority rewrote it).
|
|
expect(victim).not.toBeNull()
|
|
} catch (err) {
|
|
expect(err).toBeInstanceOf(TornRecordError)
|
|
}
|
|
}, 120000)
|
|
|
|
it('a torn mapper file (NaN ints) discards with narration; reopen succeeds and the first write mints sanely', async () => {
|
|
const dir = mkdtempSync(join(tmpdir(), 'brainy-torn-mapper-'))
|
|
dirs.push(dir)
|
|
let brain = await open(dir)
|
|
await brain.add({ data: 'pre-crash row', type: NounType.Document, metadata: { k: 1 } })
|
|
await brain.flush()
|
|
await brain.close()
|
|
brains.pop()
|
|
|
|
// The power-cut shape: the persisted mapper carries garbage integers.
|
|
const sys = join(dir, '_system')
|
|
const mapperPath = readdirSync(sys)
|
|
.filter((f) => /entityIdMapper/.test(f))
|
|
.map((f) => join(sys, f))[0]
|
|
expect(mapperPath, 'layout probe: mapper artifact exists').toBeTruthy()
|
|
const torn = { nextId: 'NaN-garbage', uuidToInt: { x: 'junk' }, intToUuid: { junk: 42 } }
|
|
if (mapperPath.endsWith('.gz')) writeFileSync(mapperPath, gzipSync(JSON.stringify(torn)))
|
|
else writeFileSync(mapperPath, JSON.stringify(torn))
|
|
expect(statSync(mapperPath).size).toBeGreaterThan(0)
|
|
|
|
// Reopen MUST succeed; the first write after recovery must mint sanely
|
|
// (rc6's fresh-write RangeError shape), and graph int resolution at
|
|
// reopen must not throw (rc6's reopen shape).
|
|
brain = await open(dir)
|
|
const fresh = await brain.add({ data: 'post-recovery write', type: NounType.Document, metadata: { k: 2 } })
|
|
expect((await brain.get(fresh))!.data).toContain('post-recovery')
|
|
await brain.flush()
|
|
expect(Number.isSafeInteger(brain.generation())).toBe(true)
|
|
}, 120000)
|
|
})
|