From 0e3facf4a8896c6fc2b4e55518b8cd468b2678fc Mon Sep 17 00:00:00 2001 From: David Snelling Date: Tue, 11 Aug 2026 09:20:30 -0700 Subject: [PATCH] =?UTF-8?q?fix(recovery):=20walks=20are=20healers=20?= =?UTF-8?q?=E2=80=94=20the=20typed/tolerant=20boundary=20redrawn=20where?= =?UTF-8?q?=20block-layer=20fault=20injection=20proved=20it=20belonged?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The quiet-loss cure regressed recovery: the new typed torn-record error was correct at identity-read time but threw inside init-time recovery walks, killing opens that previously survived. The boundary, redrawn: - IDENTITY READS (get-by-id of a specific record, CAS blob point-get): typed TornRecordError, unchanged — a caller who asked for THAT record can act on the answer. - SET-SHAPED READS AND WALKS (enumeration, pagination, batch hydration — the paths recovery rebuilds and finds page over): HEAL PAST the torn victim. The adapter's loud floor (error log + counted gauge) fires at the encounter; the walk serves the remaining rows. One crash casualty can no longer kill every query on its shard — or the open itself. - WRITES OVER TORN RECORDS ARE THE CURE: the save path's read-merge, the commit path's before-image capture, and the operations' rollback captures all treat a torn prior as the create sentinel, narrated — the incoming bytes replace the unreadable ones, and history for the id honestly restarts at that generation. Corruption can never block its own heal. - THE NaN SOURCE: torn mapper state (nextId/entries carrying garbage) discards with narration and re-derives via the existing rebuild path; the mint gains a source guard healing a non-integer counter from the live map. The reopen and first-write RangeError shapes are dead at the source, both authority branches. Pinned with the exact fault-injection scenarios: a torn entity record (including the VFS root) no longer kills the open — walks heal past it, the keeper rows serve, and the identity read of the victim itself is typed-or-healed; a torn mapper reopens and mints sanely on the first post-recovery write. Gates: tsc 0 · unit 2065/2065 · integration 828 · conformance 31/31. --- src/db/generationStore.ts | 39 +++++- src/storage/baseStorage.ts | 126 ++++++++++++++---- .../operations/StorageOperations.ts | 42 +++++- src/utils/entityIdMapper.ts | 60 ++++++++- .../recovery-walk-tolerance.test.ts | 122 +++++++++++++++++ tests/unit/storage/torn-record-loud.test.ts | Bin 10240 -> 11080 bytes 6 files changed, 350 insertions(+), 39 deletions(-) create mode 100644 tests/integration/recovery-walk-tolerance.test.ts diff --git a/src/db/generationStore.ts b/src/db/generationStore.ts index 1de6dd51..6922da6d 100644 --- a/src/db/generationStore.ts +++ b/src/db/generationStore.ts @@ -918,6 +918,37 @@ export class GenerationStore { else this.pins.set(gen, count - 1) } + + /** + * Torn-tolerant raw read for BEFORE-IMAGE contexts: a write landing on a + * TORN record (power-loss survivor) is a HEAL — the new after-image + * replaces the unreadable bytes. The before-image is unknowable, so it + * reads as the CREATE SENTINEL ({metadata:null, vector:null}) with + * narration: history for this id restarts at this generation (an asOf + * below it resolves absent for the id — the honest statement of what the + * crash destroyed). The adapter's loud floor (error + gauge) fired at + * throw time; real storage faults still propagate. + */ + private async readRawForBeforeImage( + kind: 'noun' | 'verb', + id: string + ): Promise<{ metadata: unknown | null; vector: unknown | null }> { + try { + return kind === 'noun' + ? await this.storage.readNounRaw(id) + : await this.storage.readVerbRaw(id) + } catch (err) { + if ((err as { code?: string }).code === 'TORN_RECORD') { + prodLog.warn( + `[GenerationStore] before-image of ${kind} ${id} is TORN — the incoming ` + + `write HEALS the record; its history restarts at this generation` + ) + return { metadata: null, vector: null } + } + throw err + } + } + /** @returns Total number of live pins across all generations. */ activePinCount(): number { let total = 0 @@ -1083,11 +1114,11 @@ export class GenerationStore { // conflicting batch aborts with zero staging I/O. The maps hold the // byte-identical records the staged files are written from. for (const id of nouns) { - const prev = await this.storage.readNounRaw(id) + const prev = await this.readRawForBeforeImage('noun', id) nounBefore.set(id, { kind: 'noun', metadata: prev.metadata, vector: prev.vector }) } for (const id of verbs) { - const prev = await this.storage.readVerbRaw(id) + const prev = await this.readRawForBeforeImage('verb', id) verbBefore.set(id, { kind: 'verb', metadata: prev.metadata, vector: prev.vector }) } @@ -1415,12 +1446,12 @@ export class GenerationStore { // {metadata:null, vector:null} = the create sentinel. const nounBefore = new Map() for (const id of nouns) { - const prev = await this.storage.readNounRaw(id) + const prev = await this.readRawForBeforeImage('noun', id) nounBefore.set(id, { kind: 'noun', metadata: prev.metadata, vector: prev.vector }) } const verbBefore = new Map() for (const id of verbs) { - const prev = await this.storage.readVerbRaw(id) + const prev = await this.readRawForBeforeImage('verb', id) verbBefore.set(id, { kind: 'verb', metadata: prev.metadata, vector: prev.vector }) } diff --git a/src/storage/baseStorage.ts b/src/storage/baseStorage.ts index aefa6e04..23003ede 100644 --- a/src/storage/baseStorage.ts +++ b/src/storage/baseStorage.ts @@ -677,7 +677,8 @@ export abstract class BaseStorage extends BaseStorageAdapter { } catch (error) { // A TORN blob object (exists but undecodable) must not read as // "blob absent" — that would misdiagnose disk corruption as a - // missing blob. Propagate the typed error to the blob layer. + // missing blob. This is an IDENTITY read (a caller asked for THIS + // key): propagate the typed error to the blob layer. if (isTornRecordError(error)) throw error return undefined } @@ -2183,7 +2184,11 @@ export abstract class BaseStorage extends BaseStorageAdapter { } catch (error) { // A TORN record must surface typed — a paginated read that // silently skips a corrupt row hides data loss from the caller. - if (isTornRecordError(error)) throw error + // Torn record inside an ENUMERATION/RECOVERY walk: the adapter already + // narrated + counted it (TornRecordError registers at creation); the + // walk's job is to HEAL PAST it — skip the victim, serve the rest. + // Identity point-reads (get-by-id) still throw typed upstream. + if (isTornRecordError(error)) { /* skip torn victim; loud floor already fired */ } // Skip nouns that fail to load return null } @@ -2214,7 +2219,11 @@ export abstract class BaseStorage extends BaseStorageAdapter { } } catch (error) { // A TORN record propagates (typed) — only shard-listing absence is skippable. - if (isTornRecordError(error)) throw error + // Torn record inside an ENUMERATION/RECOVERY walk: the adapter already + // narrated + counted it (TornRecordError registers at creation); the + // walk's job is to HEAL PAST it — skip the victim, serve the rest. + // Identity point-reads (get-by-id) still throw typed upstream. + if (isTornRecordError(error)) { /* skip torn victim; loud floor already fired */ } // Skip shards that have no data } } @@ -2325,7 +2334,11 @@ export abstract class BaseStorage extends BaseStorageAdapter { return { id, metadata: await this.getNounMetadata(id) } } catch (error) { // A TORN record must surface typed, never as a skipped id. - if (isTornRecordError(error)) throw error + // Torn record inside an ENUMERATION/RECOVERY walk: the adapter already + // narrated + counted it (TornRecordError registers at creation); the + // walk's job is to HEAL PAST it — skip the victim, serve the rest. + // Identity point-reads (get-by-id) still throw typed upstream. + if (isTornRecordError(error)) { /* skip torn victim; loud floor already fired */ } return null } }) @@ -2348,7 +2361,11 @@ export abstract class BaseStorage extends BaseStorageAdapter { } } catch (error) { // A TORN record propagates (typed) — only shard-listing absence is skippable. - if (isTornRecordError(error)) throw error + // Torn record inside an ENUMERATION/RECOVERY walk: the adapter already + // narrated + counted it (TornRecordError registers at creation); the + // walk's job is to HEAL PAST it — skip the victim, serve the rest. + // Identity point-reads (get-by-id) still throw typed upstream. + if (isTornRecordError(error)) { /* skip torn victim; loud floor already fired */ } // Skip shards with no data } } @@ -2561,13 +2578,21 @@ export abstract class BaseStorage extends BaseStorageAdapter { } catch (error) { // A TORN record must surface typed — a paginated read that // silently skips a corrupt row hides data loss from the caller. - if (isTornRecordError(error)) throw error + // Torn record inside an ENUMERATION/RECOVERY walk: the adapter already + // narrated + counted it (TornRecordError registers at creation); the + // walk's job is to HEAL PAST it — skip the victim, serve the rest. + // Identity point-reads (get-by-id) still throw typed upstream. + if (isTornRecordError(error)) { /* skip torn victim; loud floor already fired */ } // Skip verbs that fail to load } } } catch (error) { // A TORN record propagates (typed) — only shard-listing absence is skippable. - if (isTornRecordError(error)) throw error + // Torn record inside an ENUMERATION/RECOVERY walk: the adapter already + // narrated + counted it (TornRecordError registers at creation); the + // walk's job is to HEAL PAST it — skip the victim, serve the rest. + // Identity point-reads (get-by-id) still throw typed upstream. + if (isTornRecordError(error)) { /* skip torn victim; loud floor already fired */ } // Skip shards that have no data } } @@ -3352,7 +3377,14 @@ export abstract class BaseStorage extends BaseStorageAdapter { const path = getNounMetadataPath(id) // Determine if this is a new entity by checking if metadata already exists - const existingMetadata = await this.readCanonicalObject(path) + // Torn-tolerant: a WRITE landing on a torn record HEALS it — the read + // here only classifies new-vs-update and captures the prior subtype; + // a torn prior reads as "no previous" (fresh write) with the adapter's + // loud floor already fired. Never let corruption block its own cure. + const existingMetadata = await this.readCanonicalObject(path).catch((err) => { + if ((err as { code?: string }).code === 'TORN_RECORD') return null + throw err + }) const isNew = !existingMetadata // Save the metadata (write-cache coherent canonical write) @@ -3722,12 +3754,17 @@ export abstract class BaseStorage extends BaseStorageAdapter { if (result.value.data !== null) { results.set(result.value.path, result.value.data) } + } else if (isTornRecordError(result.reason)) { + // A torn record inside a SET-SHAPED read (batch hydration behind + // find/sort pages and recovery walks): the adapter narrated + + // counted at throw time; the batch HEALS PAST the victim and + // serves the remaining rows — one crash casualty must not kill + // every query that pages over its shard (and init-time recovery + // walks ride this exact path). Identity point-reads still throw. + continue } else { - // A rejected read is a torn record or a real storage fault — NOT an - // absent object. Batch hydration backs entity reads (getNounBatch / - // getVerbsBatch / find hydration); swallowing the rejection would - // silently drop a row the caller cannot distinguish from "never - // existed". Propagate the typed/real error loudly instead. + // A REAL storage fault (EIO-class) is not a torn victim — + // propagate loudly, never absorb. throw result.reason } } @@ -3864,7 +3901,14 @@ export abstract class BaseStorage extends BaseStorageAdapter { const path = getVerbMetadataPath(id) // Determine if this is a new verb by checking if metadata already exists - const existingMetadata = await this.readCanonicalObject(path) + // Torn-tolerant: a WRITE landing on a torn record HEALS it — the read + // here only classifies new-vs-update and captures the prior subtype; + // a torn prior reads as "no previous" (fresh write) with the adapter's + // loud floor already fired. Never let corruption block its own cure. + const existingMetadata = await this.readCanonicalObject(path).catch((err) => { + if ((err as { code?: string }).code === 'TORN_RECORD') return null + throw err + }) const isNew = !existingMetadata // Save the metadata (write-cache coherent canonical write) @@ -4696,13 +4740,21 @@ export abstract class BaseStorage extends BaseStorageAdapter { } catch (error) { // A TORN record must surface typed — an enumeration that silently // skips a corrupt row hides data loss from the caller. - if (isTornRecordError(error)) throw error + // Torn record inside an ENUMERATION/RECOVERY walk: the adapter already + // narrated + counted it (TornRecordError registers at creation); the + // walk's job is to HEAL PAST it — skip the victim, serve the rest. + // Identity point-reads (get-by-id) still throw typed upstream. + if (isTornRecordError(error)) { /* skip torn victim; loud floor already fired */ } // Skip nouns that fail to load } } } catch (error) { // A TORN record propagates (typed) — only shard-listing absence is skippable. - if (isTornRecordError(error)) throw error + // Torn record inside an ENUMERATION/RECOVERY walk: the adapter already + // narrated + counted it (TornRecordError registers at creation); the + // walk's job is to HEAL PAST it — skip the victim, serve the rest. + // Identity point-reads (get-by-id) still throw typed upstream. + if (isTornRecordError(error)) { /* skip torn victim; loud floor already fired */ } // Skip shards that have no data } } @@ -4890,14 +4942,22 @@ export abstract class BaseStorage extends BaseStorageAdapter { } catch (error) { // A TORN record must surface typed — an enumeration that silently // skips a corrupt row hides data loss from the caller. - if (isTornRecordError(error)) throw error + // Torn record inside an ENUMERATION/RECOVERY walk: the adapter already + // narrated + counted it (TornRecordError registers at creation); the + // walk's job is to HEAL PAST it — skip the victim, serve the rest. + // Identity point-reads (get-by-id) still throw typed upstream. + if (isTornRecordError(error)) { /* skip torn victim; loud floor already fired */ } // Skip verbs that fail to load prodLog.debug(`[BaseStorage] Failed to load verb from ${verbPath}:`, error) } } } catch (error) { // A TORN record propagates (typed) — only shard-listing absence is skippable. - if (isTornRecordError(error)) throw error + // Torn record inside an ENUMERATION/RECOVERY walk: the adapter already + // narrated + counted it (TornRecordError registers at creation); the + // walk's job is to HEAL PAST it — skip the victim, serve the rest. + // Identity point-reads (get-by-id) still throw typed upstream. + if (isTornRecordError(error)) { /* skip torn victim; loud floor already fired */ } // Skip shards that have no data } } @@ -5015,7 +5075,11 @@ export abstract class BaseStorage extends BaseStorageAdapter { } catch (error) { // A TORN record propagates (typed) — batch hydration must not // silently drop a corrupt row. Only shard-listing absence is skippable. - if (isTornRecordError(error)) throw error + // Torn record inside an ENUMERATION/RECOVERY walk: the adapter already + // narrated + counted it (TornRecordError registers at creation); the + // walk's job is to HEAL PAST it — skip the victim, serve the rest. + // Identity point-reads (get-by-id) still throw typed upstream. + if (isTornRecordError(error)) { /* skip torn victim; loud floor already fired */ } // Skip shards that have no data } } @@ -5103,13 +5167,21 @@ export abstract class BaseStorage extends BaseStorageAdapter { } catch (error) { // A TORN record must surface typed — an enumeration that silently // skips a corrupt row hides data loss from the caller. - if (isTornRecordError(error)) throw error + // Torn record inside an ENUMERATION/RECOVERY walk: the adapter already + // narrated + counted it (TornRecordError registers at creation); the + // walk's job is to HEAL PAST it — skip the victim, serve the rest. + // Identity point-reads (get-by-id) still throw typed upstream. + if (isTornRecordError(error)) { /* skip torn victim; loud floor already fired */ } // Skip verbs that fail to load } } } catch (error) { // A TORN record propagates (typed) — only shard-listing absence is skippable. - if (isTornRecordError(error)) throw error + // Torn record inside an ENUMERATION/RECOVERY walk: the adapter already + // narrated + counted it (TornRecordError registers at creation); the + // walk's job is to HEAL PAST it — skip the victim, serve the rest. + // Identity point-reads (get-by-id) still throw typed upstream. + if (isTornRecordError(error)) { /* skip torn victim; loud floor already fired */ } // Skip shards that have no data } } @@ -5156,13 +5228,21 @@ export abstract class BaseStorage extends BaseStorageAdapter { } catch (error) { // A TORN record must surface typed — an enumeration that silently // skips a corrupt row hides data loss from the caller. - if (isTornRecordError(error)) throw error + // Torn record inside an ENUMERATION/RECOVERY walk: the adapter already + // narrated + counted it (TornRecordError registers at creation); the + // walk's job is to HEAL PAST it — skip the victim, serve the rest. + // Identity point-reads (get-by-id) still throw typed upstream. + if (isTornRecordError(error)) { /* skip torn victim; loud floor already fired */ } // Skip verbs that fail to load } } } catch (error) { // A TORN record propagates (typed) — only shard-listing absence is skippable. - if (isTornRecordError(error)) throw error + // Torn record inside an ENUMERATION/RECOVERY walk: the adapter already + // narrated + counted it (TornRecordError registers at creation); the + // walk's job is to HEAL PAST it — skip the victim, serve the rest. + // Identity point-reads (get-by-id) still throw typed upstream. + if (isTornRecordError(error)) { /* skip torn victim; loud floor already fired */ } // Skip shards that have no data } } diff --git a/src/transaction/operations/StorageOperations.ts b/src/transaction/operations/StorageOperations.ts index 9858219b..c1e9f1c1 100644 --- a/src/transaction/operations/StorageOperations.ts +++ b/src/transaction/operations/StorageOperations.ts @@ -12,6 +12,7 @@ import type { StorageAdapter, HNSWNoun, HNSWVerb, NounMetadata, VerbMetadata } from '../../coreTypes.js' import type { Operation, RollbackAction } from '../types.js' +import { prodLog } from '../../utils/logger.js' /** * Save noun metadata with rollback support @@ -20,6 +21,30 @@ import type { Operation, RollbackAction } from '../types.js' * - If metadata existed: Restore previous metadata * - If metadata was new: Delete metadata */ + +/** + * Torn-tolerant previous-state read for ROLLBACK CAPTURE: a write or delete + * landing on a TORN record (power-loss survivor) HEALS it — the incoming + * bytes replace (or remove) the unreadable ones, and the rollback target is + * the create sentinel (null). The adapter's loud floor (error + gauge) + * already fired at throw time; this narrates the heal and proceeds. Real + * storage faults still propagate. + */ +async function tornHealsToNull(read: Promise, what: string): Promise { + try { + return await read + } catch (err) { + if ((err as { code?: string }).code === 'TORN_RECORD') { + prodLog.warn( + `[StorageOperations] previous ${what} is TORN — the incoming operation ` + + `heals it; rollback target is the create sentinel` + ) + return null + } + throw err + } +} + export class SaveNounMetadataOperation implements Operation { readonly name = 'SaveNounMetadata' @@ -34,7 +59,7 @@ export class SaveNounMetadataOperation implements Operation { // Skip read for new entities — nothing to rollback to (saves 1 storage round-trip) const previousMetadata = this.isNew ? null - : await this.storage.getNounMetadata(this.id) + : await tornHealsToNull(this.storage.getNounMetadata(this.id), 'noun metadata') // Save new metadata await this.storage.saveNounMetadata(this.id, this.metadata) @@ -75,7 +100,7 @@ export class SaveNounOperation implements Operation { // Skip read for new entities — nothing to rollback to (saves 1 storage round-trip) const previousNoun = this.isNew ? null - : await this.storage.getNoun(this.noun.id) + : await tornHealsToNull(this.storage.getNoun(this.noun.id), 'noun record') // PRESERVE stored graph state on updates. Callers stage this op with // placeholder adjacency ({connections: empty, level: 0}) because the @@ -162,8 +187,11 @@ export class DeleteNounMetadataOperation implements Operation { // Capture the FULL before-image (both legs) so the undo restores the whole // entity — a metadata-only rollback would leave the vector leg unrestored. // A null metadata read falls back to the caller's pre-delete read. - const previousNoun = await this.storage.getNoun(this.id) - const previousMetadata = (await this.storage.getNounMetadata(this.id)) ?? this.priorMetadata ?? null + const previousNoun = await tornHealsToNull(this.storage.getNoun(this.id), 'noun record') + const previousMetadata = + (await tornHealsToNull(this.storage.getNounMetadata(this.id), 'noun metadata')) ?? + this.priorMetadata ?? + null if (!previousNoun && !previousMetadata) { // Nothing to delete - no rollback needed @@ -211,7 +239,7 @@ export class SaveVerbMetadataOperation implements Operation { async execute(): Promise { // Get existing metadata (for rollback) - const previousMetadata = await this.storage.getVerbMetadata(this.id) + const previousMetadata = await tornHealsToNull(this.storage.getVerbMetadata(this.id), 'verb metadata') // Save new metadata await this.storage.saveVerbMetadata(this.id, this.metadata) @@ -247,7 +275,7 @@ export class SaveVerbOperation implements Operation { async execute(): Promise { // Get existing verb (for rollback) - const previousVerb = await this.storage.getVerb(this.verb.id) + const previousVerb = await tornHealsToNull(this.storage.getVerb(this.verb.id), 'verb record') // Save new verb await this.storage.saveVerb(this.verb) @@ -291,7 +319,7 @@ export class DeleteVerbMetadataOperation implements Operation { async execute(): Promise { // Get metadata before deletion (for rollback) - const previousMetadata = await this.storage.getVerbMetadata(this.id) + const previousMetadata = await tornHealsToNull(this.storage.getVerbMetadata(this.id), 'verb metadata') if (!previousMetadata) { // Nothing to delete - no rollback needed diff --git a/src/utils/entityIdMapper.ts b/src/utils/entityIdMapper.ts index f359719b..d3527d77 100644 --- a/src/utils/entityIdMapper.ts +++ b/src/utils/entityIdMapper.ts @@ -129,11 +129,49 @@ export class EntityIdMapper implements EntityIdMapperProvider { // metadata channel as plain JSON; the `nextId` probe above identifies // the persisted EntityIdMapperData shape. const data = metadata as unknown as EntityIdMapperData - this.nextId = data.nextId - // Rebuild maps from serialized data - this.uuidToInt = new Map(Object.entries(data.uuidToInt).map(([k, v]) => [k, Number(v)])) - this.intToUuid = new Map(Object.entries(data.intToUuid).map(([k, v]) => [Number(k), v])) + // TORN-STATE VALIDATION (power-loss survivor): a torn mapper file + // can carry NaN/garbage where integers belong — unvalidated, those + // NaNs reach BigInt() on the graph's int-resolution (reopen) and + // the mint path (first write after recovery) and kill both with + // RangeErrors. A torn mapper is DISCARDED with narration and the + // maps re-derive through the existing rebuild path (under log + // authority the mint-at-append records reproduce assignments + // exactly; under tree authority the metadata-index reconstruction + // rebuilds them — the same path a missing mapper file takes). + const validInt = (v: unknown): v is number => + typeof v === 'number' && Number.isSafeInteger(v) && v >= 0 + let torn = !validInt(data.nextId) + const uuidToInt = new Map() + const intToUuid = new Map() + if (!torn) { + for (const [k, v] of Object.entries(data.uuidToInt ?? {})) { + const n = Number(v) + if (!validInt(n)) { torn = true; break } + uuidToInt.set(k, n) + } + } + if (!torn) { + for (const [k, v] of Object.entries(data.intToUuid ?? {})) { + const n = Number(k) + if (!validInt(n) || typeof v !== 'string') { torn = true; break } + intToUuid.set(n, v) + } + } + if (torn) { + console.warn( + `[EntityIdMapper] persisted mapper state is TORN (non-integer ids — ` + + `power-loss survivor); discarding and re-deriving via the rebuild ` + + `path. Never a RangeError at reopen or first write.` + ) + this.nextId = 1 + this.uuidToInt = new Map() + this.intToUuid = new Map() + } else { + this.nextId = data.nextId + this.uuidToInt = uuidToInt + this.intToUuid = intToUuid + } } else { // Guard: mapper file missing but entities may exist on disk. // If we start from nextId=1 with existing entities, roaring bitmap @@ -178,7 +216,19 @@ export class EntityIdMapper implements EntityIdMapperProvider { return existing } - // Assign new ID + // Assign new ID. Source guard: nextId must be a finite positive integer + // — the load path validates persisted state, but a NaN here would mint + // poison ints that reach BigInt() downstream; heal to the map-derived + // floor with narration rather than propagate. + if (!Number.isSafeInteger(this.nextId) || this.nextId < 1) { + let floor = 1 + for (const n of this.intToUuid.keys()) if (n >= floor) floor = n + 1 + console.warn( + `[EntityIdMapper] nextId was non-integer (${String(this.nextId)}) — ` + + `healed to ${floor} from the live map; torn-state survivor` + ) + this.nextId = floor + } if (this.nextId > U32_ENTITY_ID_MAX) { throw new EntityIdSpaceExceeded(this.nextId) } diff --git a/tests/integration/recovery-walk-tolerance.test.ts b/tests/integration/recovery-walk-tolerance.test.ts new file mode 100644 index 00000000..6a37e3bd --- /dev/null +++ b/tests/integration/recovery-walk-tolerance.test.ts @@ -0,0 +1,122 @@ +/** + * @module tests/integration/recovery-walk-tolerance + * @description The rc6-red cures — the typed/tolerant boundary redrawn where + * block-layer fault injection proved it belonged: + * 1. WALKS ARE HEALERS: an init-time recovery/rebuild/pagination walk that + * meets a torn record narrates+counts (the adapter's loud floor) and + * HEALS PAST it — the open succeeds, remaining rows serve. rc6 died + * typed here; rc5 survived silently; the cure is loud survival. + * 2. IDENTITY READS STAY TYPED: get-by-id of the torn record itself still + * throws TornRecordError — a caller who asked for THAT record can act. + * 3. TORN MAPPER STATE (the NaN→BigInt source): a mapper file carrying + * garbage integers is discarded with narration; reopen succeeds and the + * FIRST WRITE after recovery mints sanely — never a RangeError. + */ +import { describe, it, expect, afterEach } from 'vitest' +import { mkdtempSync, rmSync, readdirSync, writeFileSync, existsSync, statSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { gzipSync } from 'node:zlib' +import { Brainy, TornRecordError } from '../../src/index.js' +import { NounType } from '../../src/types/graphTypes.js' + +const dirs: string[] = [] +const brains: Brainy[] = [] +afterEach(async () => { + for (const b of brains.splice(0)) await b.close().catch(() => {}) + for (const d of dirs.splice(0)) rmSync(d, { recursive: true, force: true }) +}) + +async function open(dir: string): Promise { + const b = new Brainy({ storage: { type: 'filesystem', path: dir }, requireSubtype: false }) + await b.init() + brains.push(b) + return b +} + +/** Find one entity metadata file under entities/nouns and tear it. */ +function tearOneNounMetadata(dir: string, excludeId?: string): string { + const nounsRoot = join(dir, 'entities', 'nouns') + const walk = (d: string): string | null => { + for (const e of readdirSync(d, { withFileTypes: true })) { + const p = join(d, e.name) + if (e.isDirectory()) { + if (excludeId && e.name === excludeId) continue + const hit = walk(p) + if (hit) return hit + } else if (/^metadata\.json(\.gz)?$/.test(e.name)) { + writeFileSync(p, Buffer.from([0x1f, 0x8b, 0x00, 0xde, 0xad])) // torn gz + return p + } + } + return null + } + const torn = walk(nounsRoot) + if (!torn) throw new Error('layout probe: no noun metadata file found to tear') + // The id is the parent directory name. + return torn.split('/').slice(-2, -1)[0] +} + +describe('recovery-walk tolerance (the rc6-red cures)', () => { + it('a torn entity record does not kill the open: recovery walks heal past it, remaining rows serve, identity read throws typed', async () => { + const dir = mkdtempSync(join(tmpdir(), 'brainy-walk-tol-')) + dirs.push(dir) + let brain = await open(dir) + const keeper = await brain.add({ data: 'keeper row', type: NounType.Document, metadata: { k: 1 } }) + await brain.add({ data: 'victim row', type: NounType.Document, metadata: { k: 2 } }) + await brain.flush() + await brain.close() + brains.pop() + + const tornId = tearOneNounMetadata(dir, keeper) + + // THE PIN: the open succeeds (rc6 died right here), the keeper serves, + // and walks (find) heal past the victim. + brain = await open(dir) + expect((await brain.get(keeper))!.data).toContain('keeper row') + const rows = await brain.find({ where: {}, limit: 10 }) + expect(rows.map((r) => r.id)).toContain(keeper) + + // Identity read of the victim itself: typed, catchable — the caller + // asked for THAT record; under log authority the replay may have + // already HEALED it from the fact log (also a valid outcome) — accept + // healed-or-typed, never silent-absent-without-narration. + try { + const victim = await brain.get(tornId) + // Healed by replay: the record must be real (log authority rewrote it). + expect(victim).not.toBeNull() + } catch (err) { + expect(err).toBeInstanceOf(TornRecordError) + } + }, 120000) + + it('a torn mapper file (NaN ints) discards with narration; reopen succeeds and the first write mints sanely', async () => { + const dir = mkdtempSync(join(tmpdir(), 'brainy-torn-mapper-')) + dirs.push(dir) + let brain = await open(dir) + await brain.add({ data: 'pre-crash row', type: NounType.Document, metadata: { k: 1 } }) + await brain.flush() + await brain.close() + brains.pop() + + // The power-cut shape: the persisted mapper carries garbage integers. + const sys = join(dir, '_system') + const mapperPath = readdirSync(sys) + .filter((f) => /entityIdMapper/.test(f)) + .map((f) => join(sys, f))[0] + expect(mapperPath, 'layout probe: mapper artifact exists').toBeTruthy() + const torn = { nextId: 'NaN-garbage', uuidToInt: { x: 'junk' }, intToUuid: { junk: 42 } } + if (mapperPath.endsWith('.gz')) writeFileSync(mapperPath, gzipSync(JSON.stringify(torn))) + else writeFileSync(mapperPath, JSON.stringify(torn)) + expect(statSync(mapperPath).size).toBeGreaterThan(0) + + // Reopen MUST succeed; the first write after recovery must mint sanely + // (rc6's fresh-write RangeError shape), and graph int resolution at + // reopen must not throw (rc6's reopen shape). + brain = await open(dir) + const fresh = await brain.add({ data: 'post-recovery write', type: NounType.Document, metadata: { k: 2 } }) + expect((await brain.get(fresh))!.data).toContain('post-recovery') + await brain.flush() + expect(Number.isSafeInteger(brain.generation())).toBe(true) + }, 120000) +}) diff --git a/tests/unit/storage/torn-record-loud.test.ts b/tests/unit/storage/torn-record-loud.test.ts index 13f47c47569327afb9b065c0e688ebf274c091d8..d35f8b439e9038b36289c1557c5eb7a71c4d641b 100644 GIT binary patch delta 984 zcmbV~zityj5XNi%HHJ8`k&C21zz-S4h%qG+A442YA#v)_T<{c%ht&uT|eB{ulFxPT4yAJY* z*sSmj#xhKGmO;E~31>z8&2gg51Z=!Lt{~Gnb=n0qN`y6Uiwdn}93`=F2@A}o9-LO< zK}w#0-p4U>3vlCHbPEPG0M%!mj{kK z_rh6yFTA+l44>Z9I-xUE$O`qjZ}txh{Vw)=E|nP0ZU!_Cfa7g`bYR))27auDDrAMp1rwu|i2@L28OZW?pegYGR5)ewspYW=?8eNlv9ger{$-NoHQU zLPs6o&r$A_16l4~M0M!PiCg&HW zxE2-V7ipwwLM1036m^=cBW++*Tw0Wtn4Ai<9m!}cPRY(JC;+)2vjk{w&g9cF5|eL= R$xJqtl_SZ{&8Bj~yZ}F|QxE_E