fix(recovery): walks are healers — the typed/tolerant boundary redrawn where block-layer fault injection proved it belonged
The quiet-loss cure regressed recovery: the new typed torn-record error was correct at identity-read time but threw inside init-time recovery walks, killing opens that previously survived. The boundary, redrawn: - IDENTITY READS (get-by-id of a specific record, CAS blob point-get): typed TornRecordError, unchanged — a caller who asked for THAT record can act on the answer. - SET-SHAPED READS AND WALKS (enumeration, pagination, batch hydration — the paths recovery rebuilds and finds page over): HEAL PAST the torn victim. The adapter's loud floor (error log + counted gauge) fires at the encounter; the walk serves the remaining rows. One crash casualty can no longer kill every query on its shard — or the open itself. - WRITES OVER TORN RECORDS ARE THE CURE: the save path's read-merge, the commit path's before-image capture, and the operations' rollback captures all treat a torn prior as the create sentinel, narrated — the incoming bytes replace the unreadable ones, and history for the id honestly restarts at that generation. Corruption can never block its own heal. - THE NaN SOURCE: torn mapper state (nextId/entries carrying garbage) discards with narration and re-derives via the existing rebuild path; the mint gains a source guard healing a non-integer counter from the live map. The reopen and first-write RangeError shapes are dead at the source, both authority branches. Pinned with the exact fault-injection scenarios: a torn entity record (including the VFS root) no longer kills the open — walks heal past it, the keeper rows serve, and the identity read of the victim itself is typed-or-healed; a torn mapper reopens and mints sanely on the first post-recovery write. Gates: tsc 0 · unit 2065/2065 · integration 828 · conformance 31/31.
This commit is contained in:
parent
214c98b4d5
commit
0e3facf4a8
6 changed files with 350 additions and 39 deletions
|
|
@ -12,6 +12,7 @@
|
|||
|
||||
import type { StorageAdapter, HNSWNoun, HNSWVerb, NounMetadata, VerbMetadata } from '../../coreTypes.js'
|
||||
import type { Operation, RollbackAction } from '../types.js'
|
||||
import { prodLog } from '../../utils/logger.js'
|
||||
|
||||
/**
|
||||
* Save noun metadata with rollback support
|
||||
|
|
@ -20,6 +21,30 @@ import type { Operation, RollbackAction } from '../types.js'
|
|||
* - If metadata existed: Restore previous metadata
|
||||
* - If metadata was new: Delete metadata
|
||||
*/
|
||||
|
||||
/**
|
||||
* Torn-tolerant previous-state read for ROLLBACK CAPTURE: a write or delete
|
||||
* landing on a TORN record (power-loss survivor) HEALS it — the incoming
|
||||
* bytes replace (or remove) the unreadable ones, and the rollback target is
|
||||
* the create sentinel (null). The adapter's loud floor (error + gauge)
|
||||
* already fired at throw time; this narrates the heal and proceeds. Real
|
||||
* storage faults still propagate.
|
||||
*/
|
||||
async function tornHealsToNull<T>(read: Promise<T>, what: string): Promise<T | null> {
|
||||
try {
|
||||
return await read
|
||||
} catch (err) {
|
||||
if ((err as { code?: string }).code === 'TORN_RECORD') {
|
||||
prodLog.warn(
|
||||
`[StorageOperations] previous ${what} is TORN — the incoming operation ` +
|
||||
`heals it; rollback target is the create sentinel`
|
||||
)
|
||||
return null
|
||||
}
|
||||
throw err
|
||||
}
|
||||
}
|
||||
|
||||
export class SaveNounMetadataOperation implements Operation {
|
||||
readonly name = 'SaveNounMetadata'
|
||||
|
||||
|
|
@ -34,7 +59,7 @@ export class SaveNounMetadataOperation implements Operation {
|
|||
// Skip read for new entities — nothing to rollback to (saves 1 storage round-trip)
|
||||
const previousMetadata = this.isNew
|
||||
? null
|
||||
: await this.storage.getNounMetadata(this.id)
|
||||
: await tornHealsToNull(this.storage.getNounMetadata(this.id), 'noun metadata')
|
||||
|
||||
// Save new metadata
|
||||
await this.storage.saveNounMetadata(this.id, this.metadata)
|
||||
|
|
@ -75,7 +100,7 @@ export class SaveNounOperation implements Operation {
|
|||
// Skip read for new entities — nothing to rollback to (saves 1 storage round-trip)
|
||||
const previousNoun = this.isNew
|
||||
? null
|
||||
: await this.storage.getNoun(this.noun.id)
|
||||
: await tornHealsToNull(this.storage.getNoun(this.noun.id), 'noun record')
|
||||
|
||||
// PRESERVE stored graph state on updates. Callers stage this op with
|
||||
// placeholder adjacency ({connections: empty, level: 0}) because the
|
||||
|
|
@ -162,8 +187,11 @@ export class DeleteNounMetadataOperation implements Operation {
|
|||
// Capture the FULL before-image (both legs) so the undo restores the whole
|
||||
// entity — a metadata-only rollback would leave the vector leg unrestored.
|
||||
// A null metadata read falls back to the caller's pre-delete read.
|
||||
const previousNoun = await this.storage.getNoun(this.id)
|
||||
const previousMetadata = (await this.storage.getNounMetadata(this.id)) ?? this.priorMetadata ?? null
|
||||
const previousNoun = await tornHealsToNull(this.storage.getNoun(this.id), 'noun record')
|
||||
const previousMetadata =
|
||||
(await tornHealsToNull(this.storage.getNounMetadata(this.id), 'noun metadata')) ??
|
||||
this.priorMetadata ??
|
||||
null
|
||||
|
||||
if (!previousNoun && !previousMetadata) {
|
||||
// Nothing to delete - no rollback needed
|
||||
|
|
@ -211,7 +239,7 @@ export class SaveVerbMetadataOperation implements Operation {
|
|||
|
||||
async execute(): Promise<RollbackAction> {
|
||||
// Get existing metadata (for rollback)
|
||||
const previousMetadata = await this.storage.getVerbMetadata(this.id)
|
||||
const previousMetadata = await tornHealsToNull(this.storage.getVerbMetadata(this.id), 'verb metadata')
|
||||
|
||||
// Save new metadata
|
||||
await this.storage.saveVerbMetadata(this.id, this.metadata)
|
||||
|
|
@ -247,7 +275,7 @@ export class SaveVerbOperation implements Operation {
|
|||
|
||||
async execute(): Promise<RollbackAction> {
|
||||
// Get existing verb (for rollback)
|
||||
const previousVerb = await this.storage.getVerb(this.verb.id)
|
||||
const previousVerb = await tornHealsToNull(this.storage.getVerb(this.verb.id), 'verb record')
|
||||
|
||||
// Save new verb
|
||||
await this.storage.saveVerb(this.verb)
|
||||
|
|
@ -291,7 +319,7 @@ export class DeleteVerbMetadataOperation implements Operation {
|
|||
|
||||
async execute(): Promise<RollbackAction> {
|
||||
// Get metadata before deletion (for rollback)
|
||||
const previousMetadata = await this.storage.getVerbMetadata(this.id)
|
||||
const previousMetadata = await tornHealsToNull(this.storage.getVerbMetadata(this.id), 'verb metadata')
|
||||
|
||||
if (!previousMetadata) {
|
||||
// Nothing to delete - no rollback needed
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue