David Snelling
0e3facf4a8
The quiet-loss cure regressed recovery: the new typed torn-record error was correct at identity-read time but threw inside init-time recovery walks, killing opens that previously survived. The boundary, redrawn: - IDENTITY READS (get-by-id of a specific record, CAS blob point-get): typed TornRecordError, unchanged — a caller who asked for THAT record can act on the answer. - SET-SHAPED READS AND WALKS (enumeration, pagination, batch hydration — the paths recovery rebuilds and finds page over): HEAL PAST the torn victim. The adapter's loud floor (error log + counted gauge) fires at the encounter; the walk serves the remaining rows. One crash casualty can no longer kill every query on its shard — or the open itself. - WRITES OVER TORN RECORDS ARE THE CURE: the save path's read-merge, the commit path's before-image capture, and the operations' rollback captures all treat a torn prior as the create sentinel, narrated — the incoming bytes replace the unreadable ones, and history for the id honestly restarts at that generation. Corruption can never block its own heal. - THE NaN SOURCE: torn mapper state (nextId/entries carrying garbage) discards with narration and re-derives via the existing rebuild path; the mint gains a source guard healing a non-integer counter from the live map. The reopen and first-write RangeError shapes are dead at the source, both authority branches. Pinned with the exact fault-injection scenarios: a torn entity record (including the VFS root) no longer kills the open — walks heal past it, the keeper rows serve, and the identity read of the victim itself is typed-or-healed; a torn mapper reopens and mints sanely on the first post-recovery write. Gates: tsc 0 · unit 2065/2065 · integration 828 · conformance 31/31.
260 lines
11 KiB
TypeScript
260 lines
11 KiB
TypeScript
/**
|
||
* @module tests/unit/storage/torn-record-loud
|
||
* @description Torn records must be LOUD, never silent. A file that EXISTS but
|
||
* cannot be decoded (truncated/garbled JSON, undecodable gzip) is disk
|
||
* corruption, not absence — the old behavior logged "gracefully skipping" and
|
||
* returned `null`, so a consumer could not distinguish "never existed" from
|
||
* "exists but torn" and nothing ever healed it. Pins the cured contract:
|
||
*
|
||
* - ENTITY read paths (get/getBatch/pagination) throw a typed
|
||
* `TornRecordError` ({path, cause}, code `TORN_RECORD`) — NEVER a silent null.
|
||
* - Genuine absence (ENOENT) still reads as clean `null` — no error, no gauge.
|
||
* - EVERY torn encounter increments the per-process torn-record gauge and
|
||
* records the path, whatever the caller surface decides.
|
||
* - SYSTEM-ARTIFACT reads (`readRawObject`: manifests/markers with recovery
|
||
* paths) map torn → `null` BY DESIGN — but only after the encounter was
|
||
* logged and counted (loud degrade, not a quiet loss).
|
||
* - Legacy dual-format recovery: a torn `.gz` with a decodable uncompressed
|
||
* fallback returns the recovered object AND still counts the torn `.gz`.
|
||
*/
|
||
import { describe, it, expect, beforeEach, afterEach } from 'vitest'
|
||
import * as fs from 'node:fs'
|
||
import * as os from 'node:os'
|
||
import * as path from 'node:path'
|
||
import { FileSystemStorage } from '../../../src/storage/adapters/fileSystemStorage.js'
|
||
import {
|
||
TornRecordError,
|
||
isTornRecordError,
|
||
getTornRecordGauge,
|
||
resetTornRecordGauge
|
||
} from '../../../src/storage/tornRecordError.js'
|
||
import type { NounMetadata } from '../../../src/coreTypes.js'
|
||
|
||
const VEC = [0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8]
|
||
|
||
/** Release a raw FileSystemStorage so a subsequent open of the dir is unblocked. */
|
||
async function teardown(s: any): Promise<void> {
|
||
try { await s.flush?.() } catch { /* best effort */ }
|
||
try { s.stopFlushRequestWatcher?.() } catch { /* best effort */ }
|
||
try { await s.releaseWriterLock?.() } catch { /* best effort */ }
|
||
}
|
||
|
||
/** Save one noun (metadata + vector) through the adapter's real write path. */
|
||
async function seedOne(s: any, id: string): Promise<void> {
|
||
await s.saveNounMetadata(id, {
|
||
noun: 'thing',
|
||
createdAt: Date.now(),
|
||
updatedAt: Date.now()
|
||
} as NounMetadata)
|
||
await s.saveNoun({ id, vector: VEC, connections: new Map(), level: 0 })
|
||
}
|
||
|
||
/** Find the on-disk file(s) for an entity leg (metadata.json / vectors.json), .gz or plain. */
|
||
function findEntityFiles(dir: string, id: string, leg: 'metadata' | 'vectors'): string[] {
|
||
const found: string[] = []
|
||
const walk = (d: string): void => {
|
||
for (const entry of fs.readdirSync(d, { withFileTypes: true })) {
|
||
const p = path.join(d, entry.name)
|
||
if (entry.isDirectory()) walk(p)
|
||
else if (
|
||
p.includes(`${path.sep}${id}${path.sep}`) &&
|
||
(entry.name === `${leg}.json` || entry.name === `${leg}.json.gz`)
|
||
) {
|
||
found.push(p)
|
||
}
|
||
}
|
||
}
|
||
walk(path.join(dir, 'entities'))
|
||
return found
|
||
}
|
||
|
||
/** Overwrite a file with bytes that can never decode as gzip or JSON. */
|
||
function corruptFile(filePath: string): void {
|
||
fs.writeFileSync(filePath, Buffer.from('{"noun":"thing","crea GARBAGE'))
|
||
}
|
||
|
||
describe('torn records are loud: typed on entity reads, counted everywhere (never a silent null)', () => {
|
||
let dirs: string[] = []
|
||
|
||
beforeEach(() => {
|
||
resetTornRecordGauge()
|
||
})
|
||
|
||
afterEach(() => {
|
||
for (const d of dirs.splice(0)) fs.rmSync(d, { recursive: true, force: true })
|
||
})
|
||
|
||
function tmpDir(): string {
|
||
const d = fs.mkdtempSync(path.join(os.tmpdir(), 'brainy-torn-'))
|
||
dirs.push(d)
|
||
return d
|
||
}
|
||
|
||
/** Seed one entity, release the writer, corrupt the chosen leg on disk, reopen cold. */
|
||
async function seedCorruptReopen(
|
||
leg: 'metadata' | 'vectors',
|
||
options?: { compression?: boolean }
|
||
): Promise<{ dir: string; storage: any; id: string; corrupted: string[] }> {
|
||
const dir = tmpDir()
|
||
const id = 'aaaaaaaa-1111-4222-8333-444444444444'
|
||
const writer = new FileSystemStorage(dir, options) as any
|
||
await writer.init()
|
||
await seedOne(writer, id)
|
||
await teardown(writer)
|
||
|
||
const files = findEntityFiles(dir, id, leg)
|
||
expect(files.length).toBeGreaterThan(0) // the record really landed on disk
|
||
for (const f of files) corruptFile(f)
|
||
|
||
const storage = new FileSystemStorage(dir, options) as any
|
||
await storage.init()
|
||
return { dir, storage, id, corrupted: files }
|
||
}
|
||
|
||
it('entity metadata read on a torn (uncompressed) record throws TornRecordError — never null', async () => {
|
||
const { storage, id } = await seedCorruptReopen('metadata', { compression: false })
|
||
try {
|
||
// Capture the outcome without letting a non-throw masquerade as a pass:
|
||
// if the read RESOLVES, the outlawed shape is back (torn read as value/absent).
|
||
const outcome = await storage.getNounMetadata(id).then(
|
||
(value: unknown) => ({ threw: false as const, value }),
|
||
(error: unknown) => ({ threw: true as const, error })
|
||
)
|
||
expect(outcome.threw).toBe(true)
|
||
const caught = (outcome as { error: unknown }).error
|
||
expect(isTornRecordError(caught)).toBe(true)
|
||
expect((caught as TornRecordError).name).toBe('TornRecordError')
|
||
expect((caught as TornRecordError).code).toBe('TORN_RECORD')
|
||
expect((caught as TornRecordError).path).toContain(id)
|
||
expect((caught as TornRecordError).cause).toBeTruthy()
|
||
|
||
const gauge = getTornRecordGauge()
|
||
expect(gauge.count).toBeGreaterThanOrEqual(1)
|
||
expect(gauge.lastPath).toContain(id)
|
||
} finally {
|
||
await teardown(storage)
|
||
}
|
||
})
|
||
|
||
it('entity vector read (getNoun) on a torn record throws typed — a row is never silently dropped', async () => {
|
||
const { storage, id } = await seedCorruptReopen('vectors', { compression: false })
|
||
try {
|
||
await expect(storage.getNoun(id)).rejects.toSatisfy((e: unknown) => isTornRecordError(e))
|
||
expect(getTornRecordGauge().count).toBeGreaterThanOrEqual(1)
|
||
} finally {
|
||
await teardown(storage)
|
||
}
|
||
})
|
||
|
||
it('batch hydration (getNounMetadataBatch) HEALS PAST the torn row: omits it with the loud floor fired — set-shaped reads degrade, never die', async () => {
|
||
// The contract redrawn by block-layer fault injection: one crash victim
|
||
// must not kill every query paging over its shard, and init-time
|
||
// recovery walks ride these exact paths. The adapter's loud floor
|
||
// (error log + gauge) fires at encounter; the batch serves the rest.
|
||
const { storage, id } = await seedCorruptReopen('metadata', { compression: false })
|
||
try {
|
||
const before = getTornRecordGauge().count
|
||
const result = await storage.getNounMetadataBatch([id])
|
||
expect(result instanceof Map ? result.get(id) : result[id]).toBeFalsy()
|
||
expect(getTornRecordGauge().count, 'the loud floor fired').toBeGreaterThan(before)
|
||
} finally {
|
||
await teardown(storage)
|
||
}
|
||
})
|
||
|
||
it('pagination/enumeration (getNounsWithPagination) HEALS PAST the torn row: walk survives, loud floor fired', async () => {
|
||
// Walks are healers: an enumeration meeting a torn record narrates +
|
||
// counts and continues — the open-time rebuild walks that ride this
|
||
// path must never die on a crash's legal victim.
|
||
const { storage, id } = await seedCorruptReopen('vectors', { compression: false })
|
||
void id
|
||
try {
|
||
const before = getTornRecordGauge().count
|
||
const page = await storage.getNounsWithPagination({ limit: 10 })
|
||
expect(Array.isArray(page.items), 'the walk survives').toBe(true)
|
||
expect(getTornRecordGauge().count, 'the loud floor fired').toBeGreaterThan(before)
|
||
} finally {
|
||
await teardown(storage)
|
||
}
|
||
})
|
||
|
||
it('ENOENT still reads as clean absent: null result, no error, gauge untouched', async () => {
|
||
const dir = tmpDir()
|
||
const storage = new FileSystemStorage(dir, { compression: false }) as any
|
||
await storage.init()
|
||
try {
|
||
const before = getTornRecordGauge().count
|
||
await expect(
|
||
storage.getNounMetadata('bbbbbbbb-1111-4222-8333-444444444444')
|
||
).resolves.toBeNull()
|
||
await expect(
|
||
storage.getNoun('bbbbbbbb-1111-4222-8333-444444444444')
|
||
).resolves.toBeNull()
|
||
await expect(storage.readRawObject('_system/never-written.json')).resolves.toBeNull()
|
||
expect(getTornRecordGauge().count).toBe(before) // absence is not corruption
|
||
} finally {
|
||
await teardown(storage)
|
||
}
|
||
})
|
||
|
||
it('system-artifact surface (readRawObject) maps torn → null BY DESIGN, but the encounter is counted', async () => {
|
||
const dir = tmpDir()
|
||
const storage = new FileSystemStorage(dir, { compression: false }) as any
|
||
await storage.init()
|
||
try {
|
||
await storage.writeRawObject('_system/some-manifest.json', { version: 1 })
|
||
corruptFile(path.join(dir, '_system/some-manifest.json'))
|
||
|
||
const before = getTornRecordGauge().count
|
||
// Artifact readers (manifests with recovery paths, markers whose verdict
|
||
// is "rescan") are designed for absent-artifact degradation: torn maps to
|
||
// their existing degrade — AFTER the loud floor (error log + gauge).
|
||
await expect(storage.readRawObject('_system/some-manifest.json')).resolves.toBeNull()
|
||
const gauge = getTornRecordGauge()
|
||
expect(gauge.count).toBe(before + 1)
|
||
expect(gauge.lastPath).toContain('some-manifest.json')
|
||
} finally {
|
||
await teardown(storage)
|
||
}
|
||
})
|
||
|
||
it('compressed installs: a torn .gz with no fallback throws typed and names the .gz path', async () => {
|
||
const { storage, id } = await seedCorruptReopen('metadata', { compression: true })
|
||
try {
|
||
let caught: unknown = null
|
||
try {
|
||
await storage.getNounMetadata(id)
|
||
} catch (e) {
|
||
caught = e
|
||
}
|
||
expect(isTornRecordError(caught)).toBe(true)
|
||
expect((caught as TornRecordError).path).toMatch(/metadata\.json\.gz$/)
|
||
expect(getTornRecordGauge().count).toBeGreaterThanOrEqual(1)
|
||
} finally {
|
||
await teardown(storage)
|
||
}
|
||
})
|
||
|
||
it('legacy dual-format: torn .gz with a decodable uncompressed fallback recovers the object AND counts the torn file', async () => {
|
||
const { storage, id, corrupted } = await seedCorruptReopen('metadata', { compression: true })
|
||
try {
|
||
// Recreate the legacy state: the corrupt .gz sits next to a valid plain file.
|
||
const gzPath = corrupted.find((f) => f.endsWith('.gz'))!
|
||
const plainPath = gzPath.replace(/\.gz$/, '')
|
||
fs.writeFileSync(
|
||
plainPath,
|
||
JSON.stringify({ noun: 'thing', createdAt: 1, updatedAt: 1 })
|
||
)
|
||
|
||
const before = getTornRecordGauge().count
|
||
const recovered = await storage.getNounMetadata(id)
|
||
expect(recovered).toBeTruthy()
|
||
expect(recovered.noun).toBe('thing') // loud recovery, not a silent skip
|
||
const gauge = getTornRecordGauge()
|
||
expect(gauge.count).toBe(before + 1) // the torn .gz was still surfaced
|
||
expect(gauge.lastPath).toMatch(/metadata\.json\.gz$/)
|
||
} finally {
|
||
await teardown(storage)
|
||
}
|
||
})
|
||
})
|