diff --git a/src/db/factLogFormat.ts b/src/db/factLogFormat.ts index 0ac93e1f..d5492da8 100644 --- a/src/db/factLogFormat.ts +++ b/src/db/factLogFormat.ts @@ -1204,6 +1204,30 @@ function buildPadFrame(totalBytes: number): Uint8Array { fillerLength += diff if (fillerLength < 0) break } + if (!converged) { + // Class-boundary holes: a single bin filler steps its header by one + // byte at each msgpack size class (bin8→bin16→bin32), leaving exactly + // one unreachable payload size per boundary (the 291-byte production + // case). Bridge with a trailing fixint (+1 byte) beside the bin — + // {bin(n)} ∪ {bin(n) + fixint} covers every size ≥ minimum. + let bridged = Math.max(0, targetPayload - payload.length - 2) + for (let i = 0; i < 8; i++) { + const candidate = attempt([ + LOG_RECORD_TYPES.PAD, + LOG_RECORD_VERSION, + new Uint8Array(bridged), + 0 + ]) + const diff = targetPayload - candidate.length + if (diff === 0) { + payload = candidate + converged = true + break + } + bridged += diff + if (bridged < 0) break + } + } if (!converged) { throw new Error(`fact log v2: a pad frame of ${totalBytes} bytes is not constructible`) } diff --git a/src/db/generationStore.ts b/src/db/generationStore.ts index 6922da6d..8f3bf625 100644 --- a/src/db/generationStore.ts +++ b/src/db/generationStore.ts @@ -1555,6 +1555,21 @@ export class GenerationStore { // the log's group-commit (many concurrent writers share ONE sync) — // an acked write's fact survives power loss, by contract. if (this.factLog) { + // TWO PHASES, TWO DISTINCT COMPENSATIONS (a production adoption + // proved the difference the hard way): rewinding the counter after + // a SUCCESSFUL append re-mints the same generation and every later + // append refuses non-monotonic — the write path wedges in a refusal + // loop. The counter may only rewind when the log provably does NOT + // carry the generation. + const unbuffer = (): void => { + this.pendingBuffer.delete(gen) + const idx = this.pendingGens.lastIndexOf(gen) + if (idx !== -1) this.pendingGens.splice(idx, 1) + this.invalidateChains() + } + // Phase 1 — APPEND. Failure = the log never took the fact: full + // compensation (un-buffer + counter rewind); a rejected write must + // not commit, and the next mint may safely reuse the number. try { await this.factLog.append( await this.buildCommitFact({ @@ -1565,24 +1580,37 @@ export class GenerationStore { ...(args.records && args.records.length > 0 ? { records: args.records } : {}) }) ) - if (this.logDurability === 'at-ack') { - await this.factLog.ensureSynced() - } } catch (err) { - // A rejected write must NOT commit: the generation was buffered - // before the append, so un-buffer it and return the counter - // reservation — otherwise the next flush would durably commit a - // generation with NO fact, a silent log gap a later replay would - // turn into loss. Canonical bytes from execute() remain as an - // uncommitted orphan — identical to a crash at this point; never - // a torn committed state. - this.pendingBuffer.delete(gen) - const idx = this.pendingGens.lastIndexOf(gen) - if (idx !== -1) this.pendingGens.splice(idx, 1) - this.invalidateChains() + unbuffer() if (this.counter === gen) this.counter = gen - 1 throw err } + // Phase 2 — the at-ack covering sync. Failure here means the fact + // IS in the log (append succeeded) but durability was not promised: + // try to remove it (dropAbove); only a SUCCESSFUL drop earns the + // counter rewind. If the drop itself fails (e.g. the fact was + // sealed by a racing rotation), the generation stays consumed and + // buffered — monotonicity holds, the flush path retries durability, + // and the caller still gets the loud failure. + if (this.logDurability === 'at-ack') { + try { + await this.factLog.ensureSynced() + } catch (err) { + try { + await this.factLog.dropAbove(gen - 1) + unbuffer() + if (this.counter === gen) this.counter = gen - 1 + } catch (dropErr) { + prodLog.warn( + `[GenerationStore] at-ack sync failed for generation ${gen} and the ` + + `appended fact could not be dropped (${(dropErr as Error).message}) — ` + + `the generation stays consumed and buffered; the flush path retries ` + + `durability. Never re-minting a number the log may carry.` + ) + } + throw err + } + } } // Test-only crash simulation. A crash here must cost the buffered // history + the appended fact in 'deferred' mode (open() truncates it diff --git a/tests/integration/sync-fail-compensation.test.ts b/tests/integration/sync-fail-compensation.test.ts new file mode 100644 index 00000000..f5032e34 --- /dev/null +++ b/tests/integration/sync-fail-compensation.test.ts @@ -0,0 +1,78 @@ +/** + * @module tests/integration/sync-fail-compensation + * @description The non-monotonic refusal-loop cure (a production adoption's + * second defect): when the at-ack covering SYNC fails AFTER a successful + * append, the counter must NOT rewind unless the appended fact is provably + * removed — rewinding while the log carries the generation re-mints the + * same number and every later append refuses non-monotonic, wedging the + * write path in a refusal loop ("writes REFUSED until it drains"). + */ +import { describe, it, expect, afterEach, vi } from 'vitest' +import { mkdtempSync, rmSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { Brainy } from '../../src/index.js' +import { NounType } from '../../src/types/graphTypes.js' + +const dirs: string[] = [] +const brains: Brainy[] = [] +afterEach(async () => { + vi.restoreAllMocks() + for (const b of brains.splice(0)) await b.close().catch(() => {}) + for (const d of dirs.splice(0)) rmSync(d, { recursive: true, force: true }) +}) + +describe('at-ack sync-failure compensation', () => { + it('a one-shot sync failure never wedges the write path: the next write mints a FRESH generation and succeeds', async () => { + const dir = mkdtempSync(join(tmpdir(), 'brainy-syncfail-')) + dirs.push(dir) + const brain = new Brainy({ storage: { type: 'filesystem', path: dir }, requireSubtype: false }) + await brain.init() // adopt-default: log authority, at-ack + brains.push(brain) + expect(brain.logAuthority().authority).toBe('log') + await brain.add({ data: 'baseline', type: NounType.Document, metadata: { n: 0 } }) + + // Fail exactly ONE covering sync (after its append lands). + // Target ensureSynced (the ACK path's covering sync) — mocking sync() + // itself gets eaten by background flushes before the victim write. + const factLog = (brain as unknown as { + generationStore: { getFactLog(): { ensureSynced(): Promise } } + }).generationStore.getFactLog() + const realEnsure = factLog.ensureSynced.bind(factLog) + let failed = false + vi.spyOn(factLog, 'ensureSynced').mockImplementation(async () => { + if (!failed) { + failed = true + throw new Error('injected sync failure (device hiccup)') + } + return realEnsure() + }) + + // The write whose sync fails: LOUD failure to the caller — never silent. + await expect( + brain.add({ data: 'sync victim', type: NounType.Document, metadata: { n: 1 } }) + ).rejects.toThrow(/sync failure/) + + // THE PIN: the very next write mints a fresh generation and SUCCEEDS — + // no non-monotonic refusal, no refusal loop, regardless of whether the + // failed write's fact was dropped or retained (both are legal outcomes; + // an equal-generation re-mint is not). + const survivor = await brain.add({ data: 'after the storm', type: NounType.Document, metadata: { n: 2 } }) + expect((await brain.get(survivor))!.data).toContain('after the storm') + await brain.flush() + expect(Number.isSafeInteger(brain.generation())).toBe(true) + + // And the log scans clean end-to-end (no torn ordering). + const scan = brain.scanFacts() + let last = 0 + if (scan) { + for await (const batch of (scan as { batches(): AsyncIterable<{ facts: Array<{ generation: number }> }> }).batches()) { + for (const f of batch.facts) { + expect(f.generation, 'strictly ascending').toBeGreaterThan(last) + last = f.generation + } + } + } + expect(last).toBeGreaterThan(0) + }, 120000) +}) diff --git a/tests/unit/db/pad-frame-total.test.ts b/tests/unit/db/pad-frame-total.test.ts new file mode 100644 index 00000000..a5c73d19 --- /dev/null +++ b/tests/unit/db/pad-frame-total.test.ts @@ -0,0 +1,29 @@ +/** + * @module tests/unit/db/pad-frame-total + * @description Pad-frame construction is TOTAL: every size from the minimum + * through 4096+257 is constructible byte-exact (a production adoption found + * the msgpack class-boundary hole at 291 bytes — sync died whole, and the + * failure cascaded into a counter rewind after a successful append). Every + * constructed pad decodes as skip-by-definition filler. + */ +import { describe, it, expect } from 'vitest' +import { encodePadFrame, minPadFrameBytes, decodeGroupV2 } from '../../../src/db/factLogFormat.js' + +describe('pad frames are constructible at EVERY size', () => { + it('exact construction from the minimum through a full sector + boundary spill', () => { + const min = minPadFrameBytes() + for (let size = min; size <= 4096 + 257; size++) { + const frame = encodePadFrame(size) + expect(frame.length, `size ${size}`).toBe(size) + } + }) + + it('the production case (291) and its class-boundary siblings decode as invisible filler', () => { + for (const size of [291, minPadFrameBytes(), 300, 511, 512, 513, 4096]) { + const frame = encodePadFrame(size) + const group = decodeGroupV2(frame) + expect(group.facts, `size ${size} is reader-invisible`).toEqual([]) + expect(group.validBytes).toBe(size) + } + }) +})