From a5a1883819f1d1661dadf369c15c045d3df7e7b9 Mon Sep 17 00:00:00 2001 From: David Snelling Date: Mon, 17 Aug 2026 12:53:04 -0700 Subject: [PATCH] =?UTF-8?q?fix(adoption):=20the=20baseline=20backfill=20ru?= =?UTF-8?q?ns=20to=20completion=20=E2=80=94=20one=20call=20adopts=20a=20pr?= =?UTF-8?q?e-log=20baseline=20of=20any=20size?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A production brain with a 12.7k-row pre-log baseline advanced exactly 800 rows per adoptLogAuthority() call (a five-pass ceiling × the oracle's 200-row listing cap), refused the flip, and sat tree-authoritative for hours across restarts. The bound was sized for drift, never for a baseline. Now: the adoption path runs the oracle uncapped so ONE scan yields the ENTIRE curable set, every pass cures all of it, and the loop runs to completion with the no-progress guard as its only stop. Pace rides the write path (one full-brain scan amortizes over thousands of cures, not two hundred): 1,000 drifted rows adopt green in one call in ~10s. Progress is narrated for a live operator. The wire report keeps its 200-row cap. Pinned: a baseline above the old ceiling adopts green in a single call. --- src/brainy.ts | 51 ++++++++-- src/db/logAuthority.ts | 11 ++- .../integration/adopt-large-baseline.test.ts | 92 +++++++++++++++++++ 3 files changed, 145 insertions(+), 9 deletions(-) create mode 100644 tests/integration/adopt-large-baseline.test.ts diff --git a/src/brainy.ts b/src/brainy.ts index d7313855..addb8bc2 100644 --- a/src/brainy.ts +++ b/src/brainy.ts @@ -8208,10 +8208,20 @@ export class Brainy implements BrainyInterface { * (digests, never bodies). */ async verifyLogAuthority(): Promise { + return this.runOracle() + } + + /** + * The oracle run behind {@link Brainy.verifyLogAuthority}; the adoption + * backfill calls it with `listAll` so one scan yields the ENTIRE curable + * mismatch set instead of the wire-capped first 200. + */ + private async runOracle(options?: { listAll?: boolean }): Promise { await this.ensureInitialized() return runLogCompletenessOracle({ storage: this.storage as unknown as LogAuthorityStorage, scanFacts: () => this.scanFacts(), + ...(options?.listAll ? { mismatchListCap: Number.POSITIVE_INFINITY } : {}), // Both sides normalize to ENTITY TRUTH before digesting: canonical // wrappers denormalize HNSW residue (connections/level) the log never // carries — digesting it would fake state-differs on any nonzero-level @@ -8256,7 +8266,7 @@ export class Brainy implements BrainyInterface { /** The adoption body — see {@link Brainy.adoptLogAuthority} (which owns the * fold-checkpoint bootstrap arm/disarm around it). */ private async adoptLogAuthorityInner(): Promise { - let report = await this.verifyLogAuthority() + let report = await this.runOracle({ listAll: true }) // BASELINE BACKFILL: curable divergences are rows whose CANONICAL truth // simply never reached the log — pre-log records (e.g. the generation-0 @@ -8268,8 +8278,18 @@ export class Brainy implements BrainyInterface { // Log-AHEAD divergences (log-live-canonical-absent / // log-tombstone-canonical-present) are NOT curable by backfill — the // log claims things the witness denies — and refuse loudly below. + // + // RUNS TO COMPLETION. Each pass sees the ENTIRE curable set (the oracle + // is run uncapped here) and cures all of it, so a pre-log baseline of + // any size adopts in ONE call — the only stop is the no-progress guard. + // A production brain with a 12.7k-row baseline once advanced exactly + // 800 rows per call (a five-pass ceiling × the 200-row wire cap) and sat + // tree-authoritative for hours; the bound was sized for drift, never + // for a baseline. Pace rides the write path now: one full-brain scan + // per pass amortizes over thousands of cures, not two hundred. let passes = 0 - while (report.verdict === 'red' && passes < 5) { + for (;;) { + if (report.verdict !== 'red') break passes++ const curable = report.mismatches.filter( (m) => m.reason === 'pre-log-record' || m.reason === 'state-differs' @@ -8290,9 +8310,19 @@ export class Brainy implements BrainyInterface { `[Brainy] adoptLogAuthority: baseline backfill pass ${passes} — re-committing ` + `${curable.length} row(s) whose canonical truth never reached the log` ) + // Progress narration for a live operator: a large baseline is minutes + // of visible motion, never a silent wait. + const narrateEvery = curable.length >= 2000 ? 1000 : curable.length >= 400 ? 200 : 0 + let cured = 0 for (const m of curable) { const raw = await this.storage.readNounRaw(m.id) if (raw.metadata === null && raw.vector === null) continue // vanished since the scan + cured++ + if (narrateEvery > 0 && cured % narrateEvery === 0) { + prodLog.info( + `[Brainy] adoptLogAuthority: backfill pass ${passes} — ${cured}/${curable.length} rows re-committed` + ) + } // LAW-SHAPE RE-COMMIT: rewrite canonical as EXACTLY the wrapper the // log's reconstruction produces (the hydration law: denormalized // enumeration fields derived from the metadata leg + the embedding @@ -8330,12 +8360,11 @@ export class Brainy implements BrainyInterface { }) }) } - const next = await this.verifyLogAuthority() - if ( - next.verdict === 'red' && - next.mismatches.length >= report.mismatches.length && - !report.mismatchListTruncated - ) { + const next = await this.runOracle({ listAll: true }) + // THE ONLY STOP: no progress. With uncapped listings both counts are + // exact, so "not fewer mismatches than before" means the cure could + // not express this divergence — refuse to spin, name it. + if (next.verdict === 'red' && next.mismatches.length >= report.mismatches.length) { throw new Error( `adoptLogAuthority(): baseline backfill made no progress ` + `(${report.mismatches.length} → ${next.mismatches.length} mismatches; first: ` + @@ -8345,6 +8374,12 @@ export class Brainy implements BrainyInterface { } report = next } + if (passes > 0) { + prodLog.info( + `[Brainy] adoptLogAuthority: baseline backfill complete in ${passes} pass(es) — ` + + `oracle ${report.verdict}, ${report.nounsChecked} noun(s) checked` + ) + } this._logAuthority = await flipToLogAuthority( this.storage as unknown as LogAuthorityStorage, diff --git a/src/db/logAuthority.ts b/src/db/logAuthority.ts index 0703d11f..b63ae715 100644 --- a/src/db/logAuthority.ts +++ b/src/db/logAuthority.ts @@ -165,7 +165,16 @@ export async function runLogCompletenessOracle(args: { getVerbs?: (opts: { pagination: { limit: number; offset?: number; cursor?: string } }) => Promise<{ items: unknown[]; hasMore?: boolean; nextCursor?: string }> + /** + * Cap on the LISTED mismatches (counts are always complete). Defaults to + * the wire-friendly {@link MISMATCH_LIST_CAP}; the adoption backfill passes + * `Infinity` so ONE scan yields the ENTIRE curable set — a production + * brain with a 12.7k-row pre-log baseline once advanced only 800 rows per + * adoption call because each pass could see (and cure) at most 200. + */ + mismatchListCap?: number }): Promise { + const listCap = args.mismatchListCap ?? MISMATCH_LIST_CAP const report: OracleReport = { verdict: 'red', generationsScanned: 0, @@ -176,7 +185,7 @@ export async function runLogCompletenessOracle(args: { mismatchListTruncated: false } const addMismatch = (m: OracleMismatch): void => { - if (report.mismatches.length < MISMATCH_LIST_CAP) report.mismatches.push(m) + if (report.mismatches.length < listCap) report.mismatches.push(m) else report.mismatchListTruncated = true } diff --git a/tests/integration/adopt-large-baseline.test.ts b/tests/integration/adopt-large-baseline.test.ts new file mode 100644 index 00000000..11a3c803 --- /dev/null +++ b/tests/integration/adopt-large-baseline.test.ts @@ -0,0 +1,92 @@ +/** + * @module tests/integration/adopt-large-baseline + * @description Adoption runs the baseline backfill TO COMPLETION in one call. + * A production brain with a 12.7k-row pre-log baseline once advanced exactly + * 800 rows per `adoptLogAuthority()` call (a five-pass ceiling × the oracle's + * 200-row listing cap), refused the flip, and sat tree-authoritative for + * hours across restarts. The pin: a baseline larger than that old ceiling + * — every row oracle-visible as `state-differs` drift — adopts GREEN in a + * SINGLE call, and the row count proves the whole set was cured, not a page. + */ +import { describe, it, expect, afterEach } from 'vitest' +import { mkdtempSync, rmSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { Brainy } from '../../src/index.js' +import { NounType } from '../../src/types/graphTypes.js' + +type RawBox = { + storage: { + readNounRaw(id: string): Promise<{ metadata: unknown; vector: unknown }> + writeNounRaw(id: string, r: { metadata: unknown; vector: unknown }): Promise + } +} + +const dirs: string[] = [] +const brains: Brainy[] = [] +afterEach(async () => { + for (const b of brains.splice(0)) await b.close().catch(() => {}) + for (const d of dirs.splice(0)) rmSync(d, { recursive: true, force: true }) +}) + +describe('adoption backfill runs to completion', () => { + it('a pre-log baseline larger than the old 800-row ceiling adopts GREEN in ONE call', async () => { + const dir = mkdtempSync(join(tmpdir(), 'brainy-large-baseline-')) + dirs.push(dir) + const brain = new Brainy({ + storage: { type: 'filesystem', path: dir }, + requireSubtype: false, + logAuthority: 'defer' + }) + await brain.init() + brains.push(brain) + + // Above the old ceiling (5 passes × 200 = 800): every row must be cured + // in the one call for the flip to be legal. + const ROWS = 1000 + const ids: string[] = [] + for (let i = 0; i < ROWS; i++) { + ids.push( + await brain.add({ + data: `baseline row ${i}`, + type: NounType.Document, + metadata: { i }, + vector: Array.from({ length: 384 }, (_, k) => ((i + k) % 7) / 7) + }) + ) + } + await brain.flush() + + // Manufacture the production shape on EVERY row: pre-hydration-law drift + // (a stored wrapper whose denormalized fields disagree with its own + // metadata leg) — each is a curable `state-differs` mismatch, so the + // oracle's full curable set is ROWS, well past any per-pass page. + const storage = (brain as unknown as RawBox).storage + for (const id of ids) { + const raw = await storage.readNounRaw(id) + const wrapper = raw.vector as Record + await storage.writeNounRaw(id, { + metadata: raw.metadata, + vector: { ...wrapper, noun: 'thing', legacyField: 'pre-law residue' } + }) + } + const before = await brain.verifyLogAuthority() + expect(before.verdict, 'the whole baseline is oracle-red').toBe('red') + // The wire report is capped at 200 — the truncation flag is what the old + // loop bounded itself on; the cure path no longer reads through it. + expect(before.mismatchListTruncated).toBe(true) + + // THE PIN: one call, green, log-authoritative — no restarts, no loop. + const report = await brain.adoptLogAuthority() + expect(report.verdict).toBe('green') + expect(brain.logAuthority().authority).toBe('log') + expect(report.nounsChecked).toBeGreaterThanOrEqual(ROWS) + + // Nothing degraded: a sample of rows still serves with intact metadata. + for (const id of [ids[0], ids[499], ids[ROWS - 1]]) { + const row = await brain.get(id) + expect(row).not.toBeNull() + expect(typeof (row!.metadata as { i: number }).i).toBe('number') + } + }, 600000) +})