diff --git a/.forgejo/workflows/ci.yml b/.forgejo/workflows/ci.yml index 5e93cd96..fec679a8 100644 --- a/.forgejo/workflows/ci.yml +++ b/.forgejo/workflows/ci.yml @@ -27,22 +27,6 @@ jobs: - run: npm ci - run: npm run test:unit - # The correctness plant's full gate: integration + conformance run here on - # dedicated iron, on every push, so a release never depends on any other - # machine being up. Verdicts live in this run's log (never inferred). - integration: - name: Integration + conformance (Node 22) - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-node@v4 - with: - node-version: '22' - cache: npm - - run: npm ci - - run: npm run test:ci-integration - - run: npx vitest run tests/conformance - bun: name: Bun (latest) runs-on: ubuntu-latest diff --git a/CHANGELOG.md b/CHANGELOG.md index 8e91a6ad..47a767ec 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,13 +2,6 @@ All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines. -### [10.2.0](https://source.soulcraft.com/soulcraft/brainy/compare/v10.1.0...v10.2.0) (2026-08-17) - -- docs(releases): the 10.2.0 consumer entry — adoption completes in one call (97538e1f) -- ci: the correctness plant runs integration + conformance on every push — a release never waits on a second machine (b17fdc8e) -- fix(adoption): the baseline backfill runs to completion — one call adopts a pre-log baseline of any size (a5a18838) - - ### [10.1.0](https://source.soulcraft.com/soulcraft/brainy/compare/v10.0.0...v10.1.0) (2026-08-13) - docs(releases): the 10.1.0 consumer entry — bounded recovery, restore founding, the two write-path cures (7d3c8696) diff --git a/RELEASES.md b/RELEASES.md index 602b84e4..8116db0e 100644 --- a/RELEASES.md +++ b/RELEASES.md @@ -31,30 +31,6 @@ is sometimes cited as a 7.x removal — those methods never existed on 7.x; the --- -## v10.2.0 — 2026-08-17 (adoption completes in one call) - -One fix, headline-sized for large stores. Pairs with the same native accelerator -version as 10.1.0 — no accelerator bump needed. - -- **The adoption backfill runs to completion.** Adopting the crash-safe storage - authority first re-commits every row the log never saw (a one-time baseline - backfill). That backfill had a fixed ceiling of 800 rows per - `adoptLogAuthority()` call — sized for small drift, not for a large pre-existing - store — so a store with a 12,700-row baseline advanced 800 rows per call and - stayed on the prior authority across restarts (a production deployment's - report). Now one call adopts a baseline of any size: the backfill sees the - entire curable set at once, cures all of it, and loops only until green — the - no-progress guard is the sole stop. Pace rides the write path (~100 rows/s - measured end to end, versus ~1.7 rows/s under the old page-per-scan shape), - and progress is narrated so an operator watching a live service sees motion. - Stores that already adopted are unaffected; stores still on the prior authority - flip in a single call on their next open or on an explicit - `adoptLogAuthority()`. -- Verification report unchanged on the wire (still lists at most 200 mismatches; - counts remain complete) — only the adoption path reads the full set. - ---- - ## v10.1.0 — 2026-08-13 (the bounded-recovery and write-path-cure release) The theme: **crash recovery is bounded, restores are durably founded, and two diff --git a/package-lock.json b/package-lock.json index e8c238f5..8219ed2c 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "@soulcraft/brainy", - "version": "10.2.0", + "version": "10.1.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@soulcraft/brainy", - "version": "10.2.0", + "version": "10.1.0", "license": "MIT", "dependencies": { "@msgpack/msgpack": "^3.1.2", diff --git a/package.json b/package.json index a366f42f..6dc73761 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@soulcraft/brainy", - "version": "10.2.0", + "version": "10.1.0", "description": "Universal Knowledge Protocol™ - World's first Triple Intelligence database unifying vector, graph, and document search in one API. Stage 3 CANONICAL: 42 nouns × 127 verbs covering 96-97% of all human knowledge.", "main": "dist/index.js", "module": "dist/index.js", diff --git a/src/brainy.ts b/src/brainy.ts index addb8bc2..d7313855 100644 --- a/src/brainy.ts +++ b/src/brainy.ts @@ -8208,20 +8208,10 @@ export class Brainy implements BrainyInterface { * (digests, never bodies). */ async verifyLogAuthority(): Promise { - return this.runOracle() - } - - /** - * The oracle run behind {@link Brainy.verifyLogAuthority}; the adoption - * backfill calls it with `listAll` so one scan yields the ENTIRE curable - * mismatch set instead of the wire-capped first 200. - */ - private async runOracle(options?: { listAll?: boolean }): Promise { await this.ensureInitialized() return runLogCompletenessOracle({ storage: this.storage as unknown as LogAuthorityStorage, scanFacts: () => this.scanFacts(), - ...(options?.listAll ? { mismatchListCap: Number.POSITIVE_INFINITY } : {}), // Both sides normalize to ENTITY TRUTH before digesting: canonical // wrappers denormalize HNSW residue (connections/level) the log never // carries — digesting it would fake state-differs on any nonzero-level @@ -8266,7 +8256,7 @@ export class Brainy implements BrainyInterface { /** The adoption body — see {@link Brainy.adoptLogAuthority} (which owns the * fold-checkpoint bootstrap arm/disarm around it). */ private async adoptLogAuthorityInner(): Promise { - let report = await this.runOracle({ listAll: true }) + let report = await this.verifyLogAuthority() // BASELINE BACKFILL: curable divergences are rows whose CANONICAL truth // simply never reached the log — pre-log records (e.g. the generation-0 @@ -8278,18 +8268,8 @@ export class Brainy implements BrainyInterface { // Log-AHEAD divergences (log-live-canonical-absent / // log-tombstone-canonical-present) are NOT curable by backfill — the // log claims things the witness denies — and refuse loudly below. - // - // RUNS TO COMPLETION. Each pass sees the ENTIRE curable set (the oracle - // is run uncapped here) and cures all of it, so a pre-log baseline of - // any size adopts in ONE call — the only stop is the no-progress guard. - // A production brain with a 12.7k-row baseline once advanced exactly - // 800 rows per call (a five-pass ceiling × the 200-row wire cap) and sat - // tree-authoritative for hours; the bound was sized for drift, never - // for a baseline. Pace rides the write path now: one full-brain scan - // per pass amortizes over thousands of cures, not two hundred. let passes = 0 - for (;;) { - if (report.verdict !== 'red') break + while (report.verdict === 'red' && passes < 5) { passes++ const curable = report.mismatches.filter( (m) => m.reason === 'pre-log-record' || m.reason === 'state-differs' @@ -8310,19 +8290,9 @@ export class Brainy implements BrainyInterface { `[Brainy] adoptLogAuthority: baseline backfill pass ${passes} — re-committing ` + `${curable.length} row(s) whose canonical truth never reached the log` ) - // Progress narration for a live operator: a large baseline is minutes - // of visible motion, never a silent wait. - const narrateEvery = curable.length >= 2000 ? 1000 : curable.length >= 400 ? 200 : 0 - let cured = 0 for (const m of curable) { const raw = await this.storage.readNounRaw(m.id) if (raw.metadata === null && raw.vector === null) continue // vanished since the scan - cured++ - if (narrateEvery > 0 && cured % narrateEvery === 0) { - prodLog.info( - `[Brainy] adoptLogAuthority: backfill pass ${passes} — ${cured}/${curable.length} rows re-committed` - ) - } // LAW-SHAPE RE-COMMIT: rewrite canonical as EXACTLY the wrapper the // log's reconstruction produces (the hydration law: denormalized // enumeration fields derived from the metadata leg + the embedding @@ -8360,11 +8330,12 @@ export class Brainy implements BrainyInterface { }) }) } - const next = await this.runOracle({ listAll: true }) - // THE ONLY STOP: no progress. With uncapped listings both counts are - // exact, so "not fewer mismatches than before" means the cure could - // not express this divergence — refuse to spin, name it. - if (next.verdict === 'red' && next.mismatches.length >= report.mismatches.length) { + const next = await this.verifyLogAuthority() + if ( + next.verdict === 'red' && + next.mismatches.length >= report.mismatches.length && + !report.mismatchListTruncated + ) { throw new Error( `adoptLogAuthority(): baseline backfill made no progress ` + `(${report.mismatches.length} → ${next.mismatches.length} mismatches; first: ` + @@ -8374,12 +8345,6 @@ export class Brainy implements BrainyInterface { } report = next } - if (passes > 0) { - prodLog.info( - `[Brainy] adoptLogAuthority: baseline backfill complete in ${passes} pass(es) — ` + - `oracle ${report.verdict}, ${report.nounsChecked} noun(s) checked` - ) - } this._logAuthority = await flipToLogAuthority( this.storage as unknown as LogAuthorityStorage, diff --git a/src/db/logAuthority.ts b/src/db/logAuthority.ts index b63ae715..0703d11f 100644 --- a/src/db/logAuthority.ts +++ b/src/db/logAuthority.ts @@ -165,16 +165,7 @@ export async function runLogCompletenessOracle(args: { getVerbs?: (opts: { pagination: { limit: number; offset?: number; cursor?: string } }) => Promise<{ items: unknown[]; hasMore?: boolean; nextCursor?: string }> - /** - * Cap on the LISTED mismatches (counts are always complete). Defaults to - * the wire-friendly {@link MISMATCH_LIST_CAP}; the adoption backfill passes - * `Infinity` so ONE scan yields the ENTIRE curable set — a production - * brain with a 12.7k-row pre-log baseline once advanced only 800 rows per - * adoption call because each pass could see (and cure) at most 200. - */ - mismatchListCap?: number }): Promise { - const listCap = args.mismatchListCap ?? MISMATCH_LIST_CAP const report: OracleReport = { verdict: 'red', generationsScanned: 0, @@ -185,7 +176,7 @@ export async function runLogCompletenessOracle(args: { mismatchListTruncated: false } const addMismatch = (m: OracleMismatch): void => { - if (report.mismatches.length < listCap) report.mismatches.push(m) + if (report.mismatches.length < MISMATCH_LIST_CAP) report.mismatches.push(m) else report.mismatchListTruncated = true } diff --git a/tests/integration/adopt-large-baseline.test.ts b/tests/integration/adopt-large-baseline.test.ts deleted file mode 100644 index 11a3c803..00000000 --- a/tests/integration/adopt-large-baseline.test.ts +++ /dev/null @@ -1,92 +0,0 @@ -/** - * @module tests/integration/adopt-large-baseline - * @description Adoption runs the baseline backfill TO COMPLETION in one call. - * A production brain with a 12.7k-row pre-log baseline once advanced exactly - * 800 rows per `adoptLogAuthority()` call (a five-pass ceiling × the oracle's - * 200-row listing cap), refused the flip, and sat tree-authoritative for - * hours across restarts. The pin: a baseline larger than that old ceiling - * — every row oracle-visible as `state-differs` drift — adopts GREEN in a - * SINGLE call, and the row count proves the whole set was cured, not a page. - */ -import { describe, it, expect, afterEach } from 'vitest' -import { mkdtempSync, rmSync } from 'node:fs' -import { tmpdir } from 'node:os' -import { join } from 'node:path' -import { Brainy } from '../../src/index.js' -import { NounType } from '../../src/types/graphTypes.js' - -type RawBox = { - storage: { - readNounRaw(id: string): Promise<{ metadata: unknown; vector: unknown }> - writeNounRaw(id: string, r: { metadata: unknown; vector: unknown }): Promise - } -} - -const dirs: string[] = [] -const brains: Brainy[] = [] -afterEach(async () => { - for (const b of brains.splice(0)) await b.close().catch(() => {}) - for (const d of dirs.splice(0)) rmSync(d, { recursive: true, force: true }) -}) - -describe('adoption backfill runs to completion', () => { - it('a pre-log baseline larger than the old 800-row ceiling adopts GREEN in ONE call', async () => { - const dir = mkdtempSync(join(tmpdir(), 'brainy-large-baseline-')) - dirs.push(dir) - const brain = new Brainy({ - storage: { type: 'filesystem', path: dir }, - requireSubtype: false, - logAuthority: 'defer' - }) - await brain.init() - brains.push(brain) - - // Above the old ceiling (5 passes × 200 = 800): every row must be cured - // in the one call for the flip to be legal. - const ROWS = 1000 - const ids: string[] = [] - for (let i = 0; i < ROWS; i++) { - ids.push( - await brain.add({ - data: `baseline row ${i}`, - type: NounType.Document, - metadata: { i }, - vector: Array.from({ length: 384 }, (_, k) => ((i + k) % 7) / 7) - }) - ) - } - await brain.flush() - - // Manufacture the production shape on EVERY row: pre-hydration-law drift - // (a stored wrapper whose denormalized fields disagree with its own - // metadata leg) — each is a curable `state-differs` mismatch, so the - // oracle's full curable set is ROWS, well past any per-pass page. - const storage = (brain as unknown as RawBox).storage - for (const id of ids) { - const raw = await storage.readNounRaw(id) - const wrapper = raw.vector as Record - await storage.writeNounRaw(id, { - metadata: raw.metadata, - vector: { ...wrapper, noun: 'thing', legacyField: 'pre-law residue' } - }) - } - const before = await brain.verifyLogAuthority() - expect(before.verdict, 'the whole baseline is oracle-red').toBe('red') - // The wire report is capped at 200 — the truncation flag is what the old - // loop bounded itself on; the cure path no longer reads through it. - expect(before.mismatchListTruncated).toBe(true) - - // THE PIN: one call, green, log-authoritative — no restarts, no loop. - const report = await brain.adoptLogAuthority() - expect(report.verdict).toBe('green') - expect(brain.logAuthority().authority).toBe('log') - expect(report.nounsChecked).toBeGreaterThanOrEqual(ROWS) - - // Nothing degraded: a sample of rows still serves with intact metadata. - for (const id of [ids[0], ids[499], ids[ROWS - 1]]) { - const row = await brain.get(id) - expect(row).not.toBeNull() - expect(typeof (row!.metadata as { i: number }).i).toBe('number') - } - }, 600000) -})