From ed7d1db97e964ad25c2b8b37afdd5bfa209a5665 Mon Sep 17 00:00:00 2001 From: David Snelling Date: Tue, 18 Aug 2026 12:53:50 -0700 Subject: [PATCH 01/84] fix(recovery): the fold streams and narrates; the checkpoint chain arms at the flip MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A production brain's first process boot after a live authority flip looked hung and was restarted three times mid-recovery — three defects with one scene. (1) THE FOLD MATERIALIZED THE LOG: peekFactsAbove(0) decoded every fact into one array (GBs of after-images on a ~7k-fact log, a GC storm, a starved write lane). The fold now STREAMS one segment-batch at a time — memory is one segment at any log size — with structural ordering asserted loudly. (2) THE FOLD WAS SILENT UNTIL DONE: minutes of boot work with zero narration is what invited the restarts. It now announces itself BEFORE the work ('do not restart, the fold is finite') and prints progress every thousand facts. (3) THE CHAIN COULD ONLY ARM AT A CRASH: a live mid-session flip left the fold checkpoint unfounded, so the brain's first unclean boot paid a whole-log fold. Adoption now founds the checkpoint AT THE FLIP — one paged full canonical barrier (bounded memory), then the stamp — so bounded recovery holds from minute zero for every store that flips, at any size. Pinned: a non-fresh flip stamps immediately; the first post-flip unclean boot folds bounded (an unflushed at-ack fact above the checkpoint is restored; a barrier-covered row below it is outside the fold). Kill matrix and both adoption suites green alongside. --- src/brainy.ts | 51 +++++++ src/db/factLog.ts | 37 +++++ src/db/generationStore.ts | 128 +++++++++++++----- .../integration/fold-checkpoint-bound.test.ts | 32 +++++ 4 files changed, 215 insertions(+), 33 deletions(-) diff --git a/src/brainy.ts b/src/brainy.ts index d1ec144b..fb1c1614 100644 --- a/src/brainy.ts +++ b/src/brainy.ts @@ -8392,6 +8392,57 @@ export class Brainy implements BrainyInterface { // Fold-checkpoint chain, phase 2: the flip is recorded — open the stamp // gate so the next flush/close barrier writes the first checkpoint. this.generationStore.completeFoldCheckpointBootstrap() + // ARM-AT-FLIP for the NON-FRESH brain (the chain refused the fresh-brain + // arm because committed > 0): run one paged FULL canonical barrier now — + // every live row's canonical bytes fsynced, bounded memory — then stamp + // the first checkpoint. Without this, the chain could only arm at the + // brain's first crash, and that crash paid a WHOLE-LOG fold: a production + // brain hit exactly that on its first post-flip boot (a full-log + // materializing fold, restarted three times mid-flight). Adoption already + // pays O(N) oracle work; one more O(N) barrier founds bounded recovery + // from minute zero. + if (!this.generationStore.foldCheckpointChainArmed()) { + const PAGE = 500 + let synced = 0 + prodLog.info( + `[Brainy] adoptLogAuthority: founding the fold checkpoint — syncing every ` + + `row's canonical bytes (paged; progress every 2000 rows)` + ) + let offset = 0 + let cursor: string | undefined + for (;;) { + const page = await this.storage.getNouns({ + pagination: cursor ? { limit: PAGE, cursor } : { limit: PAGE, offset } + }) + const ids = page.items.map((i) => (i as { id: string }).id) + if (ids.length > 0) { + await this.storage.syncEntityCanonical?.(ids, []) + synced += ids.length + if (synced % 2000 < PAGE && synced >= 2000) { + prodLog.info(`[Brainy] adoptLogAuthority: checkpoint founding — ${synced} rows synced`) + } + } + if (page.hasMore && page.nextCursor) { cursor = page.nextCursor; offset += ids.length; continue } + if (page.hasMore && !page.nextCursor) { offset += PAGE; continue } + break + } + let vOffset = 0 + let vCursor: string | undefined + for (;;) { + const page = await this.storage.getVerbs({ + pagination: vCursor ? { limit: PAGE, cursor: vCursor } : { limit: PAGE, offset: vOffset } + }) + const ids = page.items.map((i) => (i as { id: string }).id) + if (ids.length > 0) { + await this.storage.syncEntityCanonical?.([], ids) + synced += ids.length + } + if (page.hasMore && page.nextCursor) { vCursor = page.nextCursor; vOffset += ids.length; continue } + if (page.hasMore && !page.nextCursor) { vOffset += PAGE; continue } + break + } + await this.generationStore.stampFoldCheckpointAfterFullBarrier() + } return report } diff --git a/src/db/factLog.ts b/src/db/factLog.ts index 82949fb6..ca130454 100644 --- a/src/db/factLog.ts +++ b/src/db/factLog.ts @@ -770,6 +770,43 @@ export class FactLog { * segments directly; the torn tail's invalid suffix is ignored exactly * like open() would). */ + /** + * STREAMING twin of {@link FactLog.peekFactsAbove} for the recovery fold: + * yields facts above the bound one SEGMENT at a time, ascending, without + * ever materializing the whole log (a production first-boot fold OOM-class + * allocation storm came from exactly that — GBs of decoded after-images in + * one array while the process looked hung). Memory is one segment's worth. + * Works manifest-direct (safe before {@link FactLog.open}). Ordering is + * structural (segments rotate in order; appends are ordered within one) and + * ASSERTED — a violation aborts loudly, never a silent misordered replay. + */ + async *streamFactsAbove(committedGeneration: number): AsyncGenerator { + const stored = (await this.storage.readRawObject(FACTS_MANIFEST_PATH)) as FactsManifest | null + if (!stored || typeof stored !== 'object' || !Array.isArray(stored.segments)) return + if (stored.formatVersion !== FACTS_FORMAT_VERSION) return + const files = [...stored.segments.map((s) => s.file)] + if (stored.tailSegment) files.push(stored.tailSegment) + let lastGen = committedGeneration + for (const file of files) { + const bytes = await this.storage.readRawBytes(`${FACTS_PREFIX}/${file}`) + if (bytes === null) continue + const { facts } = parseSegment(file, bytes) + const batch: CommitFact[] = [] + for (const f of facts) { + if (f.generation <= committedGeneration) continue + if (f.generation <= lastGen) { + throw new Error( + `fact log: streamFactsAbove found non-ascending generations ` + + `(${f.generation} after ${lastGen} in ${file}) — refusing to replay out of order` + ) + } + lastGen = f.generation + batch.push(f) + } + if (batch.length > 0) yield batch + } + } + async peekFactsAbove(committedGeneration: number): Promise { const stored = (await this.storage.readRawObject(FACTS_MANIFEST_PATH)) as FactsManifest | null if (!stored || typeof stored !== 'object' || !Array.isArray(stored.segments)) return [] diff --git a/src/db/generationStore.ts b/src/db/generationStore.ts index 065b3659..bfb68959 100644 --- a/src/db/generationStore.ts +++ b/src/db/generationStore.ts @@ -637,33 +637,63 @@ export class GenerationStore { this.foldCheckpointChainValid = checkpoint !== null || this.committed === 0 this.foldCheckpoint = foldBound if (uncleanOpen) this.foldCheckpointChainValid = true - const factsToReplay = uncleanOpen - ? await this.factLog.peekFactsAbove(foldBound) - : orphans - if (factsToReplay.length > 0) { - let replayed = 0 - for (const fact of factsToReplay) { - for (const op of fact.ops) { - const image = - op.record === null - ? { metadata: null, vector: null } - : { metadata: op.record.metadata, vector: op.record.vector } - if (op.kind === 'verb') await this.storage.writeVerbRaw(op.id, image) - else await this.storage.writeNounRaw(op.id, image) - this.noteCheckpointDirty(op.kind, op.id) - } - replayed++ - if (fact.generation > this.committed) { - this.committed = fact.generation - this.appendCommittedGen(fact.generation) - this.setDelta(fact.generation, { - nouns: new Set(fact.ops.filter((o) => o.kind === 'noun').map((o) => o.id)), - verbs: new Set(fact.ops.filter((o) => o.kind === 'verb').map((o) => o.id)), - timestamp: fact.timestamp, - bytes: 0 - }) - } + // THE FOLD STREAMS AND NARRATES. A production first boot after a live + // flip folded ~7k facts by materializing them all (GBs of decoded + // after-images, a GC storm, a starved write lane) in SILENCE — the + // operator restarted the process three times mid-fold, each restart + // making the next boot unclean again. Two laws from that day: the + // fold consumes the log one segment-batch at a time (memory = one + // segment, any log size), and it announces itself BEFORE the work + // with progress lines DURING it — an operator who can see a fold + // converging lets it finish. + const foldKind = uncleanOpen + ? foldBound > 0 + ? `BOUNDED fold above checkpoint ${foldBound}` + : 'WHOLE-LOG fold' + : 'above-manifest replay' + let replayed = 0 + const replayFact = async (fact: CommitFact): Promise => { + for (const op of fact.ops) { + const image = + op.record === null + ? { metadata: null, vector: null } + : { metadata: op.record.metadata, vector: op.record.vector } + if (op.kind === 'verb') await this.storage.writeVerbRaw(op.id, image) + else await this.storage.writeNounRaw(op.id, image) + this.noteCheckpointDirty(op.kind, op.id) } + replayed++ + if (replayed % 1000 === 0) { + prodLog.warn( + `[GenerationStore] recovery fold in progress — ${replayed} fact(s) folded ` + + `(at generation ${fact.generation}); do not restart, the fold is finite` + ) + } + if (fact.generation > this.committed) { + this.committed = fact.generation + this.appendCommittedGen(fact.generation) + this.setDelta(fact.generation, { + nouns: new Set(fact.ops.filter((o) => o.kind === 'noun').map((o) => o.id)), + verbs: new Set(fact.ops.filter((o) => o.kind === 'verb').map((o) => o.id)), + timestamp: fact.timestamp, + bytes: 0 + }) + } + } + if (uncleanOpen) { + prodLog.warn( + `[GenerationStore] log-authority recovery: ${foldKind} beginning ` + + `(unclean shutdown detected) — streaming replay, bounded memory, ` + + `progress every 1000 facts. Do not restart the process; a restart ` + + `re-pays the whole fold.` + ) + for await (const batch of this.factLog.streamFactsAbove(foldBound)) { + for (const fact of batch) await replayFact(fact) + } + } else { + for (const fact of orphans) await replayFact(fact) + } + if (replayed > 0) { if (this.counter < this.committed) this.counter = this.committed await this.persistCounterUnlocked() const manifest: GenerationManifest = { @@ -676,13 +706,7 @@ export class GenerationStore { await this.storage.syncRawObjects([MANIFEST_PATH]) prodLog.warn( `[GenerationStore] log-authority recovery replayed ${replayed} fact(s) into ` + - `canonical (${ - uncleanOpen - ? foldBound > 0 - ? `BOUNDED fold above checkpoint ${foldBound} — unclean shutdown` - : 'WHOLE-LOG fold — unclean shutdown' - : 'above-manifest' - }; committed at ${this.committed}) — an acked write is never lost` + `canonical (${foldKind}; committed at ${this.committed}) — an acked write is never lost` ) } // A recovery fold re-applied (and the barrier below re-syncs) every @@ -897,6 +921,44 @@ export class GenerationStore { this.authorityIsLog = true } + /** Whether the fold-checkpoint chain is armed (a bounded fold is possible). */ + foldCheckpointChainArmed(): boolean { + return this.foldCheckpointChainValid + } + + /** + * @description Stamp the fold checkpoint after the caller has completed a + * FULL canonical barrier (every live row's canonical bytes fsynced, paged — + * the adoption path does this right after a non-fresh flip). The stamp + * asserts total coverage, so it may ONLY be called when the barrier walked + * everything; stamp-after-data is the caller's ordering to keep. Arms the + * chain: the brain's first unclean boot folds (checkpoint, head] instead of + * the whole log — a production first boot after a live flip paid a full-log + * fold through three mid-fold restarts because the chain could previously + * only arm at a crash. + */ + async stampFoldCheckpointAfterFullBarrier(): Promise { + return this.withMutex(async () => { + if (!this.authorityIsLog || !this.factLog) { + throw new Error( + 'stampFoldCheckpointAfterFullBarrier: only a log-authority brain stamps a fold checkpoint' + ) + } + this.foldCheckpointChainValid = true + // The full barrier supersedes any accumulated partial set. + this.checkpointDirtyNouns = new Set() + this.checkpointDirtyVerbs = new Set() + const target = this.committed + await this.storage.writeRawObject(FOLD_CHECKPOINT_PATH, { generation: target }) + await this.storage.syncRawObjects([FOLD_CHECKPOINT_PATH]) + this.foldCheckpoint = target + prodLog.info( + `[GenerationStore] fold checkpoint founded at generation ${target} — ` + + `crash recovery is bounded from this moment` + ) + }) + } + /** * @description Adoption-time chain bootstrap, abort — called when an * adoption attempt throws or refuses after phase 1. Disarms the chain and diff --git a/tests/integration/fold-checkpoint-bound.test.ts b/tests/integration/fold-checkpoint-bound.test.ts index 60bcce5e..2f21248b 100644 --- a/tests/integration/fold-checkpoint-bound.test.ts +++ b/tests/integration/fold-checkpoint-bound.test.ts @@ -161,6 +161,38 @@ describe('fold-checkpoint bound — crash recovery folds (checkpoint, head], nev expect(stamped, 'the first whole-log fold is the chain’s base case — it stamps').toBe(committedOf(reopened)) }, 120000) + it('ARM-AT-FLIP: a non-fresh adoption founds the checkpoint immediately — the first post-flip boot folds BOUNDED, never whole-log', async () => { + const dir = trackDir() + // The production shape: a brain with history flips LIVE (no crash ever). + const brain = await openBrain(dir, { logAuthority: 'defer' }) + liveBrains.push(brain) + const preFlip = await brain.add({ data: 'pre-flip resident', type: NounType.Document, metadata: { era: 'tree' } }) + await brain.flush() + expect(readCheckpoint(dir), 'no checkpoint before the flip').toBeNull() + + const report = await brain.adoptLogAuthority() + expect(report.verdict).toBe('green') + // THE PIN: the flip itself founded the checkpoint — no crash required. + const founded = readCheckpoint(dir) + expect(founded, 'checkpoint founded at flip').toBe(committedOf(brain)) + + // First post-flip boot, unclean (the production first-restart shape): + // a post-flip write above the checkpoint is restored FROM ITS AT-ACK FACT + // (deliberately NOT flushed — a flush would barrier-sync it and advance + // the stamp over it, making its loss synthetic); the pre-flip row (its + // baseline fact ≤ checkpoint, its bytes barrier-synced at the flip) is + // OUTSIDE the fold — vaporizing it synthetically proves the bound. + const postFlip = await brain.add({ data: 'post-flip write', type: NounType.Document, metadata: { era: 'log' } }) + await abandonAsCrashed(liveBrains.pop()!) + dropCanonicalNoun(dir, preFlip) + dropCanonicalNoun(dir, postFlip) + + const reopened = await openBrain(dir, { logAuthority: 'adopt' }) + liveBrains.push(reopened) + expect(await reopened.get(postFlip), 'above-checkpoint fact re-applied').not.toBeNull() + expect(await reopened.get(preFlip), 'below-checkpoint fact skipped — the fold is bounded on the FIRST post-flip boot').toBeNull() + }, 240000) + it('a tree-authority brain never stamps a checkpoint', async () => { const dir = trackDir() const brain = await openBrain(dir, { logAuthority: 'defer' }) From 900cc89564275e9647d8ea45cb099a2e24b308ff Mon Sep 17 00:00:00 2001 From: David Snelling Date: Tue, 18 Aug 2026 13:18:55 -0700 Subject: [PATCH 02/84] =?UTF-8?q?docs(releases):=20the=2010.3.1=20consumer?= =?UTF-8?q?=20entry=20=E2=80=94=20the=20fold=20that=20behaves?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- RELEASES.md | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/RELEASES.md b/RELEASES.md index 49876f5a..cc0272c3 100644 --- a/RELEASES.md +++ b/RELEASES.md @@ -31,6 +31,33 @@ is sometimes cited as a 7.x removal — those methods never existed on 7.x; the --- +## v10.3.1 — 2026-08-18 (the fold that behaves) + +Three recovery cures from one production first-boot incident (a brain's first +process restart after a live storage-authority flip looked hung and was +restarted three times mid-recovery). **Adopt this version before flipping +brains with existing history** — it is the intended adoption target for +fleets moving to the crash-safe authority. + +- **Recovery streams.** The boot-time log fold now consumes the generation + log one segment-batch at a time — memory stays bounded at one segment for + any log size. Previously it materialized every fact into one array, which + on a ~7k-fact log produced multi-GB allocation pressure and a process that + looked wedged while it worked. +- **Recovery narrates.** The fold announces itself before the work begins + ("recovery fold beginning — do not restart, the fold is finite") and prints + progress every thousand facts. A visible fold gets to finish; a silent one + gets killed by a well-meaning operator, and each kill makes the next boot + pay the whole fold again. +- **Bounded recovery from the flip itself.** Adopting the log authority now + founds the recovery checkpoint at the moment of the flip (one paged + canonical sync, bounded memory, then the stamp) — so even the FIRST unclean + shutdown after a flip replays only the log's tail. Previously the bound + could only establish itself at a completed crash recovery, which is exactly + the recovery the incident kept interrupting. + +--- + ## v10.3.0 — 2026-08-18 (the trust-and-provenance release) Four consumer-driven cures. Pairs with the same native accelerator line From 522b0cf827489f91b3cf91f95eb0af7cae6d5ae7 Mon Sep 17 00:00:00 2001 From: David Snelling Date: Tue, 18 Aug 2026 13:19:17 -0700 Subject: [PATCH 03/84] chore(release): 10.3.1 --- CHANGELOG.md | 6 ++++++ package-lock.json | 4 ++-- package.json | 2 +- 3 files changed, 9 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5ee1e723..f99584e5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,12 @@ All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines. +### [10.3.1](https://source.soulcraft.com/soulcraft/brainy/compare/v10.3.0...v10.3.1) (2026-08-18) + +- docs(releases): the 10.3.1 consumer entry — the fold that behaves (900cc895) +- fix(recovery): the fold streams and narrates; the checkpoint chain arms at the flip (ed7d1db9) + + ### [10.3.0](https://source.soulcraft.com/soulcraft/brainy/compare/v10.2.0...v10.3.0) (2026-08-18) - docs(releases): the 10.3.0 consumer entry — the trust-and-provenance release (97d75649) diff --git a/package-lock.json b/package-lock.json index a5913ac7..afce417d 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "@soulcraft/brainy", - "version": "10.3.0", + "version": "10.3.1", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@soulcraft/brainy", - "version": "10.3.0", + "version": "10.3.1", "license": "MIT", "dependencies": { "@msgpack/msgpack": "^3.1.2", diff --git a/package.json b/package.json index 241f23a8..75e5bfbc 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@soulcraft/brainy", - "version": "10.3.0", + "version": "10.3.1", "description": "Universal Knowledge Protocol™ - World's first Triple Intelligence database unifying vector, graph, and document search in one API. Stage 3 CANONICAL: 42 nouns × 127 verbs covering 96-97% of all human knowledge.", "main": "dist/index.js", "module": "dist/index.js", From 1e046aa115637b9b0971b8fe301152e318ed5bc8 Mon Sep 17 00:00:00 2001 From: David Snelling Date: Thu, 20 Aug 2026 08:22:48 -0700 Subject: [PATCH 04/84] ci(gate): the machine-health preflight and the truncation verdict guard MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two guards for every gate lane, born from the 2026-08-13 lost-day ledger. gate-preflight.sh refuses a lane on a machine that cannot be trusted to produce honest numbers — co-tenant processes named by pid and command, load average, CPU governor, disk floors — one FATAL line per violation so the operator can act from the message alone. vitest-verdict-check.sh refuses a suite log that cannot be trusted as a verdict — missing or mismatched summary counts, files that never executed (a truncated run once read as green from three files of ninety-nine), and worker-pool death signatures. Both verified live: the preflight correctly refuses this workstation naming its actual offenders; the verdict guard passes/fails five fixture shapes (clean, wrong-count, truncated, worker-death, no-summary) and both CLI modes. Wire-up into the CI lanes rides the runner program. --- scripts/gate/README.md | 85 +++++++++++ scripts/gate/gate-preflight.sh | 206 +++++++++++++++++++++++++++ scripts/gate/vitest-verdict-check.sh | 158 ++++++++++++++++++++ 3 files changed, 449 insertions(+) create mode 100644 scripts/gate/README.md create mode 100755 scripts/gate/gate-preflight.sh create mode 100755 scripts/gate/vitest-verdict-check.sh diff --git a/scripts/gate/README.md b/scripts/gate/README.md new file mode 100644 index 00000000..0a8afab0 --- /dev/null +++ b/scripts/gate/README.md @@ -0,0 +1,85 @@ +# Gate Guards + +Two standalone scripts that stand between a test/build gate and a false +verdict: one refuses to let the gate start on a noisy machine, the other +refuses to let a truncated or crashed vitest run be read as green. + +## Why these exist + +Both guards exist because of the 2026-08-13 lost-day ledger: a gate ran on +a machine under load, and separately a vitest worker pool died mid-suite +while still printing a plausible-looking summary line, and in both cases +the bad result was trusted and acted on for the better part of a day before +anyone noticed. Neither failure mode announces itself — a loaded machine +still finishes and reports numbers, and a truncated test run still prints a +`Test Files` / `Tests` line — so both guards check the evidence explicitly +rather than trusting that a gate finishing means the gate was valid. + +## gate-preflight.sh + +Run before any gate lane starts. Exits 1 the moment the machine isn't +gate-clean, with one `FATAL:` line per violation naming the exact offender +(the pid and command, the path, the measured value). Prints one `OK:` line +per check that passes. `WARNING:` lines mark checks that were skipped, not +failures. + +Checks: + +| # | Check | Default threshold | Override | +|---|-------|--------------------|----------| +| a | 1-minute load average | `nproc / 2` | `GATE_MAX_LOAD` | +| b | any non-allowlisted process over 50% of one core | 50% | `GATE_ALLOW_REGEX` (extra pattern matched against the process's args) | +| c | cpu0 scaling governor must be `performance` | — | none (warns and skips if the sysfs path is absent) | +| d | free space on `/` and `/tmp` | 10G each | `GATE_SKIP_DISK_CHECK=1` to skip entirely | + +The allowlist for check (b) is always: this script's own process tree +(its ancestors and its direct child processes), `sshd`, `systemd`, and +kernel threads (recognizable by args wrapped in brackets, e.g. +`[kworker/0:1]`). `GATE_ALLOW_REGEX` extends it — it does not replace it. + +## vitest-verdict-check.sh + +Run after every vitest lane, against that lane's captured log. Fails +loudly, quoting the exact line or string that tripped it, when the log's +own summary can't be trusted: + +- no `Test Files` (or, in `--count-tests` mode, `Tests`) summary line is + present at all +- the parenthesized total in that line doesn't match what was expected +- fewer files/tests are accounted for (passed + failed + skipped) than the + total claims — a truncated run +- the log contains `Unhandled Error` or `Timeout calling` anywhere — a dead + worker pool, regardless of what the summary line claims + +``` +vitest-verdict-check.sh +vitest-verdict-check.sh --count-tests +``` + +The first form checks `Test Files` for an exact match. The second checks +`Tests` for a minimum (a floor, not an exact count, since the total number +of individual tests moves more often than the number of test files). + +## Wiring into a CI lane + +```sh +# Before any lane that will report a verdict: +scripts/gate/gate-preflight.sh || exit 1 + +# Run the suite, capturing its output: +npx vitest run tests/unit 2>&1 | tee /tmp/unit.log + +# After every vitest lane, check the log against the actual file count: +EXPECTED_FILES=$(ls tests/unit/**/*.test.ts | wc -l) +scripts/gate/vitest-verdict-check.sh /tmp/unit.log "$EXPECTED_FILES" || exit 1 +``` + +## Exit-code contract + +| Script | Exit 0 | Exit 1 | +|--------|--------|--------| +| `gate-preflight.sh` | machine is gate-clean | one or more `FATAL:` violations printed | +| `vitest-verdict-check.sh` | log's summary is trustworthy and matches | usage error, missing/unreadable log, or one or more `FATAL:` violations printed | + +Non-zero from either script means: do not trust the gate that was about to +run, or the result of the one that just ran. diff --git a/scripts/gate/gate-preflight.sh b/scripts/gate/gate-preflight.sh new file mode 100755 index 00000000..c6208f49 --- /dev/null +++ b/scripts/gate/gate-preflight.sh @@ -0,0 +1,206 @@ +#!/bin/bash +set -euo pipefail + +# Brainy Gate Preflight +# Refuses to let a test/build gate run on a machine that isn't clean enough +# to trust the numbers it produces. See scripts/gate/README.md for why (the +# 2026-08-13 lost-day ledger). +# +# Checks: 1-minute load average, any non-allowlisted process pinning a core, +# the cpu0 scaling governor, and free space on / and /tmp. +# +# Exit 0 and print one OK line per passing check when the machine is clean. +# Exit 1 and print one FATAL line per violation, naming the offender, when +# it is not. +# +# Known trap: a helper function whose last executed statement is a `while` +# (or any command whose own exit status happens to be nonzero) hands that +# status back as the function's return value. Called as a plain statement, +# that silently kills this script under `set -e`. Every helper below ends +# on an explicit `return 0` as its own statement, never on a loop or test. +# +# The same failure mode hides in plainer-looking lines too: `var=$(cmd)` is +# a bare assignment, so `set -e` DOES treat a nonzero `cmd` (or, under +# `pipefail`, a nonzero stage anywhere in `cmd`'s pipeline) as a failure of +# that statement and kills the script right there — even mid-loop, even +# when the "failure" is routine (a process that exited before a second +# lookup, a path that doesn't exist). Every such assignment below is paired +# with an explicit `|| var=""` fallback so a routine miss degrades to an +# empty value instead of an exit. + +VIOLATIONS=0 +ANCESTOR_PIDS="" + +fatal() { + echo "FATAL: $1" + VIOLATIONS=$((VIOLATIONS + 1)) +} + +ok() { + echo "OK: $1" +} + +# Walks this process's parent chain up to pid 1, then takes one snapshot of +# its direct children (the ps/read pipeline in check_processes), and +# records both in ANCESTOR_PIDS — so the process-scan below can recognize +# its own tree (the shell/terminal/session that launched it, plus its own +# helper commands) instead of flagging it. Children are captured once, up +# front, rather than re-queried per row later, so a helper command that has +# already exited by the time it's looked up can't be mistaken for a miss. +build_ancestor_pids() { + local pid="$$" + local ppid child + ANCESTOR_PIDS=" $pid " + while [ "$pid" != "1" ]; do + ppid=$(ps -o ppid= -p "$pid" 2>/dev/null | tr -d ' ') || ppid="" + if [ -z "$ppid" ]; then + break + fi + ANCESTOR_PIDS="${ANCESTOR_PIDS}${ppid} " + pid="$ppid" + done + + while IFS= read -r child; do + [ -z "$child" ] && continue + ANCESTOR_PIDS="${ANCESTOR_PIDS}${child} " + done < <(ps --ppid "$$" -o pid= 2>/dev/null || true) + + return 0 +} + +# (a) 1-minute load average vs. threshold (default: nproc / 2). +check_load() { + local max_load="${GATE_MAX_LOAD:-}" + if [ -z "$max_load" ]; then + max_load=$(( $(nproc) / 2 )) + if [ "$max_load" -lt 1 ]; then + max_load=1 + fi + fi + + local load_1m + load_1m=$(cut -d' ' -f1 /proc/loadavg) + + if awk -v l="$load_1m" -v m="$max_load" 'BEGIN { exit !(l > m) }'; then + fatal "1-minute load average ${load_1m} exceeds threshold ${max_load} (GATE_MAX_LOAD=${max_load})" + else + ok "1-minute load average ${load_1m} is within threshold ${max_load}" + fi + return 0 +} + +# (b) any process outside the allowlist pinning more than half a core. +# Parsed with `read` into named fields, not an awk/cut chain — a fixed-column +# awk/cut split on `ps` output duplicated fields the first time this was +# tried, because process args vary in word count. `read` with a fixed list +# of variables dumps everything left over into the last one (args), which +# handles that correctly. +check_processes() { + local max_pcpu=50 + local extra_regex="${GATE_ALLOW_REGEX:-}" + local violation_found=0 + local line pcpu pid args pcpu_int + + while IFS= read -r line; do + [ -z "$line" ] && continue + read -r pcpu pid args <<< "$line" + + # Kernel threads report their comm in brackets, e.g. "[kworker/0:1]". + case "$args" in + \[*\]) continue ;; + esac + + # This script's own tree: its ancestors (shell, terminal, session) and + # its direct children, both captured once by build_ancestor_pids. + case " $ANCESTOR_PIDS " in + *" $pid "*) continue ;; + esac + + case "$args" in + *sshd*|*systemd*) continue ;; + esac + + if [ -n "$extra_regex" ] && [[ "$args" =~ $extra_regex ]]; then + continue + fi + + pcpu_int="${pcpu%.*}" + if [ -z "$pcpu_int" ]; then + pcpu_int=0 + fi + if [ "$pcpu_int" -gt "$max_pcpu" ]; then + fatal "pid ${pid} ('${args}') is using ${pcpu}% of one core" + violation_found=1 + fi + done < <(ps -eo pcpu,pid,args --sort=-pcpu | tail -n +2) + + if [ "$violation_found" -eq 0 ]; then + ok "no process outside the allowlist exceeds ${max_pcpu}% of one core" + fi + return 0 +} + +# (c) cpu0 scaling governor must be "performance". Skipped with a warning +# (not a violation) when the sysfs path doesn't exist on this machine. +check_governor() { + local gov_path="/sys/devices/system/cpu/cpu0/cpufreq/scaling_governor" + if [ ! -r "$gov_path" ]; then + echo "WARNING: ${gov_path} not present; skipping governor check" + return 0 + fi + + local governor + governor=$(cat "$gov_path" 2>/dev/null) || governor="" + if [ "$governor" != "performance" ]; then + fatal "cpu0 governor is '${governor}', not 'performance'" + else + ok "cpu0 governor is 'performance'" + fi + return 0 +} + +# (d) free-space floors on / and /tmp (default 10G each). Skip entirely via +# GATE_SKIP_DISK_CHECK=1. +check_disk() { + if [ "${GATE_SKIP_DISK_CHECK:-0}" = "1" ]; then + echo "WARNING: disk free-space check skipped (GATE_SKIP_DISK_CHECK=1)" + return 0 + fi + + local floor_gb=10 + local floor_bytes=$((floor_gb * 1024 * 1024 * 1024)) + local path avail_bytes avail_gb + + for path in / /tmp; do + avail_bytes=$(df --output=avail -B1 "$path" 2>/dev/null | tail -n 1 | tr -d ' ') || avail_bytes="" + if [ -z "$avail_bytes" ]; then + echo "WARNING: could not determine free space on ${path}; skipping" + continue + fi + if [ "$avail_bytes" -lt "$floor_bytes" ]; then + avail_gb=$((avail_bytes / 1024 / 1024 / 1024)) + fatal "${path} has only ${avail_gb}G free, below the ${floor_gb}G floor" + else + ok "${path} has enough free space (floor ${floor_gb}G)" + fi + done + return 0 +} + +echo "Brainy gate preflight" +echo "----------------------" + +build_ancestor_pids +check_load +check_processes +check_governor +check_disk + +echo "----------------------" +if [ "$VIOLATIONS" -gt 0 ]; then + echo "FATAL: gate preflight failed with ${VIOLATIONS} violation(s) — machine is not gate-clean" + exit 1 +fi + +echo "gate preflight passed — machine is gate-clean" +exit 0 diff --git a/scripts/gate/vitest-verdict-check.sh b/scripts/gate/vitest-verdict-check.sh new file mode 100755 index 00000000..36243a1d --- /dev/null +++ b/scripts/gate/vitest-verdict-check.sh @@ -0,0 +1,158 @@ +#!/bin/bash +set -euo pipefail + +# Brainy Vitest Verdict Check +# Confirms a vitest run's own summary line is trustworthy before anything +# downstream treats a green run as green. See scripts/gate/README.md for why +# (the 2026-08-13 lost-day ledger). +# +# Usage: +# vitest-verdict-check.sh +# vitest-verdict-check.sh --count-tests +# +# The first form checks the "Test Files" summary line's total against an +# exact expected count. The second checks the "Tests" summary line's total +# against a minimum. Both also fail on any sign the worker pool died +# mid-run, whether or not a summary line still made it into the log. +# +# Exit 0 and print one OK line per passing check when the log is clean. +# Exit 1 and print one FATAL line per violation, quoting the exact line or +# string that tripped it, when it is not. +# +# Known trap (shared with gate-preflight.sh): every helper below ends on an +# explicit `return 0` as its own statement, never on a loop or test, so a +# helper's last command can never hand its own exit status back as the +# function's under `set -e`. The same applies to `var=$(cmd)` assignments +# mid-helper: a bare assignment IS checked by `set -e`, so a `grep` that +# legitimately finds nothing (exit 1) would otherwise kill the script +# instead of just leaving the variable empty — every such assignment below +# is paired with an explicit `|| true` inside the substitution. + +usage() { + echo "Usage: $0 " + echo " $0 --count-tests " + exit 1 +} + +MODE="files" +if [ "${1:-}" = "--count-tests" ]; then + MODE="tests" + shift +fi + +LOG_FILE="${1:-}" +THRESHOLD="${2:-}" + +if [ -z "$LOG_FILE" ] || [ -z "$THRESHOLD" ]; then + usage +fi + +if [ ! -f "$LOG_FILE" ]; then + echo "FATAL: log file '${LOG_FILE}' does not exist" + exit 1 +fi + +if ! [[ "$THRESHOLD" =~ ^[0-9]+$ ]]; then + echo "FATAL: threshold '${THRESHOLD}' is not a non-negative integer" + exit 1 +fi + +VIOLATIONS=0 + +fatal() { + echo "FATAL: $1" + VIOLATIONS=$((VIOLATIONS + 1)) +} + +ok() { + echo "OK: $1" +} + +# Vitest colorizes its summary with ANSI escapes; strip them before parsing +# anything, or the color codes end up embedded in the fields we grep for. +CLEAN_LOG="$(sed 's/\x1b\[[0-9;]*m//g' "$LOG_FILE")" + +# Worker-pool death: if either string appears, the run's own summary line — +# even if present and even if its numbers look fine — cannot be trusted, +# because the process died mid-suite and vitest's own accounting is what +# died with it. +check_worker_death() { + if echo "$CLEAN_LOG" | grep -q "Unhandled Error"; then + fatal "log contains 'Unhandled Error' — worker pool died mid-run" + fi + if echo "$CLEAN_LOG" | grep -q "Timeout calling"; then + fatal "log contains 'Timeout calling' — worker pool died mid-run" + fi + return 0 +} + +# Shared shape between the "Test Files" and "Tests" summary lines: +#