ci(gate): the machine-health preflight and the truncation verdict guard
Two guards for every gate lane, born from the 2026-08-13 lost-day ledger. gate-preflight.sh refuses a lane on a machine that cannot be trusted to produce honest numbers — co-tenant processes named by pid and command, load average, CPU governor, disk floors — one FATAL line per violation so the operator can act from the message alone. vitest-verdict-check.sh refuses a suite log that cannot be trusted as a verdict — missing or mismatched summary counts, files that never executed (a truncated run once read as green from three files of ninety-nine), and worker-pool death signatures. Both verified live: the preflight correctly refuses this workstation naming its actual offenders; the verdict guard passes/fails five fixture shapes (clean, wrong-count, truncated, worker-death, no-summary) and both CLI modes. Wire-up into the CI lanes rides the runner program.
This commit is contained in:
parent
522b0cf827
commit
1e046aa115
3 changed files with 449 additions and 0 deletions
158
scripts/gate/vitest-verdict-check.sh
Executable file
158
scripts/gate/vitest-verdict-check.sh
Executable file
|
|
@ -0,0 +1,158 @@
|
|||
#!/bin/bash
|
||||
set -euo pipefail
|
||||
|
||||
# Brainy Vitest Verdict Check
|
||||
# Confirms a vitest run's own summary line is trustworthy before anything
|
||||
# downstream treats a green run as green. See scripts/gate/README.md for why
|
||||
# (the 2026-08-13 lost-day ledger).
|
||||
#
|
||||
# Usage:
|
||||
# vitest-verdict-check.sh <log-file> <expected-file-count>
|
||||
# vitest-verdict-check.sh --count-tests <log-file> <minimum-test-count>
|
||||
#
|
||||
# The first form checks the "Test Files" summary line's total against an
|
||||
# exact expected count. The second checks the "Tests" summary line's total
|
||||
# against a minimum. Both also fail on any sign the worker pool died
|
||||
# mid-run, whether or not a summary line still made it into the log.
|
||||
#
|
||||
# Exit 0 and print one OK line per passing check when the log is clean.
|
||||
# Exit 1 and print one FATAL line per violation, quoting the exact line or
|
||||
# string that tripped it, when it is not.
|
||||
#
|
||||
# Known trap (shared with gate-preflight.sh): every helper below ends on an
|
||||
# explicit `return 0` as its own statement, never on a loop or test, so a
|
||||
# helper's last command can never hand its own exit status back as the
|
||||
# function's under `set -e`. The same applies to `var=$(cmd)` assignments
|
||||
# mid-helper: a bare assignment IS checked by `set -e`, so a `grep` that
|
||||
# legitimately finds nothing (exit 1) would otherwise kill the script
|
||||
# instead of just leaving the variable empty — every such assignment below
|
||||
# is paired with an explicit `|| true` inside the substitution.
|
||||
|
||||
usage() {
|
||||
echo "Usage: $0 <log-file> <expected-file-count>"
|
||||
echo " $0 --count-tests <log-file> <minimum-test-count>"
|
||||
exit 1
|
||||
}
|
||||
|
||||
MODE="files"
|
||||
if [ "${1:-}" = "--count-tests" ]; then
|
||||
MODE="tests"
|
||||
shift
|
||||
fi
|
||||
|
||||
LOG_FILE="${1:-}"
|
||||
THRESHOLD="${2:-}"
|
||||
|
||||
if [ -z "$LOG_FILE" ] || [ -z "$THRESHOLD" ]; then
|
||||
usage
|
||||
fi
|
||||
|
||||
if [ ! -f "$LOG_FILE" ]; then
|
||||
echo "FATAL: log file '${LOG_FILE}' does not exist"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if ! [[ "$THRESHOLD" =~ ^[0-9]+$ ]]; then
|
||||
echo "FATAL: threshold '${THRESHOLD}' is not a non-negative integer"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
VIOLATIONS=0
|
||||
|
||||
fatal() {
|
||||
echo "FATAL: $1"
|
||||
VIOLATIONS=$((VIOLATIONS + 1))
|
||||
}
|
||||
|
||||
ok() {
|
||||
echo "OK: $1"
|
||||
}
|
||||
|
||||
# Vitest colorizes its summary with ANSI escapes; strip them before parsing
|
||||
# anything, or the color codes end up embedded in the fields we grep for.
|
||||
CLEAN_LOG="$(sed 's/\x1b\[[0-9;]*m//g' "$LOG_FILE")"
|
||||
|
||||
# Worker-pool death: if either string appears, the run's own summary line —
|
||||
# even if present and even if its numbers look fine — cannot be trusted,
|
||||
# because the process died mid-suite and vitest's own accounting is what
|
||||
# died with it.
|
||||
check_worker_death() {
|
||||
if echo "$CLEAN_LOG" | grep -q "Unhandled Error"; then
|
||||
fatal "log contains 'Unhandled Error' — worker pool died mid-run"
|
||||
fi
|
||||
if echo "$CLEAN_LOG" | grep -q "Timeout calling"; then
|
||||
fatal "log contains 'Timeout calling' — worker pool died mid-run"
|
||||
fi
|
||||
return 0
|
||||
}
|
||||
|
||||
# Shared shape between the "Test Files" and "Tests" summary lines:
|
||||
# <label> <n> passed | <n> failed | <n> skipped (<total>)
|
||||
# `compare` is "eq" (total must equal threshold) or "min" (total must be at
|
||||
# least threshold).
|
||||
check_summary_line() {
|
||||
local label="$1"
|
||||
local threshold="$2"
|
||||
local compare="$3"
|
||||
local summary_line total accounted n
|
||||
|
||||
summary_line=$(echo "$CLEAN_LOG" | grep -E "^[[:space:]]*${label}[[:space:]]+" | tail -n 1 || true)
|
||||
|
||||
if [ -z "$summary_line" ]; then
|
||||
fatal "no '${label}' summary line found in ${LOG_FILE}"
|
||||
return 0
|
||||
fi
|
||||
|
||||
total=$(echo "$summary_line" | grep -oE '\([0-9]+\)' | tr -d '()' | tail -n 1 || true)
|
||||
if [ -z "$total" ]; then
|
||||
fatal "'${label}' summary line has no parenthesized total: \"${summary_line}\""
|
||||
return 0
|
||||
fi
|
||||
|
||||
if [ "$compare" = "eq" ]; then
|
||||
if [ "$total" -ne "$threshold" ]; then
|
||||
fatal "'${label}' total is ${total}, expected ${threshold}: \"${summary_line}\""
|
||||
else
|
||||
ok "'${label}' total matches expected ${threshold}"
|
||||
fi
|
||||
else
|
||||
if [ "$total" -lt "$threshold" ]; then
|
||||
fatal "'${label}' total is ${total}, below minimum ${threshold}: \"${summary_line}\""
|
||||
else
|
||||
ok "'${label}' total ${total} meets minimum ${threshold}"
|
||||
fi
|
||||
fi
|
||||
|
||||
accounted=0
|
||||
for n in $(echo "$summary_line" | grep -oE '[0-9]+ (passed|failed|skipped)' | grep -oE '^[0-9]+'); do
|
||||
accounted=$((accounted + n))
|
||||
done
|
||||
|
||||
if [ "$accounted" -lt "$total" ]; then
|
||||
fatal "'${label}' line accounts for only ${accounted} of ${total} — truncated run: \"${summary_line}\""
|
||||
else
|
||||
ok "'${label}' line accounts for all ${total}"
|
||||
fi
|
||||
|
||||
return 0
|
||||
}
|
||||
|
||||
echo "Brainy vitest verdict check: ${LOG_FILE}"
|
||||
echo "----------------------"
|
||||
|
||||
check_worker_death
|
||||
|
||||
if [ "$MODE" = "files" ]; then
|
||||
check_summary_line "Test Files" "$THRESHOLD" "eq"
|
||||
else
|
||||
check_summary_line "Tests" "$THRESHOLD" "min"
|
||||
fi
|
||||
|
||||
echo "----------------------"
|
||||
if [ "$VIOLATIONS" -gt 0 ]; then
|
||||
echo "FATAL: vitest verdict check failed with ${VIOLATIONS} violation(s) for ${LOG_FILE}"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "vitest verdict check passed for ${LOG_FILE}"
|
||||
exit 0
|
||||
Reference in a new issue