diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 54d4f784..c58520b7 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -41,6 +41,20 @@ npm test Tests run on [Vitest](https://vitest.dev/). `npm test` runs the unit suite; see `package.json` for `test:integration`, `test:coverage`, and friends. +## Test gate + +The release gate is a bare `vitest run` (no `--config` flag) — the same +command the delta gate and CI's checks invoke. It carries the full +correctness suite and nothing else: wall-clock/scale benchmarks +(`tests/performance/**`, `tests/critical-performance-benchmark.test.ts`, +`tests/api/performance-benchmarks.test.ts`) and the two tests whose outcome +depends on the host machine or network rather than the code +(`tests/package-size-limit.test.ts` shells out to the `npm` CLI; +`tests/model-loading.test.ts` makes a real network call to download a model) +are excluded from it, because a timing threshold or a flaky network call has +no business failing a correctness check. That whole family runs on demand, +in its own exclusive slot, via `npm run test:perf`. + ## Standards - **Strict TypeScript.** No `any` escape hatches to dodge the type checker. diff --git a/package.json b/package.json index f07bb94c..f5a0325d 100644 --- a/package.json +++ b/package.json @@ -88,7 +88,7 @@ "test:watch": "NODE_OPTIONS='--max-old-space-size=8192' vitest --config tests/configs/vitest.unit.config.ts", "test:coverage": "NODE_OPTIONS='--max-old-space-size=8192' vitest run --config tests/configs/vitest.unit.config.ts --coverage", "test:unit": "NODE_OPTIONS='--max-old-space-size=8192' vitest run --config tests/configs/vitest.unit.config.ts", - "test:perf": "vitest run tests/unit/performance --reporter=basic", + "test:perf": "vitest run --config tests/configs/vitest.perf.config.ts", "test:integration": "NODE_OPTIONS='--max-old-space-size=8192' vitest run --config tests/configs/vitest.integration.config.ts", "test:semantic": "NODE_OPTIONS='--max-old-space-size=8192' vitest run --config tests/configs/vitest.semantic.config.ts", "test:all": "npm run test:unit && npm run test:integration", diff --git a/tests/configs/vitest.perf.config.ts b/tests/configs/vitest.perf.config.ts new file mode 100644 index 00000000..ca665dae --- /dev/null +++ b/tests/configs/vitest.perf.config.ts @@ -0,0 +1,56 @@ +import { defineConfig } from 'vitest/config' + +/** + * Perf/scale + environment-dependent test configuration. + * + * The exclusive on-demand slot for everything the correctness gate + * (`vitest.config.ts`, the config a bare `vitest run` picks up) excludes: + * wall-clock/scale benchmarks and the two tests whose outcome depends on + * the host machine or network rather than the code. See CONTRIBUTING.md's + * "Test gate" section and the exclude list in `vitest.config.ts` (root) for + * why each file lives here instead of the gate. + * + * `include` names this set explicitly — it is the mirror image of the + * root config's exclude list, not an independent glob, so the two stay in + * sync by inspection. Longer timeouts than the gate's 120s/60s: one case in + * tests/critical-performance-benchmark.test.ts measures ~128s of real work. + */ +export default defineConfig({ + test: { + globals: true, + setupFiles: ['./tests/setup.ts'], + environment: 'node', + + // Sequential, single fork — same isolation the gate uses, so a perf + // measurement isn't skewed by sibling test contention. + pool: 'forks', + poolOptions: { + forks: { + maxForks: 1, + minForks: 1, + singleFork: true, + isolate: true + } + }, + + testTimeout: 300000, // 5 minutes per test (the 128s case plus headroom) + hookTimeout: 120000, + teardownTimeout: 10000, + + maxConcurrency: 1, + fileParallelism: false, + + include: [ + 'tests/performance/**/*.{test,spec}.{js,ts}', + 'tests/critical-performance-benchmark.test.ts', + 'tests/api/performance-benchmarks.test.ts', + 'tests/package-size-limit.test.ts', + 'tests/model-loading.test.ts' + ], + + reporters: process.env.CI ? ['dot'] : ['basic'], + + retry: process.env.CI ? 1 : 0, + shard: process.env.VITEST_SHARD + } +}) diff --git a/vitest.config.ts b/vitest.config.ts index 116ab234..013c3c9b 100644 --- a/vitest.config.ts +++ b/vitest.config.ts @@ -2,9 +2,16 @@ import { defineConfig } from 'vitest/config' /** * Vitest Configuration - Optimized for Memory-Intensive Tests - * + * * Handles ONNX transformer model testing (4-8GB memory requirement) * Based on 2024-2025 best practices + * + * THE CORRECTNESS GATE: this is the config a bare `vitest run` (no + * `--config` flag) picks up — the delta gate and CI both invoke it that + * way. See CONTRIBUTING.md's "Test gate" section for the full picture. + * Wall-clock/scale benchmarks and tests whose outcome depends on the host + * machine or network rather than the code are excluded below and run on + * demand instead, in their own slot: `npm run test:perf`. */ export default defineConfig({ test: { @@ -38,7 +45,29 @@ export default defineConfig({ 'node_modules/**', 'dist/**', 'scripts/**', - '**/*.browser.test.ts' + '**/*.browser.test.ts', + + // Wall-clock/scale benchmark family — timing assertions and scale + // sweeps whose pass/fail depends on the host machine's speed, not on + // the code. Whole files only (a file that mixes correctness describes + // with a perf describe stays in the gate). Run on demand via + // `npm run test:perf`, which targets exactly this list. + 'tests/performance/**', + 'tests/critical-performance-benchmark.test.ts', + 'tests/api/performance-benchmarks.test.ts', + + // Environment-dependent by construction, not timing-based: + // package-size-limit shells out to the `npm` CLI (not guaranteed + // present — the functional gate lane is Bun-only host-mode with no + // Node.js runtime) and parses npm-version-specific `npm pack` notice + // text; model-loading's "Real Model Download Integration" case makes + // a genuine, unmocked network call to HuggingFace (its own header + // says "Uses REAL transformer models - NO MOCKING"), and the whole + // file imports `../src/embeddings/model-manager.js`, which no longer + // exists anywhere under src/ — neither belongs in a gate that must be + // deterministic. + 'tests/package-size-limit.test.ts', + 'tests/model-loading.test.ts' ], // REPORTERS: Dot for CI, verbose for local