test(8.0): A/B benchmark harness (open leg) — generic corpus+metrics lib, brainy-alone scaling bench, boundary guard, real-embedding recall guard

The open-core, Cortex-free half of the library A/B (handoff AJ/AK), authored once in
brainy so the proprietary A/B comparison can import it for both legs:

- tests/benchmarks/lib/corpus.js — deterministic clustered-mixture corpus generator
  (recompute-on-demand, O(clusters·dim) memory) for latency/ingest/memory at scale.
- tests/benchmarks/lib/metrics.js — percentiles, brute-force recall@k, RSS snapshot.
- tests/benchmarks/brainy-scale.js — brainy-alone scaling leg (ingest, find p50/p99 for
  vector/metadata/graph/triple, RSS). Recall is intentionally NOT measured on synthetic
  data — see below.
- tests/unit/boundary-no-cortex.test.ts — CI guard: fails if @soulcraft/cortex ever
  appears in a src/ or tests/ import or in any package.json dependency field.
- tests/integration/vector-recall.test.ts — semantic-search correctness on REAL
  embeddings (19-20/20 exact-text top-1).

Methodology note: synthetic vectors (random/one-hot/clustered/latent) are near-orthogonal
under cosine, so HNSW (any graph ANN, incl. DiskANN) cannot navigate them and recall
collapses regardless of engine — a property of the data, not the index. Brainy vector
search is verified correct on real embeddings. The A/B recall@10 column is therefore
measured on SIFT/BIGANN, identically for both legs.
This commit is contained in:
David Snelling 2026-06-15 15:51:17 -07:00
parent 33caa52c2d
commit c605b34f98
5 changed files with 479 additions and 0 deletions

View file

@ -0,0 +1,56 @@
/**
* @module tests/unit/boundary-no-cortex
* @description Enforces the open-source / proprietary boundary in CI: the public
* MIT Brainy repo must never DEPEND ON the proprietary `@soulcraft/cortex`
* package to build or test. Cortex is the one product Brainy may NAME (docs say
* `npm install @soulcraft/cortex`, error messages point at its `idSpace:'u64'`
* mode), and registering it through the public plugin API is the supported path
* but nothing in `src/` or `tests/` may statically `import`/`require` it, and it
* must not appear in any dependency field. This guards against someone turning an
* optional integration into a hard coupling that silently makes green CI depend
* on a private build. See the perf-comparison boundary decision (handoff AJ/AK).
*/
import { describe, it, expect } from 'vitest'
import * as fs from 'node:fs'
import * as path from 'node:path'
import { fileURLToPath } from 'node:url'
const repoRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '../..')
const SELF = fileURLToPath(import.meta.url)
/** Recursively collect .ts/.js/.mjs/.cjs files under a directory. */
function sourceFiles(dir: string): string[] {
const out: string[] = []
for (const entry of fs.readdirSync(dir, { withFileTypes: true })) {
if (entry.name === 'node_modules' || entry.name === 'dist' || entry.name.startsWith('.')) continue
const full = path.join(dir, entry.name)
if (entry.isDirectory()) out.push(...sourceFiles(full))
else if (/\.(ts|js|mjs|cjs)$/.test(entry.name) && full !== SELF) out.push(full)
}
return out
}
// Matches a real module specifier for cortex in an import/require/dynamic-import —
// NOT a bare string mention (docs/JSDoc/error-message/mock-name are allowed).
const CORTEX_MODULE = /(?:import\s[^'"]*from\s*|import\s*\(\s*|require\s*\(\s*)['"]@soulcraft\/cortex(?:\/[^'"]*)?['"]/
describe('open/proprietary boundary — public repo must not depend on @soulcraft/cortex', () => {
it('no src/ or tests/ file imports or requires @soulcraft/cortex', () => {
const offenders: string[] = []
for (const dir of ['src', 'tests']) {
for (const file of sourceFiles(path.join(repoRoot, dir))) {
if (CORTEX_MODULE.test(fs.readFileSync(file, 'utf8'))) {
offenders.push(path.relative(repoRoot, file))
}
}
}
expect(offenders, `These files statically depend on the proprietary package — use the public plugin API at runtime instead:\n${offenders.join('\n')}`).toEqual([])
})
it('@soulcraft/cortex is in no dependency field of package.json', () => {
const pkg = JSON.parse(fs.readFileSync(path.join(repoRoot, 'package.json'), 'utf8'))
const fields = ['dependencies', 'devDependencies', 'peerDependencies', 'optionalDependencies'] as const
const present = fields.filter((f) => pkg[f] && Object.prototype.hasOwnProperty.call(pkg[f], '@soulcraft/cortex'))
expect(present, `@soulcraft/cortex must not be a dependency (public CI must pass with it absent); found in: ${present.join(', ')}`).toEqual([])
})
})