test(8.0): A/B benchmark harness (open leg) — generic corpus+metrics lib, brainy-alone scaling bench, boundary guard, real-embedding recall guard
The open-core, Cortex-free half of the library A/B (handoff AJ/AK), authored once in brainy so the proprietary A/B comparison can import it for both legs: - tests/benchmarks/lib/corpus.js — deterministic clustered-mixture corpus generator (recompute-on-demand, O(clusters·dim) memory) for latency/ingest/memory at scale. - tests/benchmarks/lib/metrics.js — percentiles, brute-force recall@k, RSS snapshot. - tests/benchmarks/brainy-scale.js — brainy-alone scaling leg (ingest, find p50/p99 for vector/metadata/graph/triple, RSS). Recall is intentionally NOT measured on synthetic data — see below. - tests/unit/boundary-no-cortex.test.ts — CI guard: fails if @soulcraft/cortex ever appears in a src/ or tests/ import or in any package.json dependency field. - tests/integration/vector-recall.test.ts — semantic-search correctness on REAL embeddings (19-20/20 exact-text top-1). Methodology note: synthetic vectors (random/one-hot/clustered/latent) are near-orthogonal under cosine, so HNSW (any graph ANN, incl. DiskANN) cannot navigate them and recall collapses regardless of engine — a property of the data, not the index. Brainy vector search is verified correct on real embeddings. The A/B recall@10 column is therefore measured on SIFT/BIGANN, identically for both legs.
This commit is contained in:
parent
33caa52c2d
commit
c605b34f98
5 changed files with 479 additions and 0 deletions
56
tests/unit/boundary-no-cortex.test.ts
Normal file
56
tests/unit/boundary-no-cortex.test.ts
Normal file
|
|
@ -0,0 +1,56 @@
|
|||
/**
|
||||
* @module tests/unit/boundary-no-cortex
|
||||
* @description Enforces the open-source / proprietary boundary in CI: the public
|
||||
* MIT Brainy repo must never DEPEND ON the proprietary `@soulcraft/cortex`
|
||||
* package to build or test. Cortex is the one product Brainy may NAME (docs say
|
||||
* `npm install @soulcraft/cortex`, error messages point at its `idSpace:'u64'`
|
||||
* mode), and registering it through the public plugin API is the supported path —
|
||||
* but nothing in `src/` or `tests/` may statically `import`/`require` it, and it
|
||||
* must not appear in any dependency field. This guards against someone turning an
|
||||
* optional integration into a hard coupling that silently makes green CI depend
|
||||
* on a private build. See the perf-comparison boundary decision (handoff AJ/AK).
|
||||
*/
|
||||
import { describe, it, expect } from 'vitest'
|
||||
import * as fs from 'node:fs'
|
||||
import * as path from 'node:path'
|
||||
import { fileURLToPath } from 'node:url'
|
||||
|
||||
const repoRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '../..')
|
||||
const SELF = fileURLToPath(import.meta.url)
|
||||
|
||||
/** Recursively collect .ts/.js/.mjs/.cjs files under a directory. */
|
||||
function sourceFiles(dir: string): string[] {
|
||||
const out: string[] = []
|
||||
for (const entry of fs.readdirSync(dir, { withFileTypes: true })) {
|
||||
if (entry.name === 'node_modules' || entry.name === 'dist' || entry.name.startsWith('.')) continue
|
||||
const full = path.join(dir, entry.name)
|
||||
if (entry.isDirectory()) out.push(...sourceFiles(full))
|
||||
else if (/\.(ts|js|mjs|cjs)$/.test(entry.name) && full !== SELF) out.push(full)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// Matches a real module specifier for cortex in an import/require/dynamic-import —
|
||||
// NOT a bare string mention (docs/JSDoc/error-message/mock-name are allowed).
|
||||
const CORTEX_MODULE = /(?:import\s[^'"]*from\s*|import\s*\(\s*|require\s*\(\s*)['"]@soulcraft\/cortex(?:\/[^'"]*)?['"]/
|
||||
|
||||
describe('open/proprietary boundary — public repo must not depend on @soulcraft/cortex', () => {
|
||||
it('no src/ or tests/ file imports or requires @soulcraft/cortex', () => {
|
||||
const offenders: string[] = []
|
||||
for (const dir of ['src', 'tests']) {
|
||||
for (const file of sourceFiles(path.join(repoRoot, dir))) {
|
||||
if (CORTEX_MODULE.test(fs.readFileSync(file, 'utf8'))) {
|
||||
offenders.push(path.relative(repoRoot, file))
|
||||
}
|
||||
}
|
||||
}
|
||||
expect(offenders, `These files statically depend on the proprietary package — use the public plugin API at runtime instead:\n${offenders.join('\n')}`).toEqual([])
|
||||
})
|
||||
|
||||
it('@soulcraft/cortex is in no dependency field of package.json', () => {
|
||||
const pkg = JSON.parse(fs.readFileSync(path.join(repoRoot, 'package.json'), 'utf8'))
|
||||
const fields = ['dependencies', 'devDependencies', 'peerDependencies', 'optionalDependencies'] as const
|
||||
const present = fields.filter((f) => pkg[f] && Object.prototype.hasOwnProperty.call(pkg[f], '@soulcraft/cortex'))
|
||||
expect(present, `@soulcraft/cortex must not be a dependency (public CI must pass with it absent); found in: ${present.join(', ')}`).toEqual([])
|
||||
})
|
||||
})
|
||||
Loading…
Add table
Add a link
Reference in a new issue