test(8.0): A/B benchmark harness (open leg) — generic corpus+metrics lib, brainy-alone scaling bench, boundary guard, real-embedding recall guard
The open-core, Cortex-free half of the library A/B (handoff AJ/AK), authored once in brainy so the proprietary A/B comparison can import it for both legs: - tests/benchmarks/lib/corpus.js — deterministic clustered-mixture corpus generator (recompute-on-demand, O(clusters·dim) memory) for latency/ingest/memory at scale. - tests/benchmarks/lib/metrics.js — percentiles, brute-force recall@k, RSS snapshot. - tests/benchmarks/brainy-scale.js — brainy-alone scaling leg (ingest, find p50/p99 for vector/metadata/graph/triple, RSS). Recall is intentionally NOT measured on synthetic data — see below. - tests/unit/boundary-no-cortex.test.ts — CI guard: fails if @soulcraft/cortex ever appears in a src/ or tests/ import or in any package.json dependency field. - tests/integration/vector-recall.test.ts — semantic-search correctness on REAL embeddings (19-20/20 exact-text top-1). Methodology note: synthetic vectors (random/one-hot/clustered/latent) are near-orthogonal under cosine, so HNSW (any graph ANN, incl. DiskANN) cannot navigate them and recall collapses regardless of engine — a property of the data, not the index. Brainy vector search is verified correct on real embeddings. The A/B recall@10 column is therefore measured on SIFT/BIGANN, identically for both legs.
This commit is contained in:
parent
33caa52c2d
commit
c605b34f98
5 changed files with 479 additions and 0 deletions
112
tests/benchmarks/lib/metrics.js
Normal file
112
tests/benchmarks/lib/metrics.js
Normal file
|
|
@ -0,0 +1,112 @@
|
|||
/**
|
||||
* @module tests/benchmarks/lib/metrics
|
||||
* @description Measurement helpers for scaling benchmarks: latency percentiles,
|
||||
* brute-force recall@k ground truth, and a memory snapshot. Generic MIT
|
||||
* benchmark tooling shared by the open-core leg and the A/B comparison so every
|
||||
* column is computed identically. Recall uses cosine distance to match Brainy's
|
||||
* default index metric (`this.distance = cosineDistance`).
|
||||
*
|
||||
* Not shipped in the npm package (tests/ is outside `files`).
|
||||
*/
|
||||
|
||||
/**
|
||||
* @description The p-th percentile of an already-ascending-sorted array.
|
||||
* @param sorted - Ascending-sorted samples.
|
||||
* @param p - Percentile in [0, 100].
|
||||
* @returns The sample at the percentile (nearest-rank).
|
||||
*/
|
||||
export function percentile(sorted, p) {
|
||||
if (sorted.length === 0) return NaN
|
||||
const idx = Math.min(sorted.length - 1, Math.floor((p / 100) * sorted.length))
|
||||
return sorted[idx]
|
||||
}
|
||||
|
||||
/**
|
||||
* @description Summarize a list of latency samples (milliseconds).
|
||||
* @param latMs - Latency samples in ms (any order).
|
||||
* @returns `{ p50, p95, p99, mean, n }`.
|
||||
*/
|
||||
export function summarize(latMs) {
|
||||
const s = [...latMs].sort((a, b) => a - b)
|
||||
const mean = s.reduce((acc, x) => acc + x, 0) / (s.length || 1)
|
||||
return { p50: percentile(s, 50), p95: percentile(s, 95), p99: percentile(s, 99), mean, n: s.length }
|
||||
}
|
||||
|
||||
/** @description High-resolution elapsed-ms timer around an async function. */
|
||||
export async function timed(fn) {
|
||||
const t = process.hrtime.bigint()
|
||||
const value = await fn()
|
||||
return { ms: Number(process.hrtime.bigint() - t) / 1e6, value }
|
||||
}
|
||||
|
||||
/**
|
||||
* @description Cosine distance (`1 - cosineSimilarity`) — matches Brainy's
|
||||
* default index metric, so brute-force ground truth ranks identically to the index.
|
||||
* @param a - First vector.
|
||||
* @param b - Second vector.
|
||||
* @returns Cosine distance in [0, 2].
|
||||
*/
|
||||
export function cosineDistance(a, b) {
|
||||
let dot = 0, na = 0, nb = 0
|
||||
for (let i = 0; i < a.length; i++) {
|
||||
dot += a[i] * b[i]; na += a[i] * a[i]; nb += b[i] * b[i]
|
||||
}
|
||||
const denom = Math.sqrt(na) * Math.sqrt(nb)
|
||||
return denom === 0 ? 1 : 1 - dot / denom
|
||||
}
|
||||
|
||||
/**
|
||||
* @description Exact top-k indices for `query` by ascending cosine distance over
|
||||
* the whole corpus — the recall ground truth. O(n·dim) per query, so call it on a
|
||||
* sample of queries, not all of them.
|
||||
* @param query - Query vector.
|
||||
* @param getVector - `(i) => number[]` corpus accessor.
|
||||
* @param n - Corpus size.
|
||||
* @param k - Neighbours to return.
|
||||
* @returns Array of the k nearest corpus indices, nearest first.
|
||||
*/
|
||||
export function bruteForceTopK(query, getVector, n, k) {
|
||||
const heap = [] // small: keep k best as {i, d}
|
||||
for (let i = 0; i < n; i++) {
|
||||
const d = cosineDistance(query, getVector(i))
|
||||
if (heap.length < k) {
|
||||
heap.push({ i, d })
|
||||
if (heap.length === k) heap.sort((a, b) => a.d - b.d)
|
||||
} else if (d < heap[k - 1].d) {
|
||||
// insert in order, drop the worst
|
||||
let pos = k - 1
|
||||
while (pos > 0 && heap[pos - 1].d > d) { heap[pos] = heap[pos - 1]; pos-- }
|
||||
heap[pos] = { i, d }
|
||||
}
|
||||
}
|
||||
return heap.slice(0, k).map((e) => e.i)
|
||||
}
|
||||
|
||||
/**
|
||||
* @description recall@k = |approx ∩ truth| / |truth|. Keys must be comparable
|
||||
* (map entity ids to corpus indices, or vice-versa, before calling).
|
||||
* @param approxKeys - Keys the index returned.
|
||||
* @param truthKeys - Ground-truth keys.
|
||||
* @returns Recall in [0, 1].
|
||||
*/
|
||||
export function recallAtK(approxKeys, truthKeys) {
|
||||
if (truthKeys.length === 0) return 1
|
||||
const truth = new Set(truthKeys)
|
||||
let hit = 0
|
||||
for (const k of approxKeys) if (truth.has(k)) hit++
|
||||
return hit / truthKeys.length
|
||||
}
|
||||
|
||||
/**
|
||||
* @description Process memory snapshot, optionally per-entity.
|
||||
* @param entityCount - Optional entity count for the per-entity figure.
|
||||
* @returns `{ rssGB, heapGB, rssBytesPerEntity }`.
|
||||
*/
|
||||
export function memSnapshot(entityCount) {
|
||||
const m = process.memoryUsage()
|
||||
return {
|
||||
rssGB: +(m.rss / 1e9).toFixed(3),
|
||||
heapGB: +(m.heapUsed / 1e9).toFixed(3),
|
||||
rssBytesPerEntity: entityCount ? Math.round(m.rss / entityCount) : null
|
||||
}
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue