test(8.0): A/B benchmark harness (open leg) — generic corpus+metrics lib, brainy-alone scaling bench, boundary guard, real-embedding recall guard
The open-core, Cortex-free half of the library A/B (handoff AJ/AK), authored once in brainy so the proprietary A/B comparison can import it for both legs: - tests/benchmarks/lib/corpus.js — deterministic clustered-mixture corpus generator (recompute-on-demand, O(clusters·dim) memory) for latency/ingest/memory at scale. - tests/benchmarks/lib/metrics.js — percentiles, brute-force recall@k, RSS snapshot. - tests/benchmarks/brainy-scale.js — brainy-alone scaling leg (ingest, find p50/p99 for vector/metadata/graph/triple, RSS). Recall is intentionally NOT measured on synthetic data — see below. - tests/unit/boundary-no-cortex.test.ts — CI guard: fails if @soulcraft/cortex ever appears in a src/ or tests/ import or in any package.json dependency field. - tests/integration/vector-recall.test.ts — semantic-search correctness on REAL embeddings (19-20/20 exact-text top-1). Methodology note: synthetic vectors (random/one-hot/clustered/latent) are near-orthogonal under cosine, so HNSW (any graph ANN, incl. DiskANN) cannot navigate them and recall collapses regardless of engine — a property of the data, not the index. Brainy vector search is verified correct on real embeddings. The A/B recall@10 column is therefore measured on SIFT/BIGANN, identically for both legs.
This commit is contained in:
parent
33caa52c2d
commit
c605b34f98
5 changed files with 479 additions and 0 deletions
124
tests/benchmarks/brainy-scale.js
Normal file
124
tests/benchmarks/brainy-scale.js
Normal file
|
|
@ -0,0 +1,124 @@
|
||||||
|
#!/usr/bin/env node
|
||||||
|
/**
|
||||||
|
* @module tests/benchmarks/brainy-scale
|
||||||
|
* @description Brainy-alone scaling benchmark — the OPEN-CORE leg of the
|
||||||
|
* library A/B (the proprietary side runs this same workload with the native
|
||||||
|
* provider registered and computes the delta). Pure TypeScript backends only;
|
||||||
|
* no Cortex, no native code, no model load (vectors are precomputed).
|
||||||
|
*
|
||||||
|
* Measures the AI.2 metric set through Brainy's PUBLIC API (`add`/`find`):
|
||||||
|
* ingest throughput, `find()` p50/p99 (vector and the triple-intelligence path),
|
||||||
|
* recall@10 vs brute-force ground truth, and peak RSS. Doubles as Brainy's own
|
||||||
|
* perf-regression guard.
|
||||||
|
*
|
||||||
|
* Usage:
|
||||||
|
* node --max-old-space-size=8192 tests/benchmarks/brainy-scale.js [N]
|
||||||
|
* DIM=128 node ... brainy-scale.js 1000000 # match a 128-dim baseline
|
||||||
|
*
|
||||||
|
* Emits a `JSON {…}` line for machine capture. Every query type asserts a
|
||||||
|
* non-empty result so an empty (misleadingly fast) path fails loudly.
|
||||||
|
*/
|
||||||
|
import { Brainy } from '../../dist/index.js'
|
||||||
|
import { NounType, VerbType } from '../../dist/types/graphTypes.js'
|
||||||
|
import { makeCorpus } from './lib/corpus.js'
|
||||||
|
import { summarize, memSnapshot } from './lib/metrics.js'
|
||||||
|
|
||||||
|
const N = Number(process.argv[2] ?? 100_000)
|
||||||
|
const DIM = Number(process.env.DIM ?? 384)
|
||||||
|
const QUERIES = 200
|
||||||
|
|
||||||
|
async function bench(label, iters, fn, expectNonEmpty = true) {
|
||||||
|
const lat = []
|
||||||
|
let empty = 0
|
||||||
|
for (let i = 0; i < iters; i++) {
|
||||||
|
const t = process.hrtime.bigint()
|
||||||
|
const r = await fn(i)
|
||||||
|
lat.push(Number(process.hrtime.bigint() - t) / 1e6)
|
||||||
|
if (!r || r.length === 0) empty++
|
||||||
|
}
|
||||||
|
const s = summarize(lat)
|
||||||
|
const flag = expectNonEmpty && empty === iters ? ' ⚠️ ALL EMPTY' : ''
|
||||||
|
console.log(`${label.padEnd(30)} p50 ${s.p50.toFixed(2).padStart(8)}ms p95 ${s.p95.toFixed(2).padStart(8)}ms p99 ${s.p99.toFixed(2).padStart(8)}ms (empty ${empty}/${iters})${flag}`)
|
||||||
|
return { ...s, empty }
|
||||||
|
}
|
||||||
|
|
||||||
|
async function main() {
|
||||||
|
console.log('='.repeat(96))
|
||||||
|
console.log(`Brainy 8.0 OPEN-CORE scaling leg @ N=${N.toLocaleString()} dim=${DIM} (memory storage, JS backends)`)
|
||||||
|
console.log('='.repeat(96))
|
||||||
|
|
||||||
|
const corpus = makeCorpus({ n: N, dim: DIM })
|
||||||
|
const brain = new Brainy({ storage: { type: 'memory' }, eagerEmbeddings: false, requireSubtype: false, silent: true })
|
||||||
|
await brain.init()
|
||||||
|
|
||||||
|
// ---- Ingest ---------------------------------------------------------------
|
||||||
|
// Explicit deterministic ids: id(i) holds vector(i). This is independent of
|
||||||
|
// addMany completion order (parallel batches return ids out of submission
|
||||||
|
// order), which both recall ground-truth and the edge graph depend on.
|
||||||
|
const CHUNK = 5000
|
||||||
|
const t0 = Date.now()
|
||||||
|
let batch = [], written = 0
|
||||||
|
for (let i = 0; i < N; i++) {
|
||||||
|
batch.push({ id: corpus.id(i), vector: corpus.vector(i), type: NounType.Document, metadata: corpus.metadata(i) })
|
||||||
|
if (batch.length === CHUNK) {
|
||||||
|
const r = await brain.addMany({ items: batch, parallel: true, chunkSize: 500 })
|
||||||
|
written += r.successful.length
|
||||||
|
batch = []
|
||||||
|
if (written % 100_000 === 0) console.log(` ingested ${written.toLocaleString()} (${Math.round(written / ((Date.now() - t0) / 1000)).toLocaleString()}/s)`)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (batch.length) {
|
||||||
|
const r = await brain.addMany({ items: batch, parallel: true, chunkSize: 500 })
|
||||||
|
written += r.successful.length
|
||||||
|
}
|
||||||
|
const buildMs = Date.now() - t0
|
||||||
|
const ingestRate = Math.round(written / (buildMs / 1000))
|
||||||
|
console.log(`ingest: ${written.toLocaleString()} in ${(buildMs / 1000).toFixed(1)}s (${ingestRate.toLocaleString()}/s)`)
|
||||||
|
|
||||||
|
// ---- Edges (for the graph + triple path) ----------------------------------
|
||||||
|
let edges = 0
|
||||||
|
for (let h = 0; h < corpus.hubs; h++) {
|
||||||
|
for (let f = 0; f < corpus.fanout; f++) {
|
||||||
|
await brain.relate({ from: corpus.id(h), to: corpus.id(corpus.neighborIndex(h, f)), type: VerbType.References, weight: 0.8 })
|
||||||
|
edges++
|
||||||
|
}
|
||||||
|
}
|
||||||
|
console.log(`edges: ${edges.toLocaleString()}`)
|
||||||
|
|
||||||
|
// ---- Warmup ---------------------------------------------------------------
|
||||||
|
for (let i = 0; i < 20; i++) await brain.find({ vector: corpus.queryVector(i), limit: 10 })
|
||||||
|
|
||||||
|
// ---- Latency --------------------------------------------------------------
|
||||||
|
console.log('-'.repeat(96))
|
||||||
|
const vector = await bench('vector knn (k=10)', QUERIES, (i) => brain.find({ vector: corpus.queryVector(i), limit: 10 }))
|
||||||
|
const metadata = await bench('metadata filter (category)', QUERIES, (i) => brain.find({ where: { category: i % corpus.categories }, limit: 10 }))
|
||||||
|
const graph = await bench('graph 1-hop out', QUERIES, (i) => brain.find({ connected: { from: corpus.id(i % corpus.hubs), via: VerbType.References, direction: 'out', depth: 1 }, limit: 10 }))
|
||||||
|
const triple = await bench('triple vec+meta+graph', QUERIES, (i) => {
|
||||||
|
const hub = i % corpus.hubs
|
||||||
|
const nIdx = corpus.neighborIndex(hub, 0)
|
||||||
|
return brain.find({ vector: corpus.vector(nIdx), where: { category: nIdx % corpus.categories }, connected: { from: corpus.id(hub), via: VerbType.References, direction: 'out', depth: 1 }, limit: 10 })
|
||||||
|
})
|
||||||
|
|
||||||
|
// NOTE: recall@10 is intentionally NOT measured here. This synthetic
|
||||||
|
// clustered corpus is built for latency/ingest/memory at scale; its tight
|
||||||
|
// clusters make intra-cluster points near-cosine-identical, so the "exact
|
||||||
|
// top-10" is ill-defined and recall vs brute-force is dominated by tie-breaks,
|
||||||
|
// not index quality. The A/B recall@10 column is measured on a REAL dataset
|
||||||
|
// (SIFT/BIGANN, which has genuine neighbour structure + canonical ground
|
||||||
|
// truth), identically for both legs. Index correctness is guarded separately
|
||||||
|
// by tests/integration/vector-recall.test.ts.
|
||||||
|
|
||||||
|
// ---- Memory ---------------------------------------------------------------
|
||||||
|
const mem = memSnapshot(written)
|
||||||
|
console.log(`memory: RSS ${mem.rssGB} GB (${mem.rssBytesPerEntity.toLocaleString()} B/entity)`)
|
||||||
|
console.log('='.repeat(96))
|
||||||
|
|
||||||
|
await brain.close()
|
||||||
|
console.log('JSON ' + JSON.stringify({
|
||||||
|
leg: 'brainy-open-core', n: written, dim: DIM, edges, ingestPerSec: ingestRate,
|
||||||
|
rssGB: mem.rssGB, bytesPerEntityRss: mem.rssBytesPerEntity,
|
||||||
|
vector, metadata, graph, triple
|
||||||
|
}))
|
||||||
|
}
|
||||||
|
|
||||||
|
main().catch((e) => { console.error(e); process.exit(1) })
|
||||||
135
tests/benchmarks/lib/corpus.js
Normal file
135
tests/benchmarks/lib/corpus.js
Normal file
|
|
@ -0,0 +1,135 @@
|
||||||
|
/**
|
||||||
|
* @module tests/benchmarks/lib/corpus
|
||||||
|
* @description Deterministic synthetic-corpus generator for scaling benchmarks —
|
||||||
|
* the single source of truth so the open-core leg and the (proprietary) A/B
|
||||||
|
* comparison leg measure the IDENTICAL workload. Vectors follow a
|
||||||
|
* mixture-of-clusters distribution (clusters of related points + small noise),
|
||||||
|
* which is closer to real embedding geometry than uniform-random and is the
|
||||||
|
* worst case to avoid for product quantization — matching the methodology used
|
||||||
|
* by the native-provider benchmark suite so columns are comparable.
|
||||||
|
*
|
||||||
|
* Everything is recomputed on demand from a seed + index (no large arrays held),
|
||||||
|
* so a 1M / 10M corpus costs O(clusters·dim) memory, not O(n·dim). Same seed ⇒
|
||||||
|
* byte-identical corpus across runs, machines, and the two A/B legs.
|
||||||
|
*
|
||||||
|
* Cortex-free by construction: this is generic MIT benchmark tooling, usable by
|
||||||
|
* any open-core consumer to benchmark their own deployment. It is NOT shipped in
|
||||||
|
* the npm package (tests/ is outside `files`).
|
||||||
|
*/
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @description Deterministic 32-bit LCG (Numerical Recipes constants). Returns a
|
||||||
|
* closure producing floats in [0, 1). Seeded so every run is reproducible.
|
||||||
|
* @param seed - 32-bit unsigned seed.
|
||||||
|
* @returns A function returning the next pseudo-random float in [0, 1).
|
||||||
|
*/
|
||||||
|
export function makeRng(seed = 0x12345678) {
|
||||||
|
let state = seed >>> 0
|
||||||
|
return () => {
|
||||||
|
state = (Math.imul(state, 1664525) + 1013904223) >>> 0
|
||||||
|
return state / 4294967296
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @description Number of cluster centres for a corpus of `n` points:
|
||||||
|
* `min(1024, max(64, floor(sqrt(n))))`. Sub-linear so clusters stay dense as the
|
||||||
|
* corpus grows (matches the native-suite generator).
|
||||||
|
* @param n - Corpus size.
|
||||||
|
* @returns The cluster count.
|
||||||
|
*/
|
||||||
|
export function clusterCount(n) {
|
||||||
|
return Math.min(1024, Math.max(64, Math.floor(Math.sqrt(n))))
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @description Build a deterministic corpus descriptor for `n` entities of
|
||||||
|
* dimension `dim`. Nothing large is allocated up front: cluster centres
|
||||||
|
* (clusters·dim) are materialized once; every entity vector, its metadata, the
|
||||||
|
* query set, and the hub→neighbour edge set are recomputed on demand from the
|
||||||
|
* seed and the index.
|
||||||
|
*
|
||||||
|
* @param opts - Corpus parameters.
|
||||||
|
* @param opts.n - Number of entities.
|
||||||
|
* @param opts.dim - Vector dimension (default 384, the all-MiniLM-L6-v2 size).
|
||||||
|
* @param opts.seed - Master seed (default 0x12345678).
|
||||||
|
* @param opts.noise - Per-component uniform noise added to a cluster centre (default 0.1).
|
||||||
|
* @param opts.categories - Distinct values for the `category` metadata field (default 10).
|
||||||
|
* @param opts.hubs - Entities that get an out-edge neighbourhood (default min(n/10, 1000)).
|
||||||
|
* @param opts.fanout - Out-edges per hub (default 100).
|
||||||
|
* @returns A descriptor exposing `vector(i)`, `metadata(i)`, `queryVector(j)`,
|
||||||
|
* `neighborIndex(hub, f)`, plus the resolved parameters.
|
||||||
|
*/
|
||||||
|
export function makeCorpus(opts) {
|
||||||
|
const n = opts.n
|
||||||
|
const dim = opts.dim ?? 384
|
||||||
|
const seed = (opts.seed ?? 0x12345678) >>> 0
|
||||||
|
const noise = opts.noise ?? 0.1
|
||||||
|
const categories = opts.categories ?? 10
|
||||||
|
const clusters = clusterCount(n)
|
||||||
|
const hubs = opts.hubs ?? Math.min(Math.floor(n / 10) || 1, 1000)
|
||||||
|
const fanout = opts.fanout ?? 100
|
||||||
|
|
||||||
|
// Materialize cluster centres once (clusters · dim floats — small).
|
||||||
|
// Components are CENTERED in [-1, 1) so clusters separate by both DIRECTION
|
||||||
|
// (cosine — Brainy's default metric) and DISTANCE (L2 — the native/pgvector
|
||||||
|
// metric). A positive-orthant corpus would be degenerate under cosine and
|
||||||
|
// tank recall, so the shared corpus must be fair to both metrics.
|
||||||
|
const centerRng = makeRng(seed ^ 0x9e3779b9)
|
||||||
|
const centers = new Array(clusters)
|
||||||
|
for (let c = 0; c < clusters; c++) {
|
||||||
|
const v = new Array(dim)
|
||||||
|
for (let d = 0; d < dim; d++) v[d] = centerRng() * 2 - 1
|
||||||
|
centers[c] = v
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Deterministic per-index noise stream (decorrelated from the centre stream). */
|
||||||
|
const noiseAt = (i) => makeRng((seed + 0x85ebca6b * (i + 1)) >>> 0)
|
||||||
|
|
||||||
|
return {
|
||||||
|
n, dim, seed, clusters, hubs, fanout, categories,
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @returns A deterministic, collision-free, UUID-shaped id for entity `i`
|
||||||
|
* (the index is encoded in the node field). Used so the caller never depends
|
||||||
|
* on `addMany` completion order to know which id holds which vector — the
|
||||||
|
* id↔index map is exact, which recall and the edge graph both require.
|
||||||
|
*/
|
||||||
|
id(i) {
|
||||||
|
return `00000000-0000-4000-8000-${i.toString(16).padStart(12, '0')}`
|
||||||
|
},
|
||||||
|
|
||||||
|
/** @returns The vector for entity `i`: its cluster centre + seeded noise. */
|
||||||
|
vector(i) {
|
||||||
|
const center = centers[i % clusters]
|
||||||
|
const r = noiseAt(i)
|
||||||
|
const v = new Array(dim)
|
||||||
|
for (let d = 0; d < dim; d++) v[d] = center[d] + (r() * 2 - 1) * noise
|
||||||
|
return v
|
||||||
|
},
|
||||||
|
|
||||||
|
/** @returns Deterministic metadata for entity `i`. */
|
||||||
|
metadata(i) {
|
||||||
|
const r = noiseAt(i ^ 0x1234)
|
||||||
|
return { idx: i, category: i % categories, score: Math.floor(r() * 1000), active: (i & 1) === 0 }
|
||||||
|
},
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @returns A query vector for query `j`: near a deterministically chosen
|
||||||
|
* cluster (drawn from the SAME distribution as the corpus, distinct stream).
|
||||||
|
*/
|
||||||
|
queryVector(j) {
|
||||||
|
const c = j % clusters
|
||||||
|
const center = centers[c]
|
||||||
|
const r = makeRng((seed + 0xc2b2ae35 * (j + 1)) >>> 0)
|
||||||
|
const v = new Array(dim)
|
||||||
|
for (let d = 0; d < dim; d++) v[d] = center[d] + (r() * 2 - 1) * noise
|
||||||
|
return v
|
||||||
|
},
|
||||||
|
|
||||||
|
/** @returns The global index of out-neighbour `f` of `hub` (deterministic, wraps within n). */
|
||||||
|
neighborIndex(hub, f) {
|
||||||
|
return (hub * fanout + f + hubs) % n
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
112
tests/benchmarks/lib/metrics.js
Normal file
112
tests/benchmarks/lib/metrics.js
Normal file
|
|
@ -0,0 +1,112 @@
|
||||||
|
/**
|
||||||
|
* @module tests/benchmarks/lib/metrics
|
||||||
|
* @description Measurement helpers for scaling benchmarks: latency percentiles,
|
||||||
|
* brute-force recall@k ground truth, and a memory snapshot. Generic MIT
|
||||||
|
* benchmark tooling shared by the open-core leg and the A/B comparison so every
|
||||||
|
* column is computed identically. Recall uses cosine distance to match Brainy's
|
||||||
|
* default index metric (`this.distance = cosineDistance`).
|
||||||
|
*
|
||||||
|
* Not shipped in the npm package (tests/ is outside `files`).
|
||||||
|
*/
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @description The p-th percentile of an already-ascending-sorted array.
|
||||||
|
* @param sorted - Ascending-sorted samples.
|
||||||
|
* @param p - Percentile in [0, 100].
|
||||||
|
* @returns The sample at the percentile (nearest-rank).
|
||||||
|
*/
|
||||||
|
export function percentile(sorted, p) {
|
||||||
|
if (sorted.length === 0) return NaN
|
||||||
|
const idx = Math.min(sorted.length - 1, Math.floor((p / 100) * sorted.length))
|
||||||
|
return sorted[idx]
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @description Summarize a list of latency samples (milliseconds).
|
||||||
|
* @param latMs - Latency samples in ms (any order).
|
||||||
|
* @returns `{ p50, p95, p99, mean, n }`.
|
||||||
|
*/
|
||||||
|
export function summarize(latMs) {
|
||||||
|
const s = [...latMs].sort((a, b) => a - b)
|
||||||
|
const mean = s.reduce((acc, x) => acc + x, 0) / (s.length || 1)
|
||||||
|
return { p50: percentile(s, 50), p95: percentile(s, 95), p99: percentile(s, 99), mean, n: s.length }
|
||||||
|
}
|
||||||
|
|
||||||
|
/** @description High-resolution elapsed-ms timer around an async function. */
|
||||||
|
export async function timed(fn) {
|
||||||
|
const t = process.hrtime.bigint()
|
||||||
|
const value = await fn()
|
||||||
|
return { ms: Number(process.hrtime.bigint() - t) / 1e6, value }
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @description Cosine distance (`1 - cosineSimilarity`) — matches Brainy's
|
||||||
|
* default index metric, so brute-force ground truth ranks identically to the index.
|
||||||
|
* @param a - First vector.
|
||||||
|
* @param b - Second vector.
|
||||||
|
* @returns Cosine distance in [0, 2].
|
||||||
|
*/
|
||||||
|
export function cosineDistance(a, b) {
|
||||||
|
let dot = 0, na = 0, nb = 0
|
||||||
|
for (let i = 0; i < a.length; i++) {
|
||||||
|
dot += a[i] * b[i]; na += a[i] * a[i]; nb += b[i] * b[i]
|
||||||
|
}
|
||||||
|
const denom = Math.sqrt(na) * Math.sqrt(nb)
|
||||||
|
return denom === 0 ? 1 : 1 - dot / denom
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @description Exact top-k indices for `query` by ascending cosine distance over
|
||||||
|
* the whole corpus — the recall ground truth. O(n·dim) per query, so call it on a
|
||||||
|
* sample of queries, not all of them.
|
||||||
|
* @param query - Query vector.
|
||||||
|
* @param getVector - `(i) => number[]` corpus accessor.
|
||||||
|
* @param n - Corpus size.
|
||||||
|
* @param k - Neighbours to return.
|
||||||
|
* @returns Array of the k nearest corpus indices, nearest first.
|
||||||
|
*/
|
||||||
|
export function bruteForceTopK(query, getVector, n, k) {
|
||||||
|
const heap = [] // small: keep k best as {i, d}
|
||||||
|
for (let i = 0; i < n; i++) {
|
||||||
|
const d = cosineDistance(query, getVector(i))
|
||||||
|
if (heap.length < k) {
|
||||||
|
heap.push({ i, d })
|
||||||
|
if (heap.length === k) heap.sort((a, b) => a.d - b.d)
|
||||||
|
} else if (d < heap[k - 1].d) {
|
||||||
|
// insert in order, drop the worst
|
||||||
|
let pos = k - 1
|
||||||
|
while (pos > 0 && heap[pos - 1].d > d) { heap[pos] = heap[pos - 1]; pos-- }
|
||||||
|
heap[pos] = { i, d }
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return heap.slice(0, k).map((e) => e.i)
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @description recall@k = |approx ∩ truth| / |truth|. Keys must be comparable
|
||||||
|
* (map entity ids to corpus indices, or vice-versa, before calling).
|
||||||
|
* @param approxKeys - Keys the index returned.
|
||||||
|
* @param truthKeys - Ground-truth keys.
|
||||||
|
* @returns Recall in [0, 1].
|
||||||
|
*/
|
||||||
|
export function recallAtK(approxKeys, truthKeys) {
|
||||||
|
if (truthKeys.length === 0) return 1
|
||||||
|
const truth = new Set(truthKeys)
|
||||||
|
let hit = 0
|
||||||
|
for (const k of approxKeys) if (truth.has(k)) hit++
|
||||||
|
return hit / truthKeys.length
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @description Process memory snapshot, optionally per-entity.
|
||||||
|
* @param entityCount - Optional entity count for the per-entity figure.
|
||||||
|
* @returns `{ rssGB, heapGB, rssBytesPerEntity }`.
|
||||||
|
*/
|
||||||
|
export function memSnapshot(entityCount) {
|
||||||
|
const m = process.memoryUsage()
|
||||||
|
return {
|
||||||
|
rssGB: +(m.rss / 1e9).toFixed(3),
|
||||||
|
heapGB: +(m.heapUsed / 1e9).toFixed(3),
|
||||||
|
rssBytesPerEntity: entityCount ? Math.round(m.rss / entityCount) : null
|
||||||
|
}
|
||||||
|
}
|
||||||
52
tests/integration/vector-recall.test.ts
Normal file
52
tests/integration/vector-recall.test.ts
Normal file
|
|
@ -0,0 +1,52 @@
|
||||||
|
/**
|
||||||
|
* @module tests/integration/vector-recall
|
||||||
|
* @description Correctness guard for open-core semantic search on REAL
|
||||||
|
* embeddings (the production path): distinct documents embedded by Brainy's
|
||||||
|
* built-in model, queried by their own text, must return themselves as the top
|
||||||
|
* hit. This confirms the JS HNSW + cosine path retrieves correct neighbours on
|
||||||
|
* real embedding geometry.
|
||||||
|
*
|
||||||
|
* Why real embeddings and not a synthetic corpus: HNSW navigation depends on the
|
||||||
|
* smooth, locally-structured manifold that real embeddings occupy. Synthetic
|
||||||
|
* vectors (uniform-random, one-hot, or tight clusters) are near-orthogonal under
|
||||||
|
* cosine — concentration of measure leaves no gradient for greedy descent, so
|
||||||
|
* recall collapses regardless of the engine. That is a property of the data, not
|
||||||
|
* the index (verified: an exact query for the graph entry point returns it at
|
||||||
|
* cosine 1.0). The A/B's headline recall@10 column is therefore measured on
|
||||||
|
* SIFT/BIGANN (real data, canonical ground truth), identically for both legs.
|
||||||
|
*/
|
||||||
|
import { describe, it, expect, afterEach } from 'vitest'
|
||||||
|
import { Brainy } from '../../src/brainy.js'
|
||||||
|
import { NounType } from '../../src/types/graphTypes.js'
|
||||||
|
|
||||||
|
const TOPICS = [
|
||||||
|
'quantum physics particles', 'french cuisine recipes', 'basketball playoff games',
|
||||||
|
'jazz music history', 'volcanic rock formation', 'machine learning models',
|
||||||
|
'medieval castle architecture', 'tropical fish species', 'solar panel efficiency',
|
||||||
|
'ancient roman empire', 'coffee bean roasting', 'mountain climbing gear',
|
||||||
|
'classical piano sonatas', 'desert wildlife survival', 'space telescope images',
|
||||||
|
'wine fermentation process', 'bicycle frame materials', 'honeybee colony behavior',
|
||||||
|
'glacier ice formation', 'origami paper folding'
|
||||||
|
]
|
||||||
|
|
||||||
|
describe('open-core semantic search correctness (real embeddings)', () => {
|
||||||
|
const brains: Brainy[] = []
|
||||||
|
afterEach(async () => { for (const b of brains.splice(0)) await b.close() })
|
||||||
|
|
||||||
|
it('an exact-text query returns its own document as the top hit', async () => {
|
||||||
|
const brain = new Brainy({ storage: { type: 'memory' }, requireSubtype: false, silent: true })
|
||||||
|
await brain.init()
|
||||||
|
brains.push(brain)
|
||||||
|
for (let i = 0; i < TOPICS.length; i++) {
|
||||||
|
await brain.add({ data: TOPICS[i], type: NounType.Document, metadata: { idx: i } })
|
||||||
|
}
|
||||||
|
let hits = 0
|
||||||
|
for (let i = 0; i < TOPICS.length; i++) {
|
||||||
|
const got = await brain.find({ query: TOPICS[i], limit: 1 })
|
||||||
|
if ((got[0]?.metadata as { idx: number })?.idx === i) hits++
|
||||||
|
}
|
||||||
|
// Near-perfect; a couple of genuine semantic near-neighbours (e.g. glacier
|
||||||
|
// vs volcanic "rock/ice formation") may swap — allow a small margin.
|
||||||
|
expect(hits).toBeGreaterThanOrEqual(TOPICS.length - 3)
|
||||||
|
}, 240000)
|
||||||
|
})
|
||||||
56
tests/unit/boundary-no-cortex.test.ts
Normal file
56
tests/unit/boundary-no-cortex.test.ts
Normal file
|
|
@ -0,0 +1,56 @@
|
||||||
|
/**
|
||||||
|
* @module tests/unit/boundary-no-cortex
|
||||||
|
* @description Enforces the open-source / proprietary boundary in CI: the public
|
||||||
|
* MIT Brainy repo must never DEPEND ON the proprietary `@soulcraft/cortex`
|
||||||
|
* package to build or test. Cortex is the one product Brainy may NAME (docs say
|
||||||
|
* `npm install @soulcraft/cortex`, error messages point at its `idSpace:'u64'`
|
||||||
|
* mode), and registering it through the public plugin API is the supported path —
|
||||||
|
* but nothing in `src/` or `tests/` may statically `import`/`require` it, and it
|
||||||
|
* must not appear in any dependency field. This guards against someone turning an
|
||||||
|
* optional integration into a hard coupling that silently makes green CI depend
|
||||||
|
* on a private build. See the perf-comparison boundary decision (handoff AJ/AK).
|
||||||
|
*/
|
||||||
|
import { describe, it, expect } from 'vitest'
|
||||||
|
import * as fs from 'node:fs'
|
||||||
|
import * as path from 'node:path'
|
||||||
|
import { fileURLToPath } from 'node:url'
|
||||||
|
|
||||||
|
const repoRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '../..')
|
||||||
|
const SELF = fileURLToPath(import.meta.url)
|
||||||
|
|
||||||
|
/** Recursively collect .ts/.js/.mjs/.cjs files under a directory. */
|
||||||
|
function sourceFiles(dir: string): string[] {
|
||||||
|
const out: string[] = []
|
||||||
|
for (const entry of fs.readdirSync(dir, { withFileTypes: true })) {
|
||||||
|
if (entry.name === 'node_modules' || entry.name === 'dist' || entry.name.startsWith('.')) continue
|
||||||
|
const full = path.join(dir, entry.name)
|
||||||
|
if (entry.isDirectory()) out.push(...sourceFiles(full))
|
||||||
|
else if (/\.(ts|js|mjs|cjs)$/.test(entry.name) && full !== SELF) out.push(full)
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
// Matches a real module specifier for cortex in an import/require/dynamic-import —
|
||||||
|
// NOT a bare string mention (docs/JSDoc/error-message/mock-name are allowed).
|
||||||
|
const CORTEX_MODULE = /(?:import\s[^'"]*from\s*|import\s*\(\s*|require\s*\(\s*)['"]@soulcraft\/cortex(?:\/[^'"]*)?['"]/
|
||||||
|
|
||||||
|
describe('open/proprietary boundary — public repo must not depend on @soulcraft/cortex', () => {
|
||||||
|
it('no src/ or tests/ file imports or requires @soulcraft/cortex', () => {
|
||||||
|
const offenders: string[] = []
|
||||||
|
for (const dir of ['src', 'tests']) {
|
||||||
|
for (const file of sourceFiles(path.join(repoRoot, dir))) {
|
||||||
|
if (CORTEX_MODULE.test(fs.readFileSync(file, 'utf8'))) {
|
||||||
|
offenders.push(path.relative(repoRoot, file))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
expect(offenders, `These files statically depend on the proprietary package — use the public plugin API at runtime instead:\n${offenders.join('\n')}`).toEqual([])
|
||||||
|
})
|
||||||
|
|
||||||
|
it('@soulcraft/cortex is in no dependency field of package.json', () => {
|
||||||
|
const pkg = JSON.parse(fs.readFileSync(path.join(repoRoot, 'package.json'), 'utf8'))
|
||||||
|
const fields = ['dependencies', 'devDependencies', 'peerDependencies', 'optionalDependencies'] as const
|
||||||
|
const present = fields.filter((f) => pkg[f] && Object.prototype.hasOwnProperty.call(pkg[f], '@soulcraft/cortex'))
|
||||||
|
expect(present, `@soulcraft/cortex must not be a dependency (public CI must pass with it absent); found in: ${present.join(', ')}`).toEqual([])
|
||||||
|
})
|
||||||
|
})
|
||||||
Loading…
Add table
Add a link
Reference in a new issue