brainy/tests/benchmarks/brainy-scale.js

#!/usr/bin/env node
/**
 * @module tests/benchmarks/brainy-scale
 * @description Brainy-alone scaling benchmark — the OPEN-CORE leg of the
 * library A/B (the proprietary side runs this same workload with the native
 * provider registered and computes the delta). Pure TypeScript backends only;
 * no Cortex, no native code, no model load (vectors are precomputed).
 *
 * Measures the AI.2 metric set through Brainy's PUBLIC API (`add`/`find`):
 * ingest throughput, `find()` p50/p99 (vector and the triple-intelligence path),
 * recall@10 vs brute-force ground truth, and peak RSS. Doubles as Brainy's own
 * perf-regression guard.
 *
 * Usage:
 *   node --max-old-space-size=8192 tests/benchmarks/brainy-scale.js [N]
 *   DIM=128 node ... brainy-scale.js 1000000     # match a 128-dim baseline
 *
 * Emits a `JSON {…}` line for machine capture. Every query type asserts a
 * non-empty result so an empty (misleadingly fast) path fails loudly.
 */
import { Brainy } from '../../dist/index.js'
import { NounType, VerbType } from '../../dist/types/graphTypes.js'
import { makeCorpus } from './lib/corpus.js'
import { summarize, memSnapshot } from './lib/metrics.js'

const N = Number(process.argv[2] ?? 100_000)
const DIM = Number(process.env.DIM ?? 384)
const QUERIES = 200

async function bench(label, iters, fn, expectNonEmpty = true) {
  const lat = []
  let empty = 0
  for (let i = 0; i < iters; i++) {
    const t = process.hrtime.bigint()
    const r = await fn(i)
    lat.push(Number(process.hrtime.bigint() - t) / 1e6)
    if (!r || r.length === 0) empty++
  }
  const s = summarize(lat)
  const flag = expectNonEmpty && empty === iters ? '  ⚠️ ALL EMPTY' : ''
  console.log(`${label.padEnd(30)} p50 ${s.p50.toFixed(2).padStart(8)}ms  p95 ${s.p95.toFixed(2).padStart(8)}ms  p99 ${s.p99.toFixed(2).padStart(8)}ms  (empty ${empty}/${iters})${flag}`)
  return { ...s, empty }
}

async function main() {
  console.log('='.repeat(96))
  console.log(`Brainy 8.0 OPEN-CORE scaling leg @ N=${N.toLocaleString()} dim=${DIM} (memory storage, JS backends)`)
  console.log('='.repeat(96))

  const corpus = makeCorpus({ n: N, dim: DIM })
  const brain = new Brainy({ storage: { type: 'memory' }, eagerEmbeddings: false, requireSubtype: false, silent: true })
  await brain.init()

  // ---- Ingest ---------------------------------------------------------------
  // Explicit deterministic ids: id(i) holds vector(i). This is independent of
  // addMany completion order (parallel batches return ids out of submission
  // order), which both recall ground-truth and the edge graph depend on.
  const CHUNK = 5000
  const t0 = Date.now()
  let batch = [], written = 0
  for (let i = 0; i < N; i++) {
    batch.push({ id: corpus.id(i), vector: corpus.vector(i), type: NounType.Document, metadata: corpus.metadata(i) })
    if (batch.length === CHUNK) {
      const r = await brain.addMany({ items: batch, parallel: true, chunkSize: 500 })
      written += r.successful.length
      batch = []
      if (written % 100_000 === 0) console.log(`  ingested ${written.toLocaleString()} (${Math.round(written / ((Date.now() - t0) / 1000)).toLocaleString()}/s)`)
    }
  }
  if (batch.length) {
    const r = await brain.addMany({ items: batch, parallel: true, chunkSize: 500 })
    written += r.successful.length
  }
  const buildMs = Date.now() - t0
  const ingestRate = Math.round(written / (buildMs / 1000))
  console.log(`ingest: ${written.toLocaleString()} in ${(buildMs / 1000).toFixed(1)}s (${ingestRate.toLocaleString()}/s)`)

  // ---- Edges (for the graph + triple path) ----------------------------------
  let edges = 0
  for (let h = 0; h < corpus.hubs; h++) {
    for (let f = 0; f < corpus.fanout; f++) {
      await brain.relate({ from: corpus.id(h), to: corpus.id(corpus.neighborIndex(h, f)), type: VerbType.References, weight: 0.8 })
      edges++
    }
  }
  console.log(`edges: ${edges.toLocaleString()}`)

  // ---- Warmup ---------------------------------------------------------------
  for (let i = 0; i < 20; i++) await brain.find({ vector: corpus.queryVector(i), limit: 10 })

  // ---- Latency --------------------------------------------------------------
  console.log('-'.repeat(96))
  const vector = await bench('vector knn (k=10)', QUERIES, (i) => brain.find({ vector: corpus.queryVector(i), limit: 10 }))
  const metadata = await bench('metadata filter (category)', QUERIES, (i) => brain.find({ where: { category: i % corpus.categories }, limit: 10 }))
  const graph = await bench('graph 1-hop out', QUERIES, (i) => brain.find({ connected: { from: corpus.id(i % corpus.hubs), via: VerbType.References, direction: 'out', depth: 1 }, limit: 10 }))
  const triple = await bench('triple vec+meta+graph', QUERIES, (i) => {
    const hub = i % corpus.hubs
    const nIdx = corpus.neighborIndex(hub, 0)
    return brain.find({ vector: corpus.vector(nIdx), where: { category: nIdx % corpus.categories }, connected: { from: corpus.id(hub), via: VerbType.References, direction: 'out', depth: 1 }, limit: 10 })
  })

  // NOTE: recall@10 is intentionally NOT measured here. This synthetic
  // clustered corpus is built for latency/ingest/memory at scale; its tight
  // clusters make intra-cluster points near-cosine-identical, so the "exact
  // top-10" is ill-defined and recall vs brute-force is dominated by tie-breaks,
  // not index quality. The A/B recall@10 column is measured on a REAL dataset
  // (SIFT/BIGANN, which has genuine neighbour structure + canonical ground
  // truth), identically for both legs. Index correctness is guarded separately
  // by tests/integration/vector-recall.test.ts.

  // ---- Memory ---------------------------------------------------------------
  const mem = memSnapshot(written)
  console.log(`memory: RSS ${mem.rssGB} GB (${mem.rssBytesPerEntity.toLocaleString()} B/entity)`)
  console.log('='.repeat(96))

  await brain.close()
  console.log('JSON ' + JSON.stringify({
    leg: 'brainy-open-core', n: written, dim: DIM, edges, ingestPerSec: ingestRate,
    rssGB: mem.rssGB, bytesPerEntityRss: mem.rssBytesPerEntity,
    vector, metadata, graph, triple
  }))
}

main().catch((e) => { console.error(e); process.exit(1) })