371 lines
15 KiB
TypeScript
371 lines
15 KiB
TypeScript
|
/**
|
||
|
|
* @module tests/benchmarks/model-b-write-perf.spike
|
||
|
|
* @description SPIKE (throwaway, spike/model-b-write-perf branch) — measures the
|
||
|
|
* cost of Model B (per-write immutable history). We do NOT build Model B to measure
|
||
|
|
* it: `transact([1 op])` already does exactly what Model B would make every single-op
|
||
|
|
* write do (read+save before-image → write delta → write manifest → sync). So this
|
||
|
|
* benchmarks today's single-op path (`add()`/`update()`) against `transact([1])`
|
||
|
|
* (Model B per-write, un-optimized upper bound) and `transact([K])` (group-commit
|
||
|
|
* amortization), on memory (isolates CPU/retention overhead, sync is a no-op) and
|
||
|
|
* filesystem-on-NVMe (adds real fsync), with embedding neutralized (precomputed
|
||
|
|
* vector + a constant embeddings provider) so the throughput delta is pure
|
||
|
|
* commit-path overhead. Also measures on-disk history growth (bytes/write retained).
|
||
|
|
*
|
||
|
|
* Run: node_modules/.bin/tsx tests/benchmarks/model-b-write-perf.spike.ts
|
||
|
|
* Optional smaller run: SPIKE_SCALE=small node_modules/.bin/tsx tests/benchmarks/model-b-write-perf.spike.ts
|
||
|
|
*/
|
||
|
|
|
||
|
|
import { rmSync, mkdirSync, existsSync } from 'node:fs'
|
||
|
|
import { execSync } from 'node:child_process'
|
||
|
|
import { Brainy } from '../../src/index.js'
|
||
|
|
import { NounType } from '../../src/types/graphTypes.js'
|
||
|
|
import type { BrainyConfig } from '../../src/types/brainy.types.js'
|
||
|
|
|
||
|
|
// Silence brainy's verbose per-brain init logging — it floods stdout (≈1 MB at
|
||
|
|
// small scale across dozens of fresh brains) and the I/O measurably slows the run.
|
||
|
|
// Capture the real console.log first, then no-op the noisy channels; `say()` is the
|
||
|
|
// benchmark's own output channel and is unaffected.
|
||
|
|
const _log = console.log.bind(console)
|
||
|
|
const say = (...a: any[]) => _log(...a)
|
||
|
|
for (const m of ['log', 'info', 'debug', 'warn'] as const) {
|
||
|
|
;(console as any)[m] = () => {}
|
||
|
|
}
|
||
|
|
|
||
|
|
// ---- config ---------------------------------------------------------------
|
||
|
|
|
||
|
|
const SMALL = process.env.SPIKE_SCALE === 'small'
|
||
|
|
const DIM = 384
|
||
|
|
// NVMe ext4 (NOT /tmp — that is tmpfs/RAM and would understate fsync).
|
||
|
|
const FS_ROOT = '/media/dpsifr/storage/home/Projects/brainy/.spike-data'
|
||
|
|
|
||
|
|
// Scales: memory is cheap (CPU only); filesystem pays fsync per transact so it is smaller.
|
||
|
|
const N_MEM = SMALL ? 2_000 : 20_000
|
||
|
|
const N_FS = SMALL ? 500 : 4_000
|
||
|
|
const K = 100 // group-commit batch size for the transact([K]) amortization run
|
||
|
|
const WARMUP = SMALL ? 100 : 500
|
||
|
|
const REPEATS = 3 // median of N timed repeats
|
||
|
|
|
||
|
|
// ---- helpers --------------------------------------------------------------
|
||
|
|
|
||
|
|
/** Deterministic, cheap, VARIED unit vector per index (avoids embedding AND a
|
||
|
|
* degenerate all-identical-vector HNSW graph). No model, ~µs. */
|
||
|
|
function vec(i: number): number[] {
|
||
|
|
const v = new Array<number>(DIM)
|
||
|
|
let mag = 0
|
||
|
|
for (let d = 0; d < DIM; d++) {
|
||
|
|
const x = Math.sin((i + 1) * 0.001 + d * 0.1) * 0.5 + Math.cos(d * 0.05) * 0.3
|
||
|
|
v[d] = x
|
||
|
|
mag += x * x
|
||
|
|
}
|
||
|
|
mag = Math.sqrt(mag) || 1
|
||
|
|
for (let d = 0; d < DIM; d++) v[d] /= mag
|
||
|
|
return v
|
||
|
|
}
|
||
|
|
|
||
|
|
const CONST_VEC = vec(0)
|
||
|
|
|
||
|
|
/** Build a brain with embedding fully neutralized:
|
||
|
|
* - a constant 'embeddings' provider → skips the WASM eager-init (the provider
|
||
|
|
* "owns" embeddings) and makes any embed call O(1);
|
||
|
|
* - every write also passes a precomputed `vector`, so embed is never even called. */
|
||
|
|
async function mkBrain(backend: 'memory' | 'filesystem', dir?: string): Promise<Brainy> {
|
||
|
|
const storage: BrainyConfig['storage'] =
|
||
|
|
backend === 'memory' ? { type: 'memory' } : { type: 'filesystem', path: dir! }
|
||
|
|
const brain = new Brainy({
|
||
|
|
requireSubtype: false,
|
||
|
|
storage,
|
||
|
|
// CRITICAL: skip the WASM embedder eager-init (90-140s compile per brain) —
|
||
|
|
// we never embed (precomputed vectors), so the model must not load at all.
|
||
|
|
eagerEmbeddings: false,
|
||
|
|
index: { m: 16, efConstruction: 200, efSearch: 50 },
|
||
|
|
cache: { maxSize: 1000, ttl: 3600 }
|
||
|
|
} as BrainyConfig)
|
||
|
|
brain.use({
|
||
|
|
name: 'const-embedder',
|
||
|
|
activate: async (ctx: any) => {
|
||
|
|
ctx.registerProvider('embeddings', async () => CONST_VEC)
|
||
|
|
return true
|
||
|
|
}
|
||
|
|
} as any)
|
||
|
|
await brain.init()
|
||
|
|
return brain
|
||
|
|
}
|
||
|
|
|
||
|
|
function freshDir(name: string): string {
|
||
|
|
const dir = `${FS_ROOT}/${name}`
|
||
|
|
if (existsSync(dir)) rmSync(dir, { recursive: true, force: true })
|
||
|
|
mkdirSync(dir, { recursive: true })
|
||
|
|
return dir
|
||
|
|
}
|
||
|
|
|
||
|
|
function dirBytes(dir: string): number {
|
||
|
|
try {
|
||
|
|
const out = execSync(`du -sb ${dir}`, { encoding: 'utf8' })
|
||
|
|
return parseInt(out.split(/\s+/)[0], 10)
|
||
|
|
} catch {
|
||
|
|
return -1
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
interface Timing {
|
||
|
|
perSec: number
|
||
|
|
msPerOp: number
|
||
|
|
}
|
||
|
|
|
||
|
|
/** Median-of-REPEATS timing of one full N-write workload built by `make`.
|
||
|
|
* `make` returns an async fn that performs ALL N writes on a FRESH brain. */
|
||
|
|
async function timeWorkload(
|
||
|
|
label: string,
|
||
|
|
build: () => Promise<{ run: () => Promise<void>; n: number; cleanup: () => Promise<void> }>
|
||
|
|
): Promise<Timing> {
|
||
|
|
const rates: number[] = []
|
||
|
|
for (let r = 0; r < REPEATS; r++) {
|
||
|
|
const { run, n, cleanup } = await build()
|
||
|
|
const t0 = process.hrtime.bigint()
|
||
|
|
await run()
|
||
|
|
const t1 = process.hrtime.bigint()
|
||
|
|
await cleanup()
|
||
|
|
const sec = Number(t1 - t0) / 1e9
|
||
|
|
rates.push(n / sec)
|
||
|
|
}
|
||
|
|
rates.sort((a, b) => a - b)
|
||
|
|
const perSec = rates[Math.floor(rates.length / 2)]
|
||
|
|
return { perSec, msPerOp: 1000 / perSec }
|
||
|
|
}
|
||
|
|
|
||
|
|
// ---- write workloads ------------------------------------------------------
|
||
|
|
// Every workload produces N entities and is identical except for the write path,
|
||
|
|
// so index-growth + payload costs cancel in the A-vs-B delta.
|
||
|
|
|
||
|
|
function payload(i: number) {
|
||
|
|
return {
|
||
|
|
type: NounType.Document,
|
||
|
|
subtype: 'note',
|
||
|
|
data: `doc ${i}`,
|
||
|
|
vector: vec(i),
|
||
|
|
metadata: { i, batch: 'spike', tag: i % 7 }
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
/** CREATE via single-op add() (today's path). */
|
||
|
|
function createSingleOp(backend: 'memory' | 'filesystem', n: number, dirName: string) {
|
||
|
|
return async () => {
|
||
|
|
const dir = backend === 'filesystem' ? freshDir(dirName) : undefined
|
||
|
|
const brain = await mkBrain(backend, dir)
|
||
|
|
return {
|
||
|
|
n,
|
||
|
|
run: async () => {
|
||
|
|
for (let i = 0; i < n; i++) await brain.add(payload(i))
|
||
|
|
},
|
||
|
|
cleanup: async () => {
|
||
|
|
await brain.close()
|
||
|
|
if (dir) rmSync(dir, { recursive: true, force: true })
|
||
|
|
}
|
||
|
|
}
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
/** CREATE via transact([batch]) — batch=1 is Model B per-write upper bound. */
|
||
|
|
function createTransact(backend: 'memory' | 'filesystem', n: number, batch: number, dirName: string) {
|
||
|
|
return async () => {
|
||
|
|
const dir = backend === 'filesystem' ? freshDir(dirName) : undefined
|
||
|
|
const brain = await mkBrain(backend, dir)
|
||
|
|
return {
|
||
|
|
n,
|
||
|
|
run: async () => {
|
||
|
|
for (let i = 0; i < n; i += batch) {
|
||
|
|
const ops = []
|
||
|
|
for (let j = i; j < Math.min(i + batch, n); j++) {
|
||
|
|
ops.push({ op: 'add' as const, ...payload(j) })
|
||
|
|
}
|
||
|
|
await brain.transact(ops as any)
|
||
|
|
}
|
||
|
|
},
|
||
|
|
cleanup: async () => {
|
||
|
|
await brain.close()
|
||
|
|
if (dir) rmSync(dir, { recursive: true, force: true })
|
||
|
|
}
|
||
|
|
}
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
/** UPDATE churn (the case where Model B's before-image = a FULL old record).
|
||
|
|
* Seeds n entities via fast single-op add(), then times n updates via the path. */
|
||
|
|
function updateSingleOp(backend: 'memory' | 'filesystem', n: number, dirName: string) {
|
||
|
|
return async () => {
|
||
|
|
const dir = backend === 'filesystem' ? freshDir(dirName) : undefined
|
||
|
|
const brain = await mkBrain(backend, dir)
|
||
|
|
const ids: string[] = []
|
||
|
|
for (let i = 0; i < n; i++) ids.push(await brain.add(payload(i)))
|
||
|
|
return {
|
||
|
|
n,
|
||
|
|
run: async () => {
|
||
|
|
for (let i = 0; i < n; i++) await brain.update({ id: ids[i], metadata: { i, rev: 1 } })
|
||
|
|
},
|
||
|
|
cleanup: async () => {
|
||
|
|
await brain.close()
|
||
|
|
if (dir) rmSync(dir, { recursive: true, force: true })
|
||
|
|
}
|
||
|
|
}
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
function updateTransact(backend: 'memory' | 'filesystem', n: number, batch: number, dirName: string) {
|
||
|
|
return async () => {
|
||
|
|
const dir = backend === 'filesystem' ? freshDir(dirName) : undefined
|
||
|
|
const brain = await mkBrain(backend, dir)
|
||
|
|
const ids: string[] = []
|
||
|
|
for (let i = 0; i < n; i++) ids.push(await brain.add(payload(i)))
|
||
|
|
return {
|
||
|
|
n,
|
||
|
|
run: async () => {
|
||
|
|
for (let i = 0; i < n; i += batch) {
|
||
|
|
const ops = []
|
||
|
|
for (let j = i; j < Math.min(i + batch, n); j++) {
|
||
|
|
ops.push({ op: 'update' as const, id: ids[j], metadata: { i: j, rev: 1 } })
|
||
|
|
}
|
||
|
|
await brain.transact(ops as any)
|
||
|
|
}
|
||
|
|
},
|
||
|
|
cleanup: async () => {
|
||
|
|
await brain.close()
|
||
|
|
if (dir) rmSync(dir, { recursive: true, force: true })
|
||
|
|
}
|
||
|
|
}
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
// ---- history growth (filesystem only) -------------------------------------
|
||
|
|
|
||
|
|
/** Disk bytes for N adds via single-op vs via transact([1]); delta = retained history. */
|
||
|
|
async function historyGrowthAdd(n: number) {
|
||
|
|
const dSingle = freshDir('hist-add-single')
|
||
|
|
let b = await mkBrain('filesystem', dSingle)
|
||
|
|
for (let i = 0; i < n; i++) await b.add(payload(i))
|
||
|
|
await b.close()
|
||
|
|
const singleBytes = dirBytes(dSingle)
|
||
|
|
|
||
|
|
const dTx = freshDir('hist-add-tx')
|
||
|
|
b = await mkBrain('filesystem', dTx)
|
||
|
|
for (let i = 0; i < n; i++) await b.transact([{ op: 'add', ...payload(i) }] as any)
|
||
|
|
await b.close()
|
||
|
|
const txBytes = dirBytes(dTx)
|
||
|
|
|
||
|
|
rmSync(dSingle, { recursive: true, force: true })
|
||
|
|
rmSync(dTx, { recursive: true, force: true })
|
||
|
|
return { n, singleBytes, txBytes, retainedTotal: txBytes - singleBytes, retainedPerWrite: (txBytes - singleBytes) / n }
|
||
|
|
}
|
||
|
|
|
||
|
|
/** Update-churn: n entities, each updated M times via transact([1]); disk delta vs the
|
||
|
|
* same entities updated via single-op update() (no history). Captures full-record before-images. */
|
||
|
|
async function historyGrowthUpdateChurn(n: number, churn: number) {
|
||
|
|
const seed = (b: Brainy) => (async () => {
|
||
|
|
const ids: string[] = []
|
||
|
|
for (let i = 0; i < n; i++) ids.push(await b.add(payload(i)))
|
||
|
|
return ids
|
||
|
|
})()
|
||
|
|
|
||
|
|
const dSingle = freshDir('hist-upd-single')
|
||
|
|
let b = await mkBrain('filesystem', dSingle)
|
||
|
|
let ids = await seed(b)
|
||
|
|
for (let c = 0; c < churn; c++) for (let i = 0; i < n; i++) await b.update({ id: ids[i], metadata: { i, rev: c } })
|
||
|
|
await b.close()
|
||
|
|
const singleBytes = dirBytes(dSingle)
|
||
|
|
|
||
|
|
const dTx = freshDir('hist-upd-tx')
|
||
|
|
b = await mkBrain('filesystem', dTx)
|
||
|
|
ids = await seed(b)
|
||
|
|
for (let c = 0; c < churn; c++) for (let i = 0; i < n; i++) await b.transact([{ op: 'update', id: ids[i], metadata: { i, rev: c } }] as any)
|
||
|
|
await b.close()
|
||
|
|
const txBytes = dirBytes(dTx)
|
||
|
|
|
||
|
|
rmSync(dSingle, { recursive: true, force: true })
|
||
|
|
rmSync(dTx, { recursive: true, force: true })
|
||
|
|
const writes = n * churn
|
||
|
|
return { writes, singleBytes, txBytes, retainedTotal: txBytes - singleBytes, retainedPerWrite: (txBytes - singleBytes) / writes }
|
||
|
|
}
|
||
|
|
|
||
|
|
// ---- driver ---------------------------------------------------------------
|
||
|
|
|
||
|
|
async function main() {
|
||
|
|
say(`[spike] start (scale=${SMALL ? 'small' : 'full'}); verifying no-WASM embedding...`)
|
||
|
|
// Sanity: a brain must construct + init fast (no WASM compile).
|
||
|
|
{
|
||
|
|
const t = process.hrtime.bigint()
|
||
|
|
const b = await mkBrain('memory')
|
||
|
|
await b.add(payload(0))
|
||
|
|
await b.close()
|
||
|
|
say(`[spike] brain init + 1 write OK in ${(Number(process.hrtime.bigint() - t) / 1e6).toFixed(0)}ms (must be <2000ms or WASM is loading)`)
|
||
|
|
}
|
||
|
|
if (existsSync(FS_ROOT)) rmSync(FS_ROOT, { recursive: true, force: true })
|
||
|
|
mkdirSync(FS_ROOT, { recursive: true })
|
||
|
|
|
||
|
|
const results: any = { meta: { N_MEM, N_FS, K, WARMUP, REPEATS, DIM, dim: DIM, small: SMALL }, throughput: {}, history: {} }
|
||
|
|
|
||
|
|
// Warmup (JIT/alloc) on a memory brain.
|
||
|
|
{
|
||
|
|
const b = await mkBrain('memory')
|
||
|
|
for (let i = 0; i < WARMUP; i++) await b.add(payload(i))
|
||
|
|
for (let i = 0; i < WARMUP; i += K) await b.transact([{ op: 'add', ...payload(i) }] as any)
|
||
|
|
await b.close()
|
||
|
|
}
|
||
|
|
|
||
|
|
const fmt = (t: Timing) => `${Math.round(t.perSec).toLocaleString()}/s (${t.msPerOp.toFixed(3)} ms)`
|
||
|
|
|
||
|
|
for (const backend of ['memory', 'filesystem'] as const) {
|
||
|
|
const n = backend === 'memory' ? N_MEM : N_FS
|
||
|
|
say(`\n========== ${backend.toUpperCase()} backend (N=${n.toLocaleString()}) ==========`)
|
||
|
|
|
||
|
|
// CREATE
|
||
|
|
const addSingle = await timeWorkload('add-single', createSingleOp(backend, n, 'add-single'))
|
||
|
|
const addTx1 = await timeWorkload('add-tx1', createTransact(backend, n, 1, 'add-tx1'))
|
||
|
|
const addTxK = await timeWorkload('add-txK', createTransact(backend, n, K, 'add-txK'))
|
||
|
|
say(`CREATE add() ${fmt(addSingle)}`)
|
||
|
|
say(`CREATE transact([1]) ${fmt(addTx1)} <- Model B per-write (slowdown ${(addSingle.perSec / addTx1.perSec).toFixed(2)}x)`)
|
||
|
|
say(`CREATE transact([${K}]) ${fmt(addTxK)} <- group-commit (slowdown vs add() ${(addSingle.perSec / addTxK.perSec).toFixed(2)}x)`)
|
||
|
|
|
||
|
|
// UPDATE
|
||
|
|
const updSingle = await timeWorkload('upd-single', updateSingleOp(backend, n, 'upd-single'))
|
||
|
|
const updTx1 = await timeWorkload('upd-tx1', updateTransact(backend, n, 1, 'upd-tx1'))
|
||
|
|
const updTxK = await timeWorkload('upd-txK', updateTransact(backend, n, K, 'upd-txK'))
|
||
|
|
say(`UPDATE update() ${fmt(updSingle)}`)
|
||
|
|
say(`UPDATE transact([1]) ${fmt(updTx1)} <- Model B per-write (slowdown ${(updSingle.perSec / updTx1.perSec).toFixed(2)}x)`)
|
||
|
|
say(`UPDATE transact([${K}]) ${fmt(updTxK)} <- group-commit (slowdown vs update() ${(updSingle.perSec / updTxK.perSec).toFixed(2)}x)`)
|
||
|
|
|
||
|
|
results.throughput[backend] = {
|
||
|
|
n,
|
||
|
|
create: { add: addSingle, tx1: addTx1, txK: addTxK, slowdownTx1: addSingle.perSec / addTx1.perSec, slowdownTxK: addSingle.perSec / addTxK.perSec },
|
||
|
|
update: { single: updSingle, tx1: updTx1, txK: updTxK, slowdownTx1: updSingle.perSec / updTx1.perSec, slowdownTxK: updSingle.perSec / updTxK.perSec }
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
// HISTORY GROWTH (filesystem, NVMe)
|
||
|
|
say(`\n========== HISTORY GROWTH (filesystem NVMe) ==========`)
|
||
|
|
const hN = SMALL ? 500 : 3_000
|
||
|
|
const addGrowth = await historyGrowthAdd(hN)
|
||
|
|
say(`ADD-only (${hN.toLocaleString()} adds): retained history = ${(addGrowth.retainedTotal / 1024).toFixed(0)} KiB total, ${addGrowth.retainedPerWrite.toFixed(0)} bytes/write`)
|
||
|
|
const churn = SMALL ? 3 : 5
|
||
|
|
const updGrowth = await historyGrowthUpdateChurn(SMALL ? 200 : 1_000, churn)
|
||
|
|
say(`UPDATE-churn (${updGrowth.writes.toLocaleString()} updates): retained history = ${(updGrowth.retainedTotal / 1024 / 1024).toFixed(1)} MiB total, ${updGrowth.retainedPerWrite.toFixed(0)} bytes/write`)
|
||
|
|
results.history = { addGrowth, updGrowth }
|
||
|
|
|
||
|
|
// persist raw JSON for the verification workflow
|
||
|
|
const outPath = `${FS_ROOT}/../tests/benchmarks/_spike-results.json`
|
||
|
|
const fs = await import('node:fs')
|
||
|
|
fs.writeFileSync(`/media/dpsifr/storage/home/Projects/brainy/tests/benchmarks/_spike-results.json`, JSON.stringify(results, null, 2))
|
||
|
|
say(`\nRaw results → tests/benchmarks/_spike-results.json`)
|
||
|
|
|
||
|
|
// cleanup data root
|
||
|
|
if (existsSync(FS_ROOT)) rmSync(FS_ROOT, { recursive: true, force: true })
|
||
|
|
}
|
||
|
|
|
||
|
|
main()
|
||
|
|
.then(() => {
|
||
|
|
say('[spike] done.')
|
||
|
|
process.exit(0) // force exit — brain caches/timers can keep the loop alive
|
||
|
|
})
|
||
|
|
.catch((e) => {
|
||
|
|
console.error(e)
|
||
|
|
process.exit(1)
|
||
|
|
})
|