test(8.0): Model-B write-perf + scalability spike harnesses
Dev-only standalone harnesses (run via node --import tsx; not globbed by the unit/integration vitest configs). model-b-scalability.spike.ts is the evidence harness referenced by the Model-B build spec — re-validates read-vs-depth, RAM-vs-depth, reopen, and compaction at scale.
This commit is contained in:
parent
ceed70d7be
commit
afac7f9662
2 changed files with 566 additions and 0 deletions
196
tests/benchmarks/model-b-scalability.spike.ts
Normal file
196
tests/benchmarks/model-b-scalability.spike.ts
Normal file
|
|
@ -0,0 +1,196 @@
|
||||||
|
/**
|
||||||
|
* @module tests/benchmarks/model-b-scalability.spike
|
||||||
|
* @description SPIKE (throwaway, spike/model-b-write-perf branch) — the DECISION-CRITICAL
|
||||||
|
* half of the Model B evaluation. The write-perf spike showed writes are cheap at
|
||||||
|
* durability parity; the adversarial review showed the REAL risk is structural and
|
||||||
|
* grows with the number of retained generations N:
|
||||||
|
*
|
||||||
|
* 1. READ-vs-depth — resolveAt/changedBetween LINEARLY scan the global committedGens
|
||||||
|
* array → historical reads (asOf/get, diff, since) become O(N) = O(database-age).
|
||||||
|
* 2. REOPEN-vs-depth — GenerationStore.open enumerates+sorts EVERY generation dir → O(N) cold start.
|
||||||
|
* 3. RAM-vs-depth — deltaCache holds a Set per committed generation → O(N) resident heap.
|
||||||
|
* 4. COMPACTION cost — the only thing that bounds 1-3; time it + the post-compaction footprint.
|
||||||
|
*
|
||||||
|
* Each is measured at two depths (small, large). A ~linear slope CONFIRMS that brainy-TS's
|
||||||
|
* current global-committedGens design does NOT scale under Model B (every write = a generation),
|
||||||
|
* and that the canonical Model-B history must be a PER-ID INDEX (mirroring cor's delta_history
|
||||||
|
* BTreeMap, O(log n)) rather than a global scan. Memory backend is used where the scan/RAM cost
|
||||||
|
* is backend-independent (committedGens is in-memory); filesystem only where reopen/compaction
|
||||||
|
* touch disk.
|
||||||
|
*
|
||||||
|
* Run (force GC available): node --expose-gc node_modules/.bin/tsx tests/benchmarks/model-b-scalability.spike.ts
|
||||||
|
* small scale: SPIKE_SCALE=small ...
|
||||||
|
*/
|
||||||
|
|
||||||
|
import { rmSync, mkdirSync, existsSync } from 'node:fs'
|
||||||
|
import { execSync } from 'node:child_process'
|
||||||
|
import { Brainy } from '../../src/index.js'
|
||||||
|
import { NounType } from '../../src/types/graphTypes.js'
|
||||||
|
import type { BrainyConfig } from '../../src/types/brainy.types.js'
|
||||||
|
|
||||||
|
// Silence brainy's verbose per-brain logging (floods stdout, slows the run).
|
||||||
|
const _log = console.log.bind(console)
|
||||||
|
const say = (...a: any[]) => _log(...a)
|
||||||
|
for (const m of ['log', 'info', 'debug', 'warn'] as const) (console as any)[m] = () => {}
|
||||||
|
|
||||||
|
const SMALL = process.env.SPIKE_SCALE === 'small'
|
||||||
|
const DIM = 384
|
||||||
|
const FS_ROOT = '/media/dpsifr/storage/home/Projects/brainy/.spike-data'
|
||||||
|
// Two depths to read the slope. Memory legs can go deep cheaply; fs legs (fsync per gen) stay smaller.
|
||||||
|
const DEPTHS_MEM = SMALL ? [500, 5_000] : [2_000, 20_000]
|
||||||
|
const DEPTHS_FS = SMALL ? [300, 1_500] : [1_000, 6_000]
|
||||||
|
const READ_REPEATS = SMALL ? 20 : 50
|
||||||
|
|
||||||
|
function vec(i: number): number[] {
|
||||||
|
const v = new Array<number>(DIM)
|
||||||
|
let mag = 0
|
||||||
|
for (let d = 0; d < DIM; d++) { const x = Math.sin((i + 1) * 0.001 + d * 0.1); v[d] = x; mag += x * x }
|
||||||
|
mag = Math.sqrt(mag) || 1
|
||||||
|
for (let d = 0; d < DIM; d++) v[d] /= mag
|
||||||
|
return v
|
||||||
|
}
|
||||||
|
|
||||||
|
async function mkBrain(backend: 'memory' | 'filesystem', dir?: string): Promise<Brainy> {
|
||||||
|
const storage: BrainyConfig['storage'] =
|
||||||
|
backend === 'memory' ? { type: 'memory' } : { type: 'filesystem', path: dir! }
|
||||||
|
const brain = new Brainy({
|
||||||
|
requireSubtype: false,
|
||||||
|
storage,
|
||||||
|
eagerEmbeddings: false, // never load WASM — we pass precomputed vectors
|
||||||
|
// Keep history fully retained for the measurement (no mid-run compaction skewing depth).
|
||||||
|
history: { autoCompact: false } as any,
|
||||||
|
index: { m: 16, efConstruction: 200, efSearch: 50 },
|
||||||
|
cache: { maxSize: 1000, ttl: 3600 }
|
||||||
|
} as BrainyConfig)
|
||||||
|
brain.use({ name: 'const-embedder', activate: async (ctx: any) => { ctx.registerProvider('embeddings', async () => vec(0)); return true } } as any)
|
||||||
|
await brain.init()
|
||||||
|
return brain
|
||||||
|
}
|
||||||
|
|
||||||
|
function payload(i: number) {
|
||||||
|
return { type: NounType.Document, subtype: 'note', data: `doc ${i}`, vector: vec(i), metadata: { i, batch: 'spike' } }
|
||||||
|
}
|
||||||
|
|
||||||
|
function freshDir(name: string): string {
|
||||||
|
const dir = `${FS_ROOT}/${name}`
|
||||||
|
if (existsSync(dir)) rmSync(dir, { recursive: true, force: true })
|
||||||
|
mkdirSync(dir, { recursive: true })
|
||||||
|
return dir
|
||||||
|
}
|
||||||
|
function allocBytes(dir: string): number { try { return parseInt(execSync(`du -s --block-size=1 ${dir}`, { encoding: 'utf8' }).split(/\s+/)[0], 10) } catch { return -1 } }
|
||||||
|
|
||||||
|
const SEED = 32 // small working set; X (id[0]) is the never-updated probe entity
|
||||||
|
|
||||||
|
/** Build N committed generations via transact([1]). X = ids[0] is seeded then NEVER updated, so a
|
||||||
|
* read of X at the oldest pin must scan all N generations to confirm it is unchanged (worst case). */
|
||||||
|
async function buildGenerations(b: Brainy, nGens: number): Promise<{ ids: string[]; g0: number; gN: number }> {
|
||||||
|
const ids: string[] = []
|
||||||
|
// Seed via ONE transact so g0 is a committed generation (asOf needs a committed gen).
|
||||||
|
const seedOps = Array.from({ length: SEED }, (_, i) => ({ op: 'add' as const, ...payload(i) }))
|
||||||
|
const seedDb = await b.transact(seedOps as any)
|
||||||
|
for (const id of (await b.find({ type: NounType.Document, limit: SEED }))) ids.push(id.id)
|
||||||
|
await seedDb.release?.()
|
||||||
|
const g0 = b.generation()
|
||||||
|
// N generations, each a single-op-equivalent transact([1]) update on the CHURN set (ids[1..]),
|
||||||
|
// never touching X=ids[0].
|
||||||
|
for (let k = 0; k < nGens; k++) {
|
||||||
|
const id = ids[1 + (k % (ids.length - 1))]
|
||||||
|
const d = await b.transact([{ op: 'update', id, metadata: { k } }] as any)
|
||||||
|
await d.release?.()
|
||||||
|
}
|
||||||
|
return { ids, g0, gN: b.generation() }
|
||||||
|
}
|
||||||
|
|
||||||
|
function median(xs: number[]): number { const s = [...xs].sort((a, b) => a - b); return s[Math.floor(s.length / 2)] }
|
||||||
|
|
||||||
|
async function timeMs(fn: () => Promise<void>, repeats: number): Promise<number> {
|
||||||
|
const ts: number[] = []
|
||||||
|
for (let r = 0; r < repeats; r++) { const t = process.hrtime.bigint(); await fn(); ts.push(Number(process.hrtime.bigint() - t) / 1e6) }
|
||||||
|
return median(ts)
|
||||||
|
}
|
||||||
|
|
||||||
|
async function main() {
|
||||||
|
say(`[scale] start (scale=${SMALL ? 'small' : 'full'}); gc=${typeof (global as any).gc === 'function' ? 'on' : 'OFF (run with node --expose-gc)'}`)
|
||||||
|
if (existsSync(FS_ROOT)) rmSync(FS_ROOT, { recursive: true, force: true })
|
||||||
|
mkdirSync(FS_ROOT, { recursive: true })
|
||||||
|
const results: any = { readVsDepth: [], reopenVsDepth: [], ramVsDepth: [], compaction: [] }
|
||||||
|
|
||||||
|
// ---- 1. READ latency vs history depth (memory; scan is in-memory) ----
|
||||||
|
say(`\n========== READ latency vs history depth (the O(database-age) test) ==========`)
|
||||||
|
for (const N of DEPTHS_MEM) {
|
||||||
|
const b = await mkBrain('memory')
|
||||||
|
const { ids, g0, gN } = await buildGenerations(b, N)
|
||||||
|
const X = ids[0]
|
||||||
|
// asOf(oldest).get(X): resolveAt(X,g0) scans all N committedGens (X never touched) → expect O(N).
|
||||||
|
const tAsOfGet = await timeMs(async () => { const db = await b.asOf(g0); await db.get(X); await db.release?.() }, READ_REPEATS)
|
||||||
|
// diff(g0,gN): changedBetween scans the full (g0,gN] range → expect O(N).
|
||||||
|
const tDiff = await timeMs(async () => { await b.diff(g0, gN) }, Math.max(5, READ_REPEATS / 5))
|
||||||
|
await b.close()
|
||||||
|
say(`N=${N.toLocaleString()} gens: asOf(oldest).get(X) = ${tAsOfGet.toFixed(3)} ms | diff(g0,gN) = ${tDiff.toFixed(2)} ms`)
|
||||||
|
results.readVsDepth.push({ N, asOfGetMs: tAsOfGet, diffMs: tDiff })
|
||||||
|
}
|
||||||
|
{
|
||||||
|
const [a, c] = results.readVsDepth
|
||||||
|
if (a && c) say(` → asOf-get scaling ${a.N}→${c.N} (${(c.N / a.N).toFixed(0)}x depth): ${(c.asOfGetMs / a.asOfGetMs).toFixed(1)}x slower (linear≈${(c.N / a.N).toFixed(0)}x → confirms O(N) scan)`)
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---- 2. RAM resident vs history depth (memory; deltaCache holds a Set per gen) ----
|
||||||
|
say(`\n========== RAM (heapUsed) vs history depth (deltaCache O(N) Sets) ==========`)
|
||||||
|
for (const N of DEPTHS_MEM) {
|
||||||
|
if ((global as any).gc) (global as any).gc()
|
||||||
|
const before = process.memoryUsage().heapUsed
|
||||||
|
const b = await mkBrain('memory')
|
||||||
|
await buildGenerations(b, N)
|
||||||
|
if ((global as any).gc) (global as any).gc()
|
||||||
|
const after = process.memoryUsage().heapUsed
|
||||||
|
const perGen = (after - before) / N
|
||||||
|
await b.close()
|
||||||
|
say(`N=${N.toLocaleString()} gens: heapUsed +${((after - before) / 1024 / 1024).toFixed(1)} MiB (${perGen.toFixed(0)} bytes/generation resident)`)
|
||||||
|
results.ramVsDepth.push({ N, deltaBytes: after - before, bytesPerGen: perGen })
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---- 3. COLD-REOPEN vs history depth (filesystem; open() walks every gen dir) ----
|
||||||
|
say(`\n========== COLD-REOPEN time vs history depth (O(generations) dir walk) ==========`)
|
||||||
|
for (const N of DEPTHS_FS) {
|
||||||
|
const dir = freshDir(`reopen-${N}`)
|
||||||
|
let b = await mkBrain('filesystem', dir)
|
||||||
|
await buildGenerations(b, N)
|
||||||
|
await b.close()
|
||||||
|
const tOpen = await timeMs(async () => { const rb = await mkBrain('filesystem', dir); await rb.close() }, SMALL ? 3 : 5)
|
||||||
|
rmSync(dir, { recursive: true, force: true })
|
||||||
|
say(`N=${N.toLocaleString()} gens: cold reopen = ${tOpen.toFixed(0)} ms`)
|
||||||
|
results.reopenVsDepth.push({ N, reopenMs: tOpen })
|
||||||
|
}
|
||||||
|
{
|
||||||
|
const [a, c] = results.reopenVsDepth
|
||||||
|
if (a && c) say(` → reopen scaling ${a.N}→${c.N} (${(c.N / a.N).toFixed(0)}x depth): ${(c.reopenMs / a.reopenMs).toFixed(1)}x slower`)
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---- 4. COMPACTION cost + footprint (filesystem) ----
|
||||||
|
say(`\n========== COMPACTION cost (the bound on 1-3) ==========`)
|
||||||
|
{
|
||||||
|
const N = DEPTHS_FS[DEPTHS_FS.length - 1]
|
||||||
|
const dir = freshDir(`compact-${N}`)
|
||||||
|
const b = await mkBrain('filesystem', dir)
|
||||||
|
await buildGenerations(b, N)
|
||||||
|
await b.flush()
|
||||||
|
const beforeBytes = allocBytes(`${dir}/_generations`)
|
||||||
|
const t = process.hrtime.bigint()
|
||||||
|
const res = await b.compactHistory({ retainGenerations: 100 } as any)
|
||||||
|
const compMs = Number(process.hrtime.bigint() - t) / 1e6
|
||||||
|
await b.flush()
|
||||||
|
const afterBytes = allocBytes(`${dir}/_generations`)
|
||||||
|
await b.close()
|
||||||
|
rmSync(dir, { recursive: true, force: true })
|
||||||
|
say(`N=${N.toLocaleString()} gens → compactHistory({retainGenerations:100}): ${compMs.toFixed(0)} ms, removed ${(res as any)?.removedGenerations ?? '?'} gens`)
|
||||||
|
say(` _generations footprint: ${(beforeBytes / 1024 / 1024).toFixed(1)} MiB → ${(afterBytes / 1024 / 1024).toFixed(1)} MiB allocated`)
|
||||||
|
results.compaction.push({ N, compactMs: compMs, removed: (res as any)?.removedGenerations, beforeBytes, afterBytes })
|
||||||
|
}
|
||||||
|
|
||||||
|
const fs = await import('node:fs')
|
||||||
|
fs.writeFileSync('/media/dpsifr/storage/home/Projects/brainy/tests/benchmarks/_scalability-results.json', JSON.stringify(results, null, 2))
|
||||||
|
say(`\nRaw results → tests/benchmarks/_scalability-results.json`)
|
||||||
|
if (existsSync(FS_ROOT)) rmSync(FS_ROOT, { recursive: true, force: true })
|
||||||
|
}
|
||||||
|
|
||||||
|
main().then(() => { say('[scale] done.'); process.exit(0) }).catch((e) => { console.error(e); process.exit(1) })
|
||||||
370
tests/benchmarks/model-b-write-perf.spike.ts
Normal file
370
tests/benchmarks/model-b-write-perf.spike.ts
Normal file
|
|
@ -0,0 +1,370 @@
|
||||||
|
/**
|
||||||
|
* @module tests/benchmarks/model-b-write-perf.spike
|
||||||
|
* @description SPIKE (throwaway, spike/model-b-write-perf branch) — measures the
|
||||||
|
* cost of Model B (per-write immutable history). We do NOT build Model B to measure
|
||||||
|
* it: `transact([1 op])` already does exactly what Model B would make every single-op
|
||||||
|
* write do (read+save before-image → write delta → write manifest → sync). So this
|
||||||
|
* benchmarks today's single-op path (`add()`/`update()`) against `transact([1])`
|
||||||
|
* (Model B per-write, un-optimized upper bound) and `transact([K])` (group-commit
|
||||||
|
* amortization), on memory (isolates CPU/retention overhead, sync is a no-op) and
|
||||||
|
* filesystem-on-NVMe (adds real fsync), with embedding neutralized (precomputed
|
||||||
|
* vector + a constant embeddings provider) so the throughput delta is pure
|
||||||
|
* commit-path overhead. Also measures on-disk history growth (bytes/write retained).
|
||||||
|
*
|
||||||
|
* Run: node_modules/.bin/tsx tests/benchmarks/model-b-write-perf.spike.ts
|
||||||
|
* Optional smaller run: SPIKE_SCALE=small node_modules/.bin/tsx tests/benchmarks/model-b-write-perf.spike.ts
|
||||||
|
*/
|
||||||
|
|
||||||
|
import { rmSync, mkdirSync, existsSync } from 'node:fs'
|
||||||
|
import { execSync } from 'node:child_process'
|
||||||
|
import { Brainy } from '../../src/index.js'
|
||||||
|
import { NounType } from '../../src/types/graphTypes.js'
|
||||||
|
import type { BrainyConfig } from '../../src/types/brainy.types.js'
|
||||||
|
|
||||||
|
// Silence brainy's verbose per-brain init logging — it floods stdout (≈1 MB at
|
||||||
|
// small scale across dozens of fresh brains) and the I/O measurably slows the run.
|
||||||
|
// Capture the real console.log first, then no-op the noisy channels; `say()` is the
|
||||||
|
// benchmark's own output channel and is unaffected.
|
||||||
|
const _log = console.log.bind(console)
|
||||||
|
const say = (...a: any[]) => _log(...a)
|
||||||
|
for (const m of ['log', 'info', 'debug', 'warn'] as const) {
|
||||||
|
;(console as any)[m] = () => {}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---- config ---------------------------------------------------------------
|
||||||
|
|
||||||
|
const SMALL = process.env.SPIKE_SCALE === 'small'
|
||||||
|
const DIM = 384
|
||||||
|
// NVMe ext4 (NOT /tmp — that is tmpfs/RAM and would understate fsync).
|
||||||
|
const FS_ROOT = '/media/dpsifr/storage/home/Projects/brainy/.spike-data'
|
||||||
|
|
||||||
|
// Scales: memory is cheap (CPU only); filesystem pays fsync per transact so it is smaller.
|
||||||
|
const N_MEM = SMALL ? 2_000 : 20_000
|
||||||
|
const N_FS = SMALL ? 500 : 4_000
|
||||||
|
const K = 100 // group-commit batch size for the transact([K]) amortization run
|
||||||
|
const WARMUP = SMALL ? 100 : 500
|
||||||
|
const REPEATS = 3 // median of N timed repeats
|
||||||
|
|
||||||
|
// ---- helpers --------------------------------------------------------------
|
||||||
|
|
||||||
|
/** Deterministic, cheap, VARIED unit vector per index (avoids embedding AND a
|
||||||
|
* degenerate all-identical-vector HNSW graph). No model, ~µs. */
|
||||||
|
function vec(i: number): number[] {
|
||||||
|
const v = new Array<number>(DIM)
|
||||||
|
let mag = 0
|
||||||
|
for (let d = 0; d < DIM; d++) {
|
||||||
|
const x = Math.sin((i + 1) * 0.001 + d * 0.1) * 0.5 + Math.cos(d * 0.05) * 0.3
|
||||||
|
v[d] = x
|
||||||
|
mag += x * x
|
||||||
|
}
|
||||||
|
mag = Math.sqrt(mag) || 1
|
||||||
|
for (let d = 0; d < DIM; d++) v[d] /= mag
|
||||||
|
return v
|
||||||
|
}
|
||||||
|
|
||||||
|
const CONST_VEC = vec(0)
|
||||||
|
|
||||||
|
/** Build a brain with embedding fully neutralized:
|
||||||
|
* - a constant 'embeddings' provider → skips the WASM eager-init (the provider
|
||||||
|
* "owns" embeddings) and makes any embed call O(1);
|
||||||
|
* - every write also passes a precomputed `vector`, so embed is never even called. */
|
||||||
|
async function mkBrain(backend: 'memory' | 'filesystem', dir?: string): Promise<Brainy> {
|
||||||
|
const storage: BrainyConfig['storage'] =
|
||||||
|
backend === 'memory' ? { type: 'memory' } : { type: 'filesystem', path: dir! }
|
||||||
|
const brain = new Brainy({
|
||||||
|
requireSubtype: false,
|
||||||
|
storage,
|
||||||
|
// CRITICAL: skip the WASM embedder eager-init (90-140s compile per brain) —
|
||||||
|
// we never embed (precomputed vectors), so the model must not load at all.
|
||||||
|
eagerEmbeddings: false,
|
||||||
|
index: { m: 16, efConstruction: 200, efSearch: 50 },
|
||||||
|
cache: { maxSize: 1000, ttl: 3600 }
|
||||||
|
} as BrainyConfig)
|
||||||
|
brain.use({
|
||||||
|
name: 'const-embedder',
|
||||||
|
activate: async (ctx: any) => {
|
||||||
|
ctx.registerProvider('embeddings', async () => CONST_VEC)
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
} as any)
|
||||||
|
await brain.init()
|
||||||
|
return brain
|
||||||
|
}
|
||||||
|
|
||||||
|
function freshDir(name: string): string {
|
||||||
|
const dir = `${FS_ROOT}/${name}`
|
||||||
|
if (existsSync(dir)) rmSync(dir, { recursive: true, force: true })
|
||||||
|
mkdirSync(dir, { recursive: true })
|
||||||
|
return dir
|
||||||
|
}
|
||||||
|
|
||||||
|
function dirBytes(dir: string): number {
|
||||||
|
try {
|
||||||
|
const out = execSync(`du -sb ${dir}`, { encoding: 'utf8' })
|
||||||
|
return parseInt(out.split(/\s+/)[0], 10)
|
||||||
|
} catch {
|
||||||
|
return -1
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
interface Timing {
|
||||||
|
perSec: number
|
||||||
|
msPerOp: number
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Median-of-REPEATS timing of one full N-write workload built by `make`.
|
||||||
|
* `make` returns an async fn that performs ALL N writes on a FRESH brain. */
|
||||||
|
async function timeWorkload(
|
||||||
|
label: string,
|
||||||
|
build: () => Promise<{ run: () => Promise<void>; n: number; cleanup: () => Promise<void> }>
|
||||||
|
): Promise<Timing> {
|
||||||
|
const rates: number[] = []
|
||||||
|
for (let r = 0; r < REPEATS; r++) {
|
||||||
|
const { run, n, cleanup } = await build()
|
||||||
|
const t0 = process.hrtime.bigint()
|
||||||
|
await run()
|
||||||
|
const t1 = process.hrtime.bigint()
|
||||||
|
await cleanup()
|
||||||
|
const sec = Number(t1 - t0) / 1e9
|
||||||
|
rates.push(n / sec)
|
||||||
|
}
|
||||||
|
rates.sort((a, b) => a - b)
|
||||||
|
const perSec = rates[Math.floor(rates.length / 2)]
|
||||||
|
return { perSec, msPerOp: 1000 / perSec }
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---- write workloads ------------------------------------------------------
|
||||||
|
// Every workload produces N entities and is identical except for the write path,
|
||||||
|
// so index-growth + payload costs cancel in the A-vs-B delta.
|
||||||
|
|
||||||
|
function payload(i: number) {
|
||||||
|
return {
|
||||||
|
type: NounType.Document,
|
||||||
|
subtype: 'note',
|
||||||
|
data: `doc ${i}`,
|
||||||
|
vector: vec(i),
|
||||||
|
metadata: { i, batch: 'spike', tag: i % 7 }
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/** CREATE via single-op add() (today's path). */
|
||||||
|
function createSingleOp(backend: 'memory' | 'filesystem', n: number, dirName: string) {
|
||||||
|
return async () => {
|
||||||
|
const dir = backend === 'filesystem' ? freshDir(dirName) : undefined
|
||||||
|
const brain = await mkBrain(backend, dir)
|
||||||
|
return {
|
||||||
|
n,
|
||||||
|
run: async () => {
|
||||||
|
for (let i = 0; i < n; i++) await brain.add(payload(i))
|
||||||
|
},
|
||||||
|
cleanup: async () => {
|
||||||
|
await brain.close()
|
||||||
|
if (dir) rmSync(dir, { recursive: true, force: true })
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/** CREATE via transact([batch]) — batch=1 is Model B per-write upper bound. */
|
||||||
|
function createTransact(backend: 'memory' | 'filesystem', n: number, batch: number, dirName: string) {
|
||||||
|
return async () => {
|
||||||
|
const dir = backend === 'filesystem' ? freshDir(dirName) : undefined
|
||||||
|
const brain = await mkBrain(backend, dir)
|
||||||
|
return {
|
||||||
|
n,
|
||||||
|
run: async () => {
|
||||||
|
for (let i = 0; i < n; i += batch) {
|
||||||
|
const ops = []
|
||||||
|
for (let j = i; j < Math.min(i + batch, n); j++) {
|
||||||
|
ops.push({ op: 'add' as const, ...payload(j) })
|
||||||
|
}
|
||||||
|
await brain.transact(ops as any)
|
||||||
|
}
|
||||||
|
},
|
||||||
|
cleanup: async () => {
|
||||||
|
await brain.close()
|
||||||
|
if (dir) rmSync(dir, { recursive: true, force: true })
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/** UPDATE churn (the case where Model B's before-image = a FULL old record).
|
||||||
|
* Seeds n entities via fast single-op add(), then times n updates via the path. */
|
||||||
|
function updateSingleOp(backend: 'memory' | 'filesystem', n: number, dirName: string) {
|
||||||
|
return async () => {
|
||||||
|
const dir = backend === 'filesystem' ? freshDir(dirName) : undefined
|
||||||
|
const brain = await mkBrain(backend, dir)
|
||||||
|
const ids: string[] = []
|
||||||
|
for (let i = 0; i < n; i++) ids.push(await brain.add(payload(i)))
|
||||||
|
return {
|
||||||
|
n,
|
||||||
|
run: async () => {
|
||||||
|
for (let i = 0; i < n; i++) await brain.update({ id: ids[i], metadata: { i, rev: 1 } })
|
||||||
|
},
|
||||||
|
cleanup: async () => {
|
||||||
|
await brain.close()
|
||||||
|
if (dir) rmSync(dir, { recursive: true, force: true })
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function updateTransact(backend: 'memory' | 'filesystem', n: number, batch: number, dirName: string) {
|
||||||
|
return async () => {
|
||||||
|
const dir = backend === 'filesystem' ? freshDir(dirName) : undefined
|
||||||
|
const brain = await mkBrain(backend, dir)
|
||||||
|
const ids: string[] = []
|
||||||
|
for (let i = 0; i < n; i++) ids.push(await brain.add(payload(i)))
|
||||||
|
return {
|
||||||
|
n,
|
||||||
|
run: async () => {
|
||||||
|
for (let i = 0; i < n; i += batch) {
|
||||||
|
const ops = []
|
||||||
|
for (let j = i; j < Math.min(i + batch, n); j++) {
|
||||||
|
ops.push({ op: 'update' as const, id: ids[j], metadata: { i: j, rev: 1 } })
|
||||||
|
}
|
||||||
|
await brain.transact(ops as any)
|
||||||
|
}
|
||||||
|
},
|
||||||
|
cleanup: async () => {
|
||||||
|
await brain.close()
|
||||||
|
if (dir) rmSync(dir, { recursive: true, force: true })
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---- history growth (filesystem only) -------------------------------------
|
||||||
|
|
||||||
|
/** Disk bytes for N adds via single-op vs via transact([1]); delta = retained history. */
|
||||||
|
async function historyGrowthAdd(n: number) {
|
||||||
|
const dSingle = freshDir('hist-add-single')
|
||||||
|
let b = await mkBrain('filesystem', dSingle)
|
||||||
|
for (let i = 0; i < n; i++) await b.add(payload(i))
|
||||||
|
await b.close()
|
||||||
|
const singleBytes = dirBytes(dSingle)
|
||||||
|
|
||||||
|
const dTx = freshDir('hist-add-tx')
|
||||||
|
b = await mkBrain('filesystem', dTx)
|
||||||
|
for (let i = 0; i < n; i++) await b.transact([{ op: 'add', ...payload(i) }] as any)
|
||||||
|
await b.close()
|
||||||
|
const txBytes = dirBytes(dTx)
|
||||||
|
|
||||||
|
rmSync(dSingle, { recursive: true, force: true })
|
||||||
|
rmSync(dTx, { recursive: true, force: true })
|
||||||
|
return { n, singleBytes, txBytes, retainedTotal: txBytes - singleBytes, retainedPerWrite: (txBytes - singleBytes) / n }
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Update-churn: n entities, each updated M times via transact([1]); disk delta vs the
|
||||||
|
* same entities updated via single-op update() (no history). Captures full-record before-images. */
|
||||||
|
async function historyGrowthUpdateChurn(n: number, churn: number) {
|
||||||
|
const seed = (b: Brainy) => (async () => {
|
||||||
|
const ids: string[] = []
|
||||||
|
for (let i = 0; i < n; i++) ids.push(await b.add(payload(i)))
|
||||||
|
return ids
|
||||||
|
})()
|
||||||
|
|
||||||
|
const dSingle = freshDir('hist-upd-single')
|
||||||
|
let b = await mkBrain('filesystem', dSingle)
|
||||||
|
let ids = await seed(b)
|
||||||
|
for (let c = 0; c < churn; c++) for (let i = 0; i < n; i++) await b.update({ id: ids[i], metadata: { i, rev: c } })
|
||||||
|
await b.close()
|
||||||
|
const singleBytes = dirBytes(dSingle)
|
||||||
|
|
||||||
|
const dTx = freshDir('hist-upd-tx')
|
||||||
|
b = await mkBrain('filesystem', dTx)
|
||||||
|
ids = await seed(b)
|
||||||
|
for (let c = 0; c < churn; c++) for (let i = 0; i < n; i++) await b.transact([{ op: 'update', id: ids[i], metadata: { i, rev: c } }] as any)
|
||||||
|
await b.close()
|
||||||
|
const txBytes = dirBytes(dTx)
|
||||||
|
|
||||||
|
rmSync(dSingle, { recursive: true, force: true })
|
||||||
|
rmSync(dTx, { recursive: true, force: true })
|
||||||
|
const writes = n * churn
|
||||||
|
return { writes, singleBytes, txBytes, retainedTotal: txBytes - singleBytes, retainedPerWrite: (txBytes - singleBytes) / writes }
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---- driver ---------------------------------------------------------------
|
||||||
|
|
||||||
|
async function main() {
|
||||||
|
say(`[spike] start (scale=${SMALL ? 'small' : 'full'}); verifying no-WASM embedding...`)
|
||||||
|
// Sanity: a brain must construct + init fast (no WASM compile).
|
||||||
|
{
|
||||||
|
const t = process.hrtime.bigint()
|
||||||
|
const b = await mkBrain('memory')
|
||||||
|
await b.add(payload(0))
|
||||||
|
await b.close()
|
||||||
|
say(`[spike] brain init + 1 write OK in ${(Number(process.hrtime.bigint() - t) / 1e6).toFixed(0)}ms (must be <2000ms or WASM is loading)`)
|
||||||
|
}
|
||||||
|
if (existsSync(FS_ROOT)) rmSync(FS_ROOT, { recursive: true, force: true })
|
||||||
|
mkdirSync(FS_ROOT, { recursive: true })
|
||||||
|
|
||||||
|
const results: any = { meta: { N_MEM, N_FS, K, WARMUP, REPEATS, DIM, dim: DIM, small: SMALL }, throughput: {}, history: {} }
|
||||||
|
|
||||||
|
// Warmup (JIT/alloc) on a memory brain.
|
||||||
|
{
|
||||||
|
const b = await mkBrain('memory')
|
||||||
|
for (let i = 0; i < WARMUP; i++) await b.add(payload(i))
|
||||||
|
for (let i = 0; i < WARMUP; i += K) await b.transact([{ op: 'add', ...payload(i) }] as any)
|
||||||
|
await b.close()
|
||||||
|
}
|
||||||
|
|
||||||
|
const fmt = (t: Timing) => `${Math.round(t.perSec).toLocaleString()}/s (${t.msPerOp.toFixed(3)} ms)`
|
||||||
|
|
||||||
|
for (const backend of ['memory', 'filesystem'] as const) {
|
||||||
|
const n = backend === 'memory' ? N_MEM : N_FS
|
||||||
|
say(`\n========== ${backend.toUpperCase()} backend (N=${n.toLocaleString()}) ==========`)
|
||||||
|
|
||||||
|
// CREATE
|
||||||
|
const addSingle = await timeWorkload('add-single', createSingleOp(backend, n, 'add-single'))
|
||||||
|
const addTx1 = await timeWorkload('add-tx1', createTransact(backend, n, 1, 'add-tx1'))
|
||||||
|
const addTxK = await timeWorkload('add-txK', createTransact(backend, n, K, 'add-txK'))
|
||||||
|
say(`CREATE add() ${fmt(addSingle)}`)
|
||||||
|
say(`CREATE transact([1]) ${fmt(addTx1)} <- Model B per-write (slowdown ${(addSingle.perSec / addTx1.perSec).toFixed(2)}x)`)
|
||||||
|
say(`CREATE transact([${K}]) ${fmt(addTxK)} <- group-commit (slowdown vs add() ${(addSingle.perSec / addTxK.perSec).toFixed(2)}x)`)
|
||||||
|
|
||||||
|
// UPDATE
|
||||||
|
const updSingle = await timeWorkload('upd-single', updateSingleOp(backend, n, 'upd-single'))
|
||||||
|
const updTx1 = await timeWorkload('upd-tx1', updateTransact(backend, n, 1, 'upd-tx1'))
|
||||||
|
const updTxK = await timeWorkload('upd-txK', updateTransact(backend, n, K, 'upd-txK'))
|
||||||
|
say(`UPDATE update() ${fmt(updSingle)}`)
|
||||||
|
say(`UPDATE transact([1]) ${fmt(updTx1)} <- Model B per-write (slowdown ${(updSingle.perSec / updTx1.perSec).toFixed(2)}x)`)
|
||||||
|
say(`UPDATE transact([${K}]) ${fmt(updTxK)} <- group-commit (slowdown vs update() ${(updSingle.perSec / updTxK.perSec).toFixed(2)}x)`)
|
||||||
|
|
||||||
|
results.throughput[backend] = {
|
||||||
|
n,
|
||||||
|
create: { add: addSingle, tx1: addTx1, txK: addTxK, slowdownTx1: addSingle.perSec / addTx1.perSec, slowdownTxK: addSingle.perSec / addTxK.perSec },
|
||||||
|
update: { single: updSingle, tx1: updTx1, txK: updTxK, slowdownTx1: updSingle.perSec / updTx1.perSec, slowdownTxK: updSingle.perSec / updTxK.perSec }
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// HISTORY GROWTH (filesystem, NVMe)
|
||||||
|
say(`\n========== HISTORY GROWTH (filesystem NVMe) ==========`)
|
||||||
|
const hN = SMALL ? 500 : 3_000
|
||||||
|
const addGrowth = await historyGrowthAdd(hN)
|
||||||
|
say(`ADD-only (${hN.toLocaleString()} adds): retained history = ${(addGrowth.retainedTotal / 1024).toFixed(0)} KiB total, ${addGrowth.retainedPerWrite.toFixed(0)} bytes/write`)
|
||||||
|
const churn = SMALL ? 3 : 5
|
||||||
|
const updGrowth = await historyGrowthUpdateChurn(SMALL ? 200 : 1_000, churn)
|
||||||
|
say(`UPDATE-churn (${updGrowth.writes.toLocaleString()} updates): retained history = ${(updGrowth.retainedTotal / 1024 / 1024).toFixed(1)} MiB total, ${updGrowth.retainedPerWrite.toFixed(0)} bytes/write`)
|
||||||
|
results.history = { addGrowth, updGrowth }
|
||||||
|
|
||||||
|
// persist raw JSON for the verification workflow
|
||||||
|
const outPath = `${FS_ROOT}/../tests/benchmarks/_spike-results.json`
|
||||||
|
const fs = await import('node:fs')
|
||||||
|
fs.writeFileSync(`/media/dpsifr/storage/home/Projects/brainy/tests/benchmarks/_spike-results.json`, JSON.stringify(results, null, 2))
|
||||||
|
say(`\nRaw results → tests/benchmarks/_spike-results.json`)
|
||||||
|
|
||||||
|
// cleanup data root
|
||||||
|
if (existsSync(FS_ROOT)) rmSync(FS_ROOT, { recursive: true, force: true })
|
||||||
|
}
|
||||||
|
|
||||||
|
main()
|
||||||
|
.then(() => {
|
||||||
|
say('[spike] done.')
|
||||||
|
process.exit(0) // force exit — brain caches/timers can keep the loop alive
|
||||||
|
})
|
||||||
|
.catch((e) => {
|
||||||
|
console.error(e)
|
||||||
|
process.exit(1)
|
||||||
|
})
|
||||||
Loading…
Add table
Add a link
Reference in a new issue