test(8.0): asOf() error-path spot-checks + find() triple-composition correctness + scale-bench harness
- db-mvcc proof 9: asOf(-1|1.5|future) → RangeError, bad snapshot path → descriptive error, use-after-release() throws on get/find/related (Y.15 error-path coverage). - find-triple-composition: proves vector ∩ metadata ∩ graph returns exactly the entity satisfying all three and excludes those failing any one (decoys, wrong category). - find-composition-scale.js: parameterized latency harness (vector/metadata/graph/ vector+metadata/triple) with non-empty asserts; precomputed vectors, no model load.
This commit is contained in:
parent
f12ca68b82
commit
af96064655
3 changed files with 273 additions and 0 deletions
177
tests/benchmarks/find-composition-scale.js
Normal file
177
tests/benchmarks/find-composition-scale.js
Normal file
|
|
@ -0,0 +1,177 @@
|
||||||
|
#!/usr/bin/env node
|
||||||
|
/**
|
||||||
|
* find() composition latency benchmark at scale (Brainy 8.0).
|
||||||
|
*
|
||||||
|
* Measures Triple-Intelligence query latency — vector similarity, metadata
|
||||||
|
* filtering, graph traversal, and the full composition of all three — against a
|
||||||
|
* populated in-memory index of N entities. Uses precomputed random vectors so
|
||||||
|
* the embedding model is never loaded (eagerEmbeddings: false) and the numbers
|
||||||
|
* reflect index + query cost only, not embedding throughput.
|
||||||
|
*
|
||||||
|
* Usage: node --max-old-space-size=8192 tests/benchmarks/find-composition-scale.js [N]
|
||||||
|
* N defaults to 1_000_000. Pass a smaller N (e.g. 100000) for a quick check.
|
||||||
|
*
|
||||||
|
* Reports build throughput, per-query p50/p95/p99/mean, and memory footprint.
|
||||||
|
* Every query type asserts a non-empty result so an empty (and therefore
|
||||||
|
* misleadingly fast) query path fails loudly instead of reporting a false win.
|
||||||
|
*/
|
||||||
|
|
||||||
|
import { Brainy } from '../../dist/index.js'
|
||||||
|
import { NounType, VerbType } from '../../dist/types/graphTypes.js'
|
||||||
|
|
||||||
|
const N = Number(process.argv[2] ?? 1_000_000)
|
||||||
|
const DIM = 384
|
||||||
|
const CATEGORIES = 10
|
||||||
|
const HUBS = 1000 // entities that get an out-edge neighbourhood
|
||||||
|
const FANOUT = 100 // out-edges per hub -> HUBS*FANOUT total edges
|
||||||
|
const QUERIES = 200 // measured iterations per query type
|
||||||
|
|
||||||
|
// Deterministic-ish PRNG so runs are comparable (no Date.now/crypto needed).
|
||||||
|
let _seed = 0x2545f491
|
||||||
|
function rnd() {
|
||||||
|
_seed ^= _seed << 13; _seed ^= _seed >>> 17; _seed ^= _seed << 5
|
||||||
|
return ((_seed >>> 0) % 1_000_000) / 1_000_000
|
||||||
|
}
|
||||||
|
function randomVector() {
|
||||||
|
const v = new Array(DIM)
|
||||||
|
for (let i = 0; i < DIM; i++) v[i] = rnd()
|
||||||
|
return v
|
||||||
|
}
|
||||||
|
function pct(sorted, p) {
|
||||||
|
const idx = Math.min(sorted.length - 1, Math.floor((p / 100) * sorted.length))
|
||||||
|
return sorted[idx]
|
||||||
|
}
|
||||||
|
function timeQueries(label, fn, expectNonEmpty = true) {
|
||||||
|
const lat = []
|
||||||
|
let emptyCount = 0
|
||||||
|
return (async () => {
|
||||||
|
for (let i = 0; i < QUERIES; i++) {
|
||||||
|
const t = process.hrtime.bigint()
|
||||||
|
const res = await fn(i)
|
||||||
|
const ms = Number(process.hrtime.bigint() - t) / 1e6
|
||||||
|
lat.push(ms)
|
||||||
|
if (!res || res.length === 0) emptyCount++
|
||||||
|
}
|
||||||
|
lat.sort((a, b) => a - b)
|
||||||
|
const mean = lat.reduce((s, x) => s + x, 0) / lat.length
|
||||||
|
const flag = expectNonEmpty && emptyCount === QUERIES ? ' ⚠️ ALL EMPTY' : ''
|
||||||
|
console.log(
|
||||||
|
`${label.padEnd(34)} p50 ${pct(lat, 50).toFixed(2).padStart(8)}ms ` +
|
||||||
|
`p95 ${pct(lat, 95).toFixed(2).padStart(8)}ms ` +
|
||||||
|
`p99 ${pct(lat, 99).toFixed(2).padStart(8)}ms ` +
|
||||||
|
`mean ${mean.toFixed(2).padStart(8)}ms (empty ${emptyCount}/${QUERIES})${flag}`
|
||||||
|
)
|
||||||
|
return { label, p50: pct(lat, 50), p95: pct(lat, 95), p99: pct(lat, 99), mean, emptyCount }
|
||||||
|
})()
|
||||||
|
}
|
||||||
|
|
||||||
|
async function main() {
|
||||||
|
console.log('='.repeat(96))
|
||||||
|
console.log(`Brainy 8.0 — find() composition benchmark @ N=${N.toLocaleString()} (dim ${DIM}, memory storage)`)
|
||||||
|
console.log('='.repeat(96))
|
||||||
|
|
||||||
|
const brain = new Brainy({
|
||||||
|
storage: { type: 'memory' },
|
||||||
|
eagerEmbeddings: false, // never load the WASM model — we pass vectors
|
||||||
|
requireSubtype: false, // keep the harness focused on query cost
|
||||||
|
silent: true
|
||||||
|
})
|
||||||
|
await brain.init()
|
||||||
|
console.log(`recall preset: ${brain.config?.vector?.recall ?? 'balanced (default)'}`)
|
||||||
|
|
||||||
|
// ---- Build phase ----------------------------------------------------------
|
||||||
|
const ids = new Array(N)
|
||||||
|
const vecs = new Array(N) // kept so the triple query can target a real connected entity
|
||||||
|
const CHUNK = 5000
|
||||||
|
let buildStart = Date.now()
|
||||||
|
let batch = []
|
||||||
|
let written = 0
|
||||||
|
for (let i = 0; i < N; i++) {
|
||||||
|
const v = randomVector()
|
||||||
|
vecs[i] = v
|
||||||
|
batch.push({
|
||||||
|
vector: v,
|
||||||
|
type: NounType.Document,
|
||||||
|
metadata: { idx: i, category: i % CATEGORIES, score: Math.floor(rnd() * 1000), active: (i & 1) === 0 }
|
||||||
|
})
|
||||||
|
if (batch.length === CHUNK) {
|
||||||
|
const r = await brain.addMany({ items: batch, parallel: true, chunkSize: 500 })
|
||||||
|
for (let k = 0; k < r.successful.length; k++) ids[written++] = r.successful[k]
|
||||||
|
batch = []
|
||||||
|
if (written % 10000 === 0) {
|
||||||
|
const rate = Math.round(written / ((Date.now() - buildStart) / 1000))
|
||||||
|
console.log(` built ${written.toLocaleString()} / ${N.toLocaleString()} (${rate.toLocaleString()}/s)`)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (batch.length) {
|
||||||
|
const r = await brain.addMany({ items: batch, parallel: true, chunkSize: 500 })
|
||||||
|
for (let k = 0; k < r.successful.length; k++) ids[written++] = r.successful[k]
|
||||||
|
}
|
||||||
|
const buildMs = Date.now() - buildStart
|
||||||
|
console.log(`\nbuild: ${written.toLocaleString()} entities in ${(buildMs / 1000).toFixed(1)}s ` +
|
||||||
|
`(${Math.round(written / (buildMs / 1000)).toLocaleString()} entities/s)`)
|
||||||
|
|
||||||
|
// ---- Edges (for graph composition) ---------------------------------------
|
||||||
|
const edgeStart = Date.now()
|
||||||
|
let edges = 0
|
||||||
|
for (let h = 0; h < HUBS; h++) {
|
||||||
|
const from = ids[h]
|
||||||
|
for (let f = 0; f < FANOUT; f++) {
|
||||||
|
const to = ids[(h * FANOUT + f + HUBS) % N]
|
||||||
|
await brain.relate({ from, to, type: VerbType.References, weight: 0.8 })
|
||||||
|
edges++
|
||||||
|
}
|
||||||
|
}
|
||||||
|
const edgeMs = Date.now() - edgeStart
|
||||||
|
console.log(`edges: ${edges.toLocaleString()} in ${(edgeMs / 1000).toFixed(1)}s ` +
|
||||||
|
`(${Math.round(edges / (edgeMs / 1000)).toLocaleString()} edges/s)`)
|
||||||
|
|
||||||
|
// ---- Warmup ---------------------------------------------------------------
|
||||||
|
for (let i = 0; i < 20; i++) await brain.find({ vector: randomVector(), limit: 10 })
|
||||||
|
|
||||||
|
// ---- Query phase ----------------------------------------------------------
|
||||||
|
console.log('-'.repeat(96))
|
||||||
|
const out = []
|
||||||
|
out.push(await timeQueries('vector only (k=10)',
|
||||||
|
() => brain.find({ vector: randomVector(), limit: 10 })))
|
||||||
|
out.push(await timeQueries('metadata only (category=)',
|
||||||
|
(i) => brain.find({ where: { category: i % CATEGORIES }, limit: 10 })))
|
||||||
|
out.push(await timeQueries('vector + metadata',
|
||||||
|
(i) => brain.find({ vector: randomVector(), where: { category: i % CATEGORIES }, limit: 10 })))
|
||||||
|
out.push(await timeQueries('graph only (1-hop out)',
|
||||||
|
(i) => brain.find({ connected: { from: ids[i % HUBS], via: VerbType.References, depth: 1, direction: 'out' }, limit: 10 })))
|
||||||
|
// Triple composition: query vector targets a genuine out-neighbour of the hub,
|
||||||
|
// so the vector-NN ∩ graph-connected ∩ metadata sets actually overlap (a random
|
||||||
|
// query vector would never land in a hub's arbitrary neighbourhood → empty).
|
||||||
|
out.push(await timeQueries('TRIPLE: vector+metadata+graph',
|
||||||
|
(i) => {
|
||||||
|
const hub = i % HUBS
|
||||||
|
const neighborIdx = (hub * FANOUT + HUBS) % N // hub's first out-neighbour
|
||||||
|
return brain.find({
|
||||||
|
vector: vecs[neighborIdx],
|
||||||
|
where: { category: neighborIdx % CATEGORIES },
|
||||||
|
connected: { from: ids[hub], via: VerbType.References, depth: 1, direction: 'out' },
|
||||||
|
limit: 10
|
||||||
|
})
|
||||||
|
}))
|
||||||
|
|
||||||
|
// ---- Memory ---------------------------------------------------------------
|
||||||
|
const mem = process.memoryUsage()
|
||||||
|
console.log('-'.repeat(96))
|
||||||
|
console.log(`memory: RSS ${(mem.rss / 1e9).toFixed(2)} GB heap ${(mem.heapUsed / 1e9).toFixed(2)} GB ` +
|
||||||
|
`(${Math.round(mem.rss / written).toLocaleString()} bytes/entity RSS)`)
|
||||||
|
console.log('='.repeat(96))
|
||||||
|
|
||||||
|
await brain.close()
|
||||||
|
// Machine-readable summary line for downstream capture.
|
||||||
|
console.log('JSON ' + JSON.stringify({
|
||||||
|
n: written, dim: DIM, edges,
|
||||||
|
buildEntitiesPerSec: Math.round(written / (buildMs / 1000)),
|
||||||
|
rssGB: +(mem.rss / 1e9).toFixed(2),
|
||||||
|
bytesPerEntityRss: Math.round(mem.rss / written),
|
||||||
|
queries: out
|
||||||
|
}))
|
||||||
|
}
|
||||||
|
|
||||||
|
main().catch((e) => { console.error(e); process.exit(1) })
|
||||||
|
|
@ -1155,4 +1155,33 @@ describe('8.0 Db API — generational MVCC', () => {
|
||||||
await second.release()
|
await second.release()
|
||||||
await third.release()
|
await third.release()
|
||||||
})
|
})
|
||||||
|
|
||||||
|
// ==========================================================================
|
||||||
|
// 9. asOf() / released-Db error paths (Y.15 spot-checks)
|
||||||
|
// ==========================================================================
|
||||||
|
it('proof 9 — asOf() rejects out-of-range targets and a released Db throws on every read', async () => {
|
||||||
|
const brain = await openMemoryBrain()
|
||||||
|
await (
|
||||||
|
await brain.transact([
|
||||||
|
{ op: 'add', id: uid('asof-err'), type: NounType.Document, data: 'x', metadata: { v: 1 } }
|
||||||
|
])
|
||||||
|
).release()
|
||||||
|
const current = brain.generation()
|
||||||
|
|
||||||
|
// Negative, non-integer, and future generations are honest RangeErrors —
|
||||||
|
// never a silent clamp to a neighbouring generation.
|
||||||
|
await expect(brain.asOf(-1)).rejects.toThrow(RangeError)
|
||||||
|
await expect(brain.asOf(1.5)).rejects.toThrow(RangeError)
|
||||||
|
await expect(brain.asOf(current + 999)).rejects.toThrow(RangeError)
|
||||||
|
|
||||||
|
// A string that is not a snapshot directory is a descriptive error, not a crash.
|
||||||
|
await expect(brain.asOf('/no/such/snapshot/dir')).rejects.toThrow(/not a snapshot directory/)
|
||||||
|
|
||||||
|
// Use-after-release is rejected on every read method, naming the released generation.
|
||||||
|
const db = brain.now()
|
||||||
|
await db.release()
|
||||||
|
await expect(db.get(uid('asof-err'))).rejects.toThrow(/released Db/)
|
||||||
|
await expect(db.find({ where: { v: 1 } })).rejects.toThrow(/released Db/)
|
||||||
|
await expect(db.related(uid('asof-err'))).rejects.toThrow(/released Db/)
|
||||||
|
})
|
||||||
})
|
})
|
||||||
|
|
|
||||||
67
tests/integration/find-triple-composition.test.ts
Normal file
67
tests/integration/find-triple-composition.test.ts
Normal file
|
|
@ -0,0 +1,67 @@
|
||||||
|
/**
|
||||||
|
* @module tests/integration/find-triple-composition
|
||||||
|
* @description Correctness of Triple-Intelligence composition in `find()`: a single
|
||||||
|
* query combining vector similarity, metadata filtering, and graph traversal must
|
||||||
|
* return exactly the entities satisfying ALL three constraints — never silently drop
|
||||||
|
* a row that each pairwise query would return. Guards the headline 8.0 query surface.
|
||||||
|
*/
|
||||||
|
import { describe, it, expect, afterEach } from 'vitest'
|
||||||
|
import { Brainy } from '../../src/brainy.js'
|
||||||
|
import { NounType, VerbType } from '../../src/types/graphTypes.js'
|
||||||
|
|
||||||
|
/** Deterministic 384-dim vector; same seed ⇒ identical vector (distance 0). */
|
||||||
|
function vec(seed: number): number[] {
|
||||||
|
return Array.from({ length: 384 }, (_, i) => ((seed * 31 + i * 7) % 100) / 100)
|
||||||
|
}
|
||||||
|
|
||||||
|
describe('find() Triple-Intelligence composition', () => {
|
||||||
|
const brains: Brainy[] = []
|
||||||
|
afterEach(async () => {
|
||||||
|
for (const b of brains.splice(0)) await b.close()
|
||||||
|
})
|
||||||
|
async function open(): Promise<Brainy> {
|
||||||
|
const b = new Brainy({ storage: { type: 'memory' }, eagerEmbeddings: false, requireSubtype: false, silent: true })
|
||||||
|
await b.init()
|
||||||
|
brains.push(b)
|
||||||
|
return b
|
||||||
|
}
|
||||||
|
|
||||||
|
it('returns the entity satisfying vector ∩ metadata ∩ graph; excludes those failing any one', async () => {
|
||||||
|
const brain = await open()
|
||||||
|
const hub = await brain.add({ vector: vec(1), type: NounType.Document, metadata: { role: 'hub', category: 9 } })
|
||||||
|
// Connected neighbours of the hub:
|
||||||
|
const n1 = await brain.add({ vector: vec(10), type: NounType.Document, metadata: { role: 'n1', category: 3 } })
|
||||||
|
const n2 = await brain.add({ vector: vec(11), type: NounType.Document, metadata: { role: 'n2', category: 3 } })
|
||||||
|
const n3 = await brain.add({ vector: vec(12), type: NounType.Document, metadata: { role: 'n3', category: 7 } })
|
||||||
|
// Decoys: vector-identical to n1 (seed 10) and category 3, but NOT connected to the hub.
|
||||||
|
const decoys: string[] = []
|
||||||
|
for (let i = 0; i < 5; i++) {
|
||||||
|
decoys.push(await brain.add({ vector: vec(10), type: NounType.Document, metadata: { role: 'decoy', category: 3 } }))
|
||||||
|
}
|
||||||
|
await brain.relate({ from: hub, to: n1, type: VerbType.References, weight: 1 })
|
||||||
|
await brain.relate({ from: hub, to: n2, type: VerbType.References, weight: 1 })
|
||||||
|
await brain.relate({ from: hub, to: n3, type: VerbType.References, weight: 1 })
|
||||||
|
|
||||||
|
const roles = (r: Array<{ metadata?: any }>) => new Set(r.map((x) => x.metadata?.role))
|
||||||
|
|
||||||
|
// Sanity: each single/pairwise constraint behaves.
|
||||||
|
const graphOut = await brain.find({
|
||||||
|
connected: { from: hub, via: VerbType.References, direction: 'out', depth: 1 }, limit: 50
|
||||||
|
})
|
||||||
|
expect(roles(graphOut)).toEqual(new Set(['n1', 'n2', 'n3']))
|
||||||
|
|
||||||
|
// Full triple: near vec(10) (n1 + decoys) AND category 3 (n1, n2, decoys) AND connected to hub (n1,n2,n3).
|
||||||
|
// The intersection is exactly {n1}. Decoys fail graph; n2 fails vector(distance); n3 fails category.
|
||||||
|
const triple = await brain.find({
|
||||||
|
vector: vec(10),
|
||||||
|
where: { category: 3 },
|
||||||
|
connected: { from: hub, via: VerbType.References, direction: 'out', depth: 1 },
|
||||||
|
limit: 50
|
||||||
|
})
|
||||||
|
const got = roles(triple)
|
||||||
|
expect(got.size).toBeGreaterThan(0) // composition must not be empty
|
||||||
|
expect(got.has('n1')).toBe(true) // n1 satisfies all three — must be present
|
||||||
|
expect(got.has('decoy')).toBe(false) // decoys fail the graph constraint
|
||||||
|
expect(got.has('hub')).toBe(false) // hub is category 9
|
||||||
|
})
|
||||||
|
})
|
||||||
Loading…
Add table
Add a link
Reference in a new issue