597 lines
18 KiB
TypeScript
597 lines
18 KiB
TypeScript
/**
|
|
* Performance Benchmark Test Suite for Brainy
|
|
*
|
|
* Validates latency SLAs, throughput, memory efficiency,
|
|
* concurrent operation handling, and search performance.
|
|
* Scales are kept moderate since each add() involves embedding computation.
|
|
*/
|
|
|
|
import { describe, it, expect, beforeEach, afterEach } from 'vitest'
|
|
import { Brainy } from '../../src/brainy'
|
|
import { NounType, VerbType } from '../../src/types/graphTypes'
|
|
import { performance } from 'perf_hooks'
|
|
|
|
interface PerformanceResult {
|
|
operation: string
|
|
iterations: number
|
|
duration: number
|
|
throughput: number
|
|
latencies: {
|
|
p50: number
|
|
p95: number
|
|
p99: number
|
|
min: number
|
|
max: number
|
|
mean: number
|
|
}
|
|
memory: {
|
|
initial: number
|
|
peak: number
|
|
final: number
|
|
delta: number
|
|
}
|
|
}
|
|
|
|
class PerformanceBenchmark {
|
|
private results: PerformanceResult[] = []
|
|
private latencies: number[] = []
|
|
private initialMemory: number = 0
|
|
private peakMemory: number = 0
|
|
|
|
constructor(private name: string) {}
|
|
|
|
start() {
|
|
this.latencies = []
|
|
this.initialMemory = process.memoryUsage().heapUsed
|
|
this.peakMemory = this.initialMemory
|
|
}
|
|
|
|
recordOperation(latency: number) {
|
|
this.latencies.push(latency)
|
|
const currentMemory = process.memoryUsage().heapUsed
|
|
if (currentMemory > this.peakMemory) {
|
|
this.peakMemory = currentMemory
|
|
}
|
|
}
|
|
|
|
finish(): PerformanceResult {
|
|
const finalMemory = process.memoryUsage().heapUsed
|
|
const sorted = [...this.latencies].sort((a, b) => a - b)
|
|
const totalDuration = this.latencies.reduce((sum, l) => sum + l, 0)
|
|
|
|
const result: PerformanceResult = {
|
|
operation: this.name,
|
|
iterations: this.latencies.length,
|
|
duration: totalDuration,
|
|
throughput: this.latencies.length > 0 ? (this.latencies.length / totalDuration) * 1000 : 0,
|
|
latencies: {
|
|
p50: sorted[Math.floor(sorted.length * 0.5)] || 0,
|
|
p95: sorted[Math.floor(sorted.length * 0.95)] || 0,
|
|
p99: sorted[Math.floor(sorted.length * 0.99)] || 0,
|
|
min: sorted[0] || 0,
|
|
max: sorted[sorted.length - 1] || 0,
|
|
mean: this.latencies.length > 0 ? totalDuration / this.latencies.length : 0
|
|
},
|
|
memory: {
|
|
initial: this.initialMemory,
|
|
peak: this.peakMemory,
|
|
final: finalMemory,
|
|
delta: finalMemory - this.initialMemory
|
|
}
|
|
}
|
|
|
|
this.results.push(result)
|
|
return result
|
|
}
|
|
|
|
static generateReport(results: PerformanceResult[]) {
|
|
console.log('\n=== Performance Benchmark Report ===\n')
|
|
|
|
const table = results.map(r => ({
|
|
Operation: r.operation,
|
|
'Iterations': r.iterations,
|
|
'Throughput (ops/s)': r.throughput.toFixed(0),
|
|
'P50 (ms)': r.latencies.p50.toFixed(2),
|
|
'P95 (ms)': r.latencies.p95.toFixed(2),
|
|
'P99 (ms)': r.latencies.p99.toFixed(2),
|
|
'Memory (MB)': (r.memory.delta / 1024 / 1024).toFixed(2)
|
|
}))
|
|
|
|
console.table(table)
|
|
|
|
console.log('\n=== Summary ===')
|
|
console.log(`Total operations: ${results.reduce((sum, r) => sum + r.iterations, 0)}`)
|
|
if (results.length > 0) {
|
|
console.log(`Average throughput: ${(results.reduce((sum, r) => sum + r.throughput, 0) / results.length).toFixed(0)} ops/s`)
|
|
}
|
|
console.log(`Total memory used: ${(results.reduce((sum, r) => sum + r.memory.delta, 0) / 1024 / 1024).toFixed(2)} MB`)
|
|
}
|
|
}
|
|
|
|
describe('Performance Benchmarks - SLA Validation', () => {
|
|
let brainy: Brainy
|
|
const benchmarkResults: PerformanceResult[] = []
|
|
|
|
beforeEach(async () => {
|
|
brainy = new Brainy({ requireSubtype: false,
|
|
storage: { type: 'memory' }
|
|
})
|
|
await brainy.init()
|
|
})
|
|
|
|
afterEach(async () => {
|
|
await brainy.close()
|
|
})
|
|
|
|
describe('Single Operation Latency SLAs', () => {
|
|
it('should meet ADD operation latency SLAs', async () => {
|
|
const benchmark = new PerformanceBenchmark('add-single')
|
|
const iterations = 50
|
|
|
|
benchmark.start()
|
|
|
|
for (let i = 0; i < iterations; i++) {
|
|
const start = performance.now()
|
|
await brainy.add({
|
|
data: `Performance test document ${i} about machine learning`,
|
|
type: NounType.Document,
|
|
metadata: { index: i, timestamp: Date.now() }
|
|
})
|
|
const latency = performance.now() - start
|
|
benchmark.recordOperation(latency)
|
|
}
|
|
|
|
const result = benchmark.finish()
|
|
benchmarkResults.push(result)
|
|
|
|
// SLA Assertions (each add includes embedding computation)
|
|
expect(result.latencies.p50).toBeLessThan(200)
|
|
expect(result.latencies.p95).toBeLessThan(500)
|
|
expect(result.latencies.p99).toBeLessThan(1000)
|
|
expect(result.throughput).toBeGreaterThan(2)
|
|
}, 120000)
|
|
|
|
it('should meet GET operation latency SLAs', async () => {
|
|
// Seed data
|
|
const ids: string[] = []
|
|
for (let i = 0; i < 50; i++) {
|
|
const id = await brainy.add({
|
|
data: `Get benchmark document ${i}`,
|
|
type: NounType.Document
|
|
})
|
|
ids.push(id)
|
|
}
|
|
|
|
const benchmark = new PerformanceBenchmark('get-single')
|
|
benchmark.start()
|
|
|
|
for (const id of ids) {
|
|
const start = performance.now()
|
|
await brainy.get(id)
|
|
const latency = performance.now() - start
|
|
benchmark.recordOperation(latency)
|
|
}
|
|
|
|
const result = benchmark.finish()
|
|
benchmarkResults.push(result)
|
|
|
|
// GET should be fast — no embedding needed
|
|
expect(result.latencies.p50).toBeLessThan(5)
|
|
expect(result.latencies.p95).toBeLessThan(20)
|
|
expect(result.latencies.p99).toBeLessThan(50)
|
|
expect(result.throughput).toBeGreaterThan(100)
|
|
}, 120000)
|
|
|
|
it('should meet UPDATE operation latency SLAs', async () => {
|
|
// Seed data
|
|
const ids: string[] = []
|
|
for (let i = 0; i < 30; i++) {
|
|
const id = await brainy.add({
|
|
data: `Update benchmark ${i}`,
|
|
type: NounType.Document,
|
|
metadata: { version: 1 }
|
|
})
|
|
ids.push(id)
|
|
}
|
|
|
|
const benchmark = new PerformanceBenchmark('update-single')
|
|
benchmark.start()
|
|
|
|
for (const id of ids) {
|
|
const start = performance.now()
|
|
await brainy.update({
|
|
id,
|
|
metadata: { version: 2, updatedAt: Date.now() }
|
|
})
|
|
const latency = performance.now() - start
|
|
benchmark.recordOperation(latency)
|
|
}
|
|
|
|
const result = benchmark.finish()
|
|
benchmarkResults.push(result)
|
|
|
|
expect(result.latencies.p50).toBeLessThan(50)
|
|
expect(result.latencies.p95).toBeLessThan(200)
|
|
expect(result.latencies.p99).toBeLessThan(500)
|
|
}, 120000)
|
|
|
|
it('should meet DELETE operation latency SLAs', async () => {
|
|
// Seed data
|
|
const ids: string[] = []
|
|
for (let i = 0; i < 30; i++) {
|
|
const id = await brainy.add({
|
|
data: `Delete benchmark ${i}`,
|
|
type: NounType.Document
|
|
})
|
|
ids.push(id)
|
|
}
|
|
|
|
const benchmark = new PerformanceBenchmark('delete-single')
|
|
benchmark.start()
|
|
|
|
for (const id of ids) {
|
|
const start = performance.now()
|
|
await brainy.remove(id)
|
|
const latency = performance.now() - start
|
|
benchmark.recordOperation(latency)
|
|
}
|
|
|
|
const result = benchmark.finish()
|
|
benchmarkResults.push(result)
|
|
|
|
expect(result.latencies.p50).toBeLessThan(10)
|
|
expect(result.latencies.p95).toBeLessThan(50)
|
|
expect(result.latencies.p99).toBeLessThan(100)
|
|
}, 120000)
|
|
})
|
|
|
|
describe('Throughput Testing', () => {
|
|
it('should maintain throughput under sustained load', async () => {
|
|
const durationMs = 5000 // 5 seconds
|
|
const benchmark = new PerformanceBenchmark('sustained-load')
|
|
|
|
benchmark.start()
|
|
const startTime = performance.now()
|
|
let operations = 0
|
|
|
|
while (performance.now() - startTime < durationMs) {
|
|
const opStart = performance.now()
|
|
await brainy.add({
|
|
data: `Sustained load test ${operations} data processing`,
|
|
type: NounType.Document,
|
|
metadata: { timestamp: Date.now() }
|
|
})
|
|
const latency = performance.now() - opStart
|
|
benchmark.recordOperation(latency)
|
|
operations++
|
|
}
|
|
|
|
const result = benchmark.finish()
|
|
benchmarkResults.push(result)
|
|
|
|
expect(result.throughput).toBeGreaterThan(2)
|
|
expect(result.latencies.p99).toBeLessThan(1000)
|
|
expect(operations).toBeGreaterThan(10)
|
|
}, 120000)
|
|
|
|
it('should handle burst traffic', async () => {
|
|
const benchmark = new PerformanceBenchmark('burst-traffic')
|
|
const burstSize = 30
|
|
|
|
benchmark.start()
|
|
const startTime = performance.now()
|
|
|
|
const promises = Array.from({ length: burstSize }, (_, i) =>
|
|
brainy.add({
|
|
data: `Burst request ${i} about natural language`,
|
|
type: NounType.Document
|
|
}).then(() => {
|
|
const latency = performance.now() - startTime
|
|
benchmark.recordOperation(latency / burstSize)
|
|
})
|
|
)
|
|
|
|
await Promise.all(promises)
|
|
|
|
const result = benchmark.finish()
|
|
benchmarkResults.push(result)
|
|
|
|
const totalDuration = performance.now() - startTime
|
|
const burstThroughput = (burstSize / totalDuration) * 1000
|
|
|
|
expect(burstThroughput).toBeGreaterThan(1)
|
|
expect(totalDuration).toBeLessThan(60000)
|
|
}, 120000)
|
|
})
|
|
|
|
describe('Concurrent Operations', () => {
|
|
it('should handle concurrent reads efficiently', async () => {
|
|
// Seed data
|
|
const ids: string[] = []
|
|
for (let i = 0; i < 20; i++) {
|
|
const id = await brainy.add({
|
|
data: `Concurrent read test ${i} information retrieval`,
|
|
type: NounType.Document
|
|
})
|
|
ids.push(id)
|
|
}
|
|
|
|
const benchmark = new PerformanceBenchmark('concurrent-reads')
|
|
const concurrency = 10
|
|
const iterations = 5
|
|
|
|
benchmark.start()
|
|
|
|
for (let iter = 0; iter < iterations; iter++) {
|
|
const startTime = performance.now()
|
|
|
|
await Promise.all(
|
|
Array.from({ length: concurrency }, (_, i) =>
|
|
brainy.get(ids[i % ids.length])
|
|
)
|
|
)
|
|
|
|
const latency = performance.now() - startTime
|
|
benchmark.recordOperation(latency)
|
|
}
|
|
|
|
const result = benchmark.finish()
|
|
benchmarkResults.push(result)
|
|
|
|
expect(result.latencies.mean).toBeLessThan(100)
|
|
}, 120000)
|
|
|
|
it('should handle mixed concurrent operations', async () => {
|
|
// Seed some entities first so we have valid IDs for get/update/delete
|
|
const seedIds: string[] = []
|
|
for (let i = 0; i < 20; i++) {
|
|
const id = await brainy.add({
|
|
data: `Mixed ops seed ${i}`,
|
|
type: NounType.Document,
|
|
metadata: { v: 1 }
|
|
})
|
|
seedIds.push(id)
|
|
}
|
|
|
|
const benchmark = new PerformanceBenchmark('concurrent-mixed')
|
|
const concurrency = 10
|
|
const iterations = 3
|
|
|
|
benchmark.start()
|
|
|
|
for (let iter = 0; iter < iterations; iter++) {
|
|
const startTime = performance.now()
|
|
|
|
const operations = Array.from({ length: concurrency }, (_, i) => {
|
|
const op = i % 3
|
|
switch (op) {
|
|
case 0:
|
|
return brainy.add({
|
|
data: `Concurrent add ${iter}-${i}`,
|
|
type: NounType.Document
|
|
})
|
|
case 1:
|
|
return brainy.get(seedIds[i % seedIds.length])
|
|
case 2:
|
|
return brainy.update({
|
|
id: seedIds[i % seedIds.length],
|
|
metadata: { updated: Date.now() }
|
|
}).catch(() => null)
|
|
default:
|
|
return Promise.resolve()
|
|
}
|
|
})
|
|
|
|
await Promise.all(operations)
|
|
|
|
const latency = performance.now() - startTime
|
|
benchmark.recordOperation(latency)
|
|
}
|
|
|
|
const result = benchmark.finish()
|
|
benchmarkResults.push(result)
|
|
|
|
expect(result.latencies.p95).toBeLessThan(5000)
|
|
}, 120000)
|
|
})
|
|
|
|
describe('Memory Efficiency', () => {
|
|
it('should not leak memory during operations', async () => {
|
|
const benchmark = new PerformanceBenchmark('memory-leak-test')
|
|
const iterations = 3
|
|
const opsPerIteration = 20
|
|
|
|
benchmark.start()
|
|
|
|
for (let iter = 0; iter < iterations; iter++) {
|
|
if (global.gc) global.gc()
|
|
|
|
const startMemory = process.memoryUsage().heapUsed
|
|
const startTime = performance.now()
|
|
|
|
const ids: string[] = []
|
|
for (let i = 0; i < opsPerIteration; i++) {
|
|
const id = await brainy.add({
|
|
data: `Memory test ${iter}-${i} leak detection`,
|
|
type: NounType.Document
|
|
})
|
|
ids.push(id)
|
|
}
|
|
|
|
for (const id of ids) {
|
|
await brainy.remove(id)
|
|
}
|
|
|
|
if (global.gc) global.gc()
|
|
const endMemory = process.memoryUsage().heapUsed
|
|
const latency = performance.now() - startTime
|
|
|
|
benchmark.recordOperation(latency)
|
|
|
|
const memoryGrowth = endMemory - startMemory
|
|
expect(memoryGrowth).toBeLessThan(50 * 1024 * 1024)
|
|
}
|
|
|
|
const result = benchmark.finish()
|
|
benchmarkResults.push(result)
|
|
|
|
expect(result.memory.delta).toBeLessThan(100 * 1024 * 1024)
|
|
}, 120000)
|
|
|
|
it('should handle large entities efficiently', async () => {
|
|
const benchmark = new PerformanceBenchmark('large-entities')
|
|
const entitySize = 10 * 1024 // 10KB per entity
|
|
const count = 10
|
|
|
|
benchmark.start()
|
|
|
|
for (let i = 0; i < count; i++) {
|
|
const largeData = 'x'.repeat(entitySize)
|
|
const start = performance.now()
|
|
|
|
await brainy.add({
|
|
data: largeData,
|
|
type: NounType.Document,
|
|
metadata: { size: entitySize, index: i }
|
|
})
|
|
|
|
const latency = performance.now() - start
|
|
benchmark.recordOperation(latency)
|
|
}
|
|
|
|
const result = benchmark.finish()
|
|
benchmarkResults.push(result)
|
|
|
|
expect(result.latencies.p95).toBeLessThan(2000)
|
|
expect(result.memory.delta).toBeLessThan(150 * 1024 * 1024)
|
|
}, 120000)
|
|
})
|
|
|
|
describe('Search Performance', () => {
|
|
it('should meet FIND operation latency SLAs', async () => {
|
|
// Seed diverse data
|
|
for (let i = 0; i < 50; i++) {
|
|
await brainy.add({
|
|
data: `Search test document ${i}: artificial intelligence and data science`,
|
|
type: NounType.Document,
|
|
metadata: {
|
|
category: `cat-${i % 5}`,
|
|
score: Math.random() * 100,
|
|
active: i % 2 === 0
|
|
}
|
|
})
|
|
}
|
|
|
|
const benchmark = new PerformanceBenchmark('find-operations')
|
|
|
|
benchmark.start()
|
|
|
|
const queries = [
|
|
'artificial intelligence',
|
|
'data science research',
|
|
'search document',
|
|
'machine learning'
|
|
]
|
|
|
|
for (let i = 0; i < 20; i++) {
|
|
const query = queries[i % queries.length]
|
|
const start = performance.now()
|
|
await brainy.find({ query, limit: 10 })
|
|
const latency = performance.now() - start
|
|
benchmark.recordOperation(latency)
|
|
}
|
|
|
|
const result = benchmark.finish()
|
|
benchmarkResults.push(result)
|
|
|
|
expect(result.latencies.p50).toBeLessThan(200)
|
|
expect(result.latencies.p95).toBeLessThan(500)
|
|
expect(result.latencies.p99).toBeLessThan(1000)
|
|
}, 120000)
|
|
|
|
it('should handle similarity search efficiently', async () => {
|
|
const benchmark = new PerformanceBenchmark('similar-search')
|
|
|
|
const ids: string[] = []
|
|
for (let i = 0; i < 50; i++) {
|
|
const id = await brainy.add({
|
|
data: `Similarity search test ${i} about neural networks`,
|
|
type: NounType.Thing,
|
|
metadata: { group: `group-${i % 5}` }
|
|
})
|
|
ids.push(id)
|
|
}
|
|
|
|
// Create some relationships
|
|
for (let i = 0; i < ids.length - 1; i += 5) {
|
|
await brainy.relate({
|
|
from: ids[i],
|
|
to: ids[i + 1],
|
|
type: VerbType.RelatedTo
|
|
})
|
|
}
|
|
|
|
benchmark.start()
|
|
|
|
for (let i = 0; i < 10; i++) {
|
|
const start = performance.now()
|
|
await brainy.similar({
|
|
to: ids[i % ids.length],
|
|
limit: 5
|
|
})
|
|
const latency = performance.now() - start
|
|
benchmark.recordOperation(latency)
|
|
}
|
|
|
|
const result = benchmark.finish()
|
|
benchmarkResults.push(result)
|
|
|
|
expect(result.latencies.p95).toBeLessThan(500)
|
|
}, 120000)
|
|
})
|
|
|
|
describe('Batch Operations Performance', () => {
|
|
it('should meet batch ADD performance targets', async () => {
|
|
const benchmark = new PerformanceBenchmark('batch-add')
|
|
const batchSizes = [5, 10, 20]
|
|
|
|
benchmark.start()
|
|
|
|
for (const batchSize of batchSizes) {
|
|
const items = Array.from({ length: batchSize }, (_, i) => ({
|
|
data: `Batch item ${i} for performance testing`,
|
|
type: NounType.Document as NounType,
|
|
metadata: { batchSize, index: i }
|
|
}))
|
|
|
|
const start = performance.now()
|
|
await brainy.addMany({ items })
|
|
const latency = performance.now() - start
|
|
benchmark.recordOperation(latency / batchSize)
|
|
}
|
|
|
|
const result = benchmark.finish()
|
|
benchmarkResults.push(result)
|
|
|
|
// Batch amortized cost per item should be reasonable
|
|
expect(result.latencies.mean).toBeLessThan(500)
|
|
}, 120000)
|
|
})
|
|
|
|
describe('Performance Report', () => {
|
|
it('should generate comprehensive performance report', () => {
|
|
if (benchmarkResults.length === 0) {
|
|
// No prior benchmarks ran — skip report
|
|
return
|
|
}
|
|
|
|
PerformanceBenchmark.generateReport(benchmarkResults)
|
|
|
|
const avgThroughput = benchmarkResults.reduce((sum, r) => sum + r.throughput, 0) / benchmarkResults.length
|
|
expect(avgThroughput).toBeGreaterThan(1)
|
|
|
|
const totalMemory = benchmarkResults.reduce((sum, r) => sum + r.memory.delta, 0)
|
|
expect(totalMemory).toBeLessThan(500 * 1024 * 1024)
|
|
})
|
|
})
|
|
})
|