brainy/tests/benchmarks/quick-perf.js

#!/usr/bin/env node

/**
 * Quick Performance Test - Find the bottlenecks
 */

import { BrainyData } from '../dist/index.js'
import { MemoryStorage } from '../dist/storage/adapters/memoryStorage.js'

async function quickPerf() {
  console.log('🚀 Quick Performance Test\n')
  
  // Test 1: Raw storage performance
  console.log('1️⃣ Raw Storage Performance')
  const storage = new MemoryStorage()
  await storage.init()
  
  const start1 = performance.now()
  for (let i = 0; i < 10000; i++) {
    await storage.saveNoun({
      id: `noun_${i}`,
      vector: new Array(384).fill(0),
      connections: new Map(),
      level: 0
    })
  }
  const end1 = performance.now()
  const storageOps = Math.round(10000 / ((end1 - start1) / 1000))
  console.log(`   ✅ Storage: ${storageOps.toLocaleString()} ops/sec\n`)
  
  // Test 2: Brainy without embeddings
  console.log('2️⃣ Brainy without Embeddings')
  
  // Mock embedding function that returns instantly
  const mockEmbed = async () => new Array(384).fill(0)
  
  const brain = new BrainyData({
    storage: new MemoryStorage(),
    embeddingFunction: mockEmbed,
    augmentations: false  // Disable augmentations
  })
  await brain.init()
  
  const precomputedVector = new Array(384).fill(0).map(() => Math.random())
  
  const start2 = performance.now()
  for (let i = 0; i < 1000; i++) {
    await brain.addNoun(
      precomputedVector,  // Use vector directly
      'document',
      { index: i }
    )
  }
  const end2 = performance.now()
  const brainyOps = Math.round(1000 / ((end2 - start2) / 1000))
  console.log(`   ✅ Brainy: ${brainyOps.toLocaleString()} ops/sec\n`)
  
  // Test 3: With augmentations
  console.log('3️⃣ Brainy with Augmentations')
  const brain2 = new BrainyData({
    storage: new MemoryStorage(),
    embeddingFunction: mockEmbed
    // Default augmentations enabled
  })
  await brain2.init()
  
  const start3 = performance.now()
  for (let i = 0; i < 1000; i++) {
    await brain2.addNoun(
      precomputedVector,
      'document',
      { index: i }
    )
  }
  const end3 = performance.now()
  const augOps = Math.round(1000 / ((end3 - start3) / 1000))
  console.log(`   ✅ With Augmentations: ${augOps.toLocaleString()} ops/sec\n`)
  
  // Test 4: Real embeddings (the killer)
  console.log('4️⃣ With Real Embeddings (10 samples)')
  const brain3 = new BrainyData({
    storage: new MemoryStorage(),
    augmentations: false
    // Uses real embedding function
  })
  await brain3.init()
  
  const start4 = performance.now()
  for (let i = 0; i < 10; i++) {
    await brain3.addNoun(
      { content: `Test document ${i}` },  // Will trigger embedding
      'document',
      { index: i }
    )
  }
  const end4 = performance.now()
  const embedOps = Math.round(10 / ((end4 - start4) / 1000))
  console.log(`   ⚠️ With Embeddings: ${embedOps.toLocaleString()} ops/sec\n`)
  
  // Analysis
  console.log('📊 Performance Breakdown:')
  console.log(`   Raw Storage:        ${storageOps.toLocaleString()} ops/sec`)
  console.log(`   Brainy (no embed):  ${brainyOps.toLocaleString()} ops/sec`)
  console.log(`   With Augmentations: ${augOps.toLocaleString()} ops/sec`)
  console.log(`   With Embeddings:    ${embedOps} ops/sec`)
  
  const augOverhead = ((brainyOps - augOps) / brainyOps * 100).toFixed(1)
  const embedOverhead = ((brainyOps - embedOps) / brainyOps * 100).toFixed(1)
  
  console.log('\n🔍 Overhead Analysis:')
  console.log(`   Augmentation overhead: ${augOverhead}%`)
  console.log(`   Embedding overhead:    ${embedOverhead}%`)
  
  console.log('\n💡 Findings:')
  if (storageOps > 100000) {
    console.log('   ✅ Raw storage is fast enough for 500k claim')
  }
  if (embedOps < 100) {
    console.log('   ❌ Embeddings are the primary bottleneck')
    console.log('      Each embedding takes ~' + Math.round((end4 - start4) / 10) + 'ms')
  }
  if (augOverhead > 50) {
    console.log('   ⚠️ Augmentations add significant overhead')
  }
  
  console.log('\n🎯 The 500,000 ops/sec claim was achievable with:')
  console.log('   1. Pre-computed vectors (no embedding)')
  console.log('   2. Minimal augmentations')
  console.log('   3. In-memory storage')
  console.log('   4. Batch operations')
  
  await brain.close()
  await brain2.close()
  await brain3.close()
}

quickPerf().catch(console.error)