feat: implement clean embedding architecture with Q8/FP32 precision control

- Unified embedding system with single EmbeddingManager
- Q8 model support with 75% smaller footprint (23MB vs 90MB)
- Intelligent precision auto-selection based on environment
- Clean cached embeddings with TTL and memory management
- Zero-config setup with smart defaults
- Complete storage structure documentation
- Removed legacy worker and hybrid managers
- Streamlined model configuration and precision management

src/embeddings/lightweight-embedder.ts

@@ -1,153 +0,0 @@
-/**
- * Lightweight Embedding Alternative
- * 
- * Uses pre-computed embeddings for common terms
- * Falls back to ONNX for unknown terms
- * 
- * This reduces memory usage by 90% for typical queries
- */
-
-import { Vector } from '../coreTypes.js'
-
-// Pre-computed embeddings for top 10,000 common terms
-// In production, this would be loaded from a file
-const PRECOMPUTED_EMBEDDINGS: Record<string, Vector> = {
-  // Programming languages
-  'javascript': new Array(384).fill(0).map((_, i) => Math.sin(i * 0.1)),
-  'python': new Array(384).fill(0).map((_, i) => Math.cos(i * 0.1)),
-  'typescript': new Array(384).fill(0).map((_, i) => Math.sin(i * 0.15)),
-  'java': new Array(384).fill(0).map((_, i) => Math.cos(i * 0.15)),
-  'rust': new Array(384).fill(0).map((_, i) => Math.sin(i * 0.2)),
-  'go': new Array(384).fill(0).map((_, i) => Math.cos(i * 0.2)),
-  
-  // Frameworks
-  'react': new Array(384).fill(0).map((_, i) => Math.sin(i * 0.25)),
-  'vue': new Array(384).fill(0).map((_, i) => Math.cos(i * 0.25)),
-  'angular': new Array(384).fill(0).map((_, i) => Math.sin(i * 0.3)),
-  'svelte': new Array(384).fill(0).map((_, i) => Math.cos(i * 0.3)),
-  
-  // Databases
-  'postgresql': new Array(384).fill(0).map((_, i) => Math.sin(i * 0.35)),
-  'mysql': new Array(384).fill(0).map((_, i) => Math.cos(i * 0.35)),
-  'mongodb': new Array(384).fill(0).map((_, i) => Math.sin(i * 0.4)),
-  'redis': new Array(384).fill(0).map((_, i) => Math.cos(i * 0.4)),
-  
-  // Common terms
-  'database': new Array(384).fill(0).map((_, i) => Math.sin(i * 0.45)),
-  'api': new Array(384).fill(0).map((_, i) => Math.cos(i * 0.45)),
-  'server': new Array(384).fill(0).map((_, i) => Math.sin(i * 0.5)),
-  'client': new Array(384).fill(0).map((_, i) => Math.cos(i * 0.5)),
-  'frontend': new Array(384).fill(0).map((_, i) => Math.sin(i * 0.55)),
-  'backend': new Array(384).fill(0).map((_, i) => Math.cos(i * 0.55)),
-  
-  // Add more pre-computed embeddings here...
-}
-
-// Simple word similarity using character n-grams
-function computeSimpleEmbedding(text: string): Vector {
-  const normalized = text.toLowerCase().trim()
-  const vector = new Array(384).fill(0)
-  
-  // Character trigrams for simple semantic similarity
-  for (let i = 0; i < normalized.length - 2; i++) {
-    const trigram = normalized.slice(i, i + 3)
-    const hash = trigram.charCodeAt(0) * 31 + 
-                 trigram.charCodeAt(1) * 7 + 
-                 trigram.charCodeAt(2)
-    const index = Math.abs(hash) % 384
-    vector[index] += 1 / (normalized.length - 2)
-  }
-  
-  // Normalize vector
-  const magnitude = Math.sqrt(vector.reduce((sum, val) => sum + val * val, 0))
-  if (magnitude > 0) {
-    for (let i = 0; i < vector.length; i++) {
-      vector[i] /= magnitude
-    }
-  }
-  
-  return vector
-}
-
-export class LightweightEmbedder {
-  private onnxEmbedder: any = null
-  private stats = {
-    precomputedHits: 0,
-    simpleComputes: 0,
-    onnxComputes: 0
-  }
-  
-  async embed(text: string | string[]): Promise<Vector | Vector[]> {
-    if (Array.isArray(text)) {
-      return Promise.all(text.map(t => this.embedSingle(t)))
-    }
-    return this.embedSingle(text)
-  }
-  
-  private async embedSingle(text: string): Promise<Vector> {
-    const normalized = text.toLowerCase().trim()
-    
-    // 1. Check pre-computed embeddings (instant, zero memory)
-    if (PRECOMPUTED_EMBEDDINGS[normalized]) {
-      this.stats.precomputedHits++
-      return PRECOMPUTED_EMBEDDINGS[normalized]
-    }
-    
-    // 2. Check for close matches in pre-computed
-    for (const [term, embedding] of Object.entries(PRECOMPUTED_EMBEDDINGS)) {
-      if (normalized.includes(term) || term.includes(normalized)) {
-        this.stats.precomputedHits++
-        // Return slightly modified version to maintain uniqueness
-        return embedding.map(v => v * 0.95)
-      }
-    }
-    
-    // 3. For short text, use simple embedding (fast, low memory)
-    if (normalized.length < 50) {
-      this.stats.simpleComputes++
-      return computeSimpleEmbedding(normalized)
-    }
-    
-    // 4. Last resort: Load ONNX model (only if really needed)
-    if (!this.onnxEmbedder) {
-      console.log('⚠️ Loading ONNX model for complex text...')
-      const { TransformerEmbedding } = await import('../utils/embedding.js')
-      this.onnxEmbedder = new TransformerEmbedding({ 
-        precision: 'fp32',
-        verbose: false 
-      })
-      await this.onnxEmbedder.init()
-    }
-    
-    this.stats.onnxComputes++
-    return await this.onnxEmbedder.embed(text)
-  }
-  
-  getStats() {
-    return {
-      ...this.stats,
-      totalEmbeddings: this.stats.precomputedHits + 
-                      this.stats.simpleComputes + 
-                      this.stats.onnxComputes,
-      cacheHitRate: this.stats.precomputedHits / 
-                   (this.stats.precomputedHits + 
-                    this.stats.simpleComputes + 
-                    this.stats.onnxComputes)
-    }
-  }
-  
-  // Pre-load common embeddings from file
-  async loadPrecomputed(filePath?: string) {
-    if (!filePath) return
-    
-    try {
-      const fs = await import('fs/promises')
-      const data = await fs.readFile(filePath, 'utf-8')
-      const embeddings = JSON.parse(data)
-      Object.assign(PRECOMPUTED_EMBEDDINGS, embeddings)
-      console.log(`✅ Loaded ${Object.keys(embeddings).length} pre-computed embeddings`)
-    } catch (error) {
-      console.warn('Could not load pre-computed embeddings:', error)
-    }
-  }
-}