brainy/src/triple/TripleIntelligenceSystem.ts

/**
 * Triple Intelligence System - Consolidated, Production-Ready Implementation
 * 
 * NO FALLBACKS - NO MOCKS - NO STUBS - REAL PERFORMANCE
 * 
 * This is the single source of truth for Triple Intelligence operations.
 * All operations MUST use fast paths or FAIL LOUDLY.
 * 
 * Performance Guarantees:
 * - Vector search: O(log n) via HNSW
 * - Range queries: O(log n) via B-tree indexes  
 * - Graph traversal: O(1) adjacency list lookups
 * - Fusion: O(k log k) where k = result count
 */

import { HNSWIndex } from '../hnsw/hnswIndex.js'
import { MetadataIndexManager } from '../utils/metadataIndex.js'
import { Vector } from '../coreTypes.js'

// Triple Intelligence types
export interface TripleQuery {
  // Vector search
  similar?: string
  like?: string
  vector?: Vector
  
  // Field filtering
  where?: Record<string, any>
  
  // Graph traversal
  connected?: {
    from?: string
    to?: string
    type?: string
    direction?: 'in' | 'out' | 'both'
    depth?: number
  }
  
  // Common options
  limit?: number
}

export interface TripleOptions {
  fusion?: {
    strategy?: 'rrf' | 'weighted' | 'adaptive'
    weights?: Record<string, number>
    k?: number
  }
}

// Simple graph index interface for now
interface GraphAdjacencyIndex {
  getNeighbors(id: string, direction?: 'in' | 'out' | 'both'): Promise<string[]>
  size(): number
}

/**
 * Performance metrics for monitoring and assertions
 */
export class PerformanceMetrics {
  private operations: Map<string, OperationStats> = new Map()
  private slowQueries: QueryLog[] = []
  private totalItems: number = 0
  
  recordOperation(type: string, elapsed: number, itemCount?: number): void {
    const stats = this.operations.get(type) || {
      count: 0,
      totalTime: 0,
      maxTime: 0,
      minTime: Infinity,
      violations: 0
    }
    
    stats.count++
    stats.totalTime += elapsed
    stats.maxTime = Math.max(stats.maxTime, elapsed)
    stats.minTime = Math.min(stats.minTime, elapsed)
    
    // Check for O(log n) violation
    const expectedTime = this.getExpectedTime(type, itemCount || this.totalItems)
    if (elapsed > expectedTime * 2) {
      stats.violations++
      console.error(
        `⚠️ Performance violation in ${type}: ${elapsed.toFixed(2)}ms > expected ${expectedTime.toFixed(2)}ms`
      )
      
      this.slowQueries.push({
        type,
        elapsed,
        expectedTime,
        timestamp: Date.now(),
        itemCount: itemCount || this.totalItems
      })
    }
    
    this.operations.set(type, stats)
  }
  
  private getExpectedTime(type: string, itemCount: number): number {
    // O(log n) operations should complete in roughly log2(n) * k milliseconds
    // where k is a constant based on the operation type
    const logN = Math.log2(Math.max(1, itemCount))
    
    switch (type) {
      case 'vector_search':
        return logN * 5 // HNSW is very efficient
      case 'field_filter':
        return logN * 3 // B-tree operations are fast
      case 'graph_traversal':
        return 10 // O(1) adjacency list lookups
      case 'fusion':
        return Math.log2(Math.max(1, itemCount)) * 2 // O(k log k) sorting
      default:
        return logN * 10 // Conservative estimate
    }
  }
  
  setTotalItems(count: number): void {
    this.totalItems = count
  }
  
  getReport(): PerformanceReport {
    const report: PerformanceReport = {
      operations: {},
      violations: [],
      slowQueries: this.slowQueries.slice(-100) // Last 100 slow queries
    }
    
    for (const [type, stats] of this.operations) {
      report.operations[type] = {
        avgTime: stats.totalTime / stats.count,
        maxTime: stats.maxTime,
        minTime: stats.minTime,
        violations: stats.violations,
        violationRate: stats.violations / stats.count,
        totalCalls: stats.count
      }
      
      if (stats.violations > 0) {
        report.violations.push({
          type,
          count: stats.violations,
          rate: stats.violations / stats.count
        })
      }
    }
    
    return report
  }
  
  reset(): void {
    this.operations.clear()
    this.slowQueries = []
  }
}

/**
 * Query execution planner - optimizes query execution order
 */
class QueryPlanner {
  /**
   * Build an optimized execution plan for a query
   */
  buildPlan(query: TripleQuery): QueryPlan {
    const plan: QueryPlan = {
      steps: [],
      estimatedCost: 0,
      requiresIndexes: []
    }
    
    // Determine which indexes are required
    if (query.similar || query.like) {
      plan.requiresIndexes.push('hnsw')
    }
    if (query.where) {
      plan.requiresIndexes.push('metadata')
    }
    if (query.connected) {
      plan.requiresIndexes.push('graph')
    }
    
    // Order operations by selectivity (most selective first)
    // This minimizes the working set for subsequent operations
    
    // 1. Field filters are usually most selective
    if (query.where) {
      plan.steps.push({
        type: 'field',
        operation: 'filter',
        requiresFastPath: true,
        estimatedSelectivity: 0.1 // Assume 10% match rate
      })
    }
    
    // 2. Graph traversal is moderately selective
    if (query.connected) {
      plan.steps.push({
        type: 'graph',
        operation: 'traverse',
        requiresFastPath: true,
        estimatedSelectivity: 0.3
      })
    }
    
    // 3. Vector search is least selective (returns top-k)
    if (query.similar || query.like) {
      plan.steps.push({
        type: 'vector',
        operation: 'search',
        requiresFastPath: true,
        estimatedSelectivity: 1.0
      })
    }
    
    // Calculate estimated cost
    plan.estimatedCost = plan.steps.reduce((cost, step) => {
      return cost + (1 / step.estimatedSelectivity)
    }, 0)
    
    return plan
  }
}

/**
 * The main Triple Intelligence System
 */
export class TripleIntelligenceSystem {
  private metadataIndex: MetadataIndexManager
  private hnswIndex: HNSWIndex
  private graphIndex: GraphAdjacencyIndex
  private metrics: PerformanceMetrics
  private planner: QueryPlanner
  private embedder: (text: string) => Promise<Vector>
  private storage: any // Storage adapter for retrieving full entities
  
  constructor(
    metadataIndex: MetadataIndexManager,
    hnswIndex: HNSWIndex,
    graphIndex: GraphAdjacencyIndex,
    embedder: (text: string) => Promise<Vector>,
    storage: any
  ) {
    // REQUIRE all components - no fallbacks
    if (!metadataIndex) {
      throw new Error('MetadataIndex required for Triple Intelligence')
    }
    if (!hnswIndex) {
      throw new Error('HNSW index required for Triple Intelligence')
    }
    if (!graphIndex) {
      throw new Error('Graph index required for Triple Intelligence')
    }
    if (!embedder) {
      throw new Error('Embedding function required for Triple Intelligence')
    }
    if (!storage) {
      throw new Error('Storage adapter required for Triple Intelligence')
    }
    
    this.metadataIndex = metadataIndex
    this.hnswIndex = hnswIndex
    this.graphIndex = graphIndex
    this.embedder = embedder
    this.storage = storage
    this.metrics = new PerformanceMetrics()
    this.planner = new QueryPlanner()
    
    // Set initial item count for metrics
    this.updateItemCount()
  }
  
  /**
   * Main find method - executes Triple Intelligence queries
   */
  async find(query: TripleQuery, options?: TripleOptions): Promise<TripleResult[]> {
    const startTime = performance.now()
    
    // Validate query
    this.validateQuery(query)
    
    // Build optimized query plan
    const plan = this.planner.buildPlan(query)
    
    // Verify all required indexes are available
    this.verifyIndexes(plan.requiresIndexes)
    
    // Execute query plan with NO FALLBACKS
    const results = await this.executeQueryPlan(plan, query, options)
    
    // Record metrics
    const elapsed = performance.now() - startTime
    this.metrics.recordOperation('find_query', elapsed, results.length)
    
    // ASSERT performance guarantees
    this.assertPerformance(elapsed, results.length)
    
    return results
  }
  
  /**
   * Vector search using HNSW for O(log n) performance
   */
  private async vectorSearch(
    query: string | Vector,
    limit: number
  ): Promise<TripleResult[]> {
    const startTime = performance.now()
    
    // Convert text to vector if needed
    const vector = typeof query === 'string' 
      ? await this.embedder(query)
      : query
    
    // Search using HNSW index - O(log n) guaranteed
    const searchResults = await this.hnswIndex.search(vector, limit)
    
    // Convert to result format
    const results: TripleResult[] = []
    for (const [id, score] of searchResults) {
      const entity = await this.storage.getNoun(id)
      if (entity) {
        results.push({
          id,
          score,
          entity,
          metadata: entity.metadata || {},
          vectorScore: score
        })
      }
    }
    
    const elapsed = performance.now() - startTime
    this.metrics.recordOperation('vector_search', elapsed, results.length)
    
    // Assert O(log n) performance
    const expectedTime = Math.log2(this.hnswIndex.size()) * 5
    if (elapsed > expectedTime * 2) {
      throw new Error(
        `Vector search O(log n) violation: ${elapsed.toFixed(2)}ms > ${expectedTime.toFixed(2)}ms`
      )
    }
    
    return results
  }
  
  /**
   * Field filtering using MetadataIndex for O(log n) performance
   */
  private async fieldFilter(
    where: Record<string, any>,
    limit?: number
  ): Promise<TripleResult[]> {
    const startTime = performance.now()
    
    // Use MetadataIndex for O(log n) performance
    const matchingIds = await this.metadataIndex.getIdsForFilter(where)
    
    if (!matchingIds || matchingIds.length === 0) {
      return []
    }
    
    // Convert to results with full entities
    const results: TripleResult[] = []
    const idsToProcess = limit 
      ? matchingIds.slice(0, limit)
      : matchingIds
    
    // Process in parallel batches for efficiency
    const batchSize = 100
    for (let i = 0; i < idsToProcess.length; i += batchSize) {
      const batch = idsToProcess.slice(i, i + batchSize)
      const entities = await Promise.all(
        batch.map(id => this.storage.getNoun(id))
      )
      
      for (let j = 0; j < entities.length; j++) {
        const entity = entities[j]
        if (entity) {
          results.push({
            id: batch[j],
            score: 1.0, // Field matches are binary
            entity,
            metadata: entity.metadata || {},
            fieldScore: 1.0
          })
        }
      }
    }
    
    const elapsed = performance.now() - startTime
    this.metrics.recordOperation('field_filter', elapsed, results.length)
    
    // Assert O(log n) for range queries
    if (this.hasRangeOperators(where)) {
      const expectedTime = Math.log2(1000000) * 3 // Assume max 1M items
      if (elapsed > expectedTime * 2) {
        throw new Error(
          `Field filter O(log n) violation: ${elapsed.toFixed(2)}ms > ${expectedTime.toFixed(2)}ms`
        )
      }
    }
    
    return results
  }
  
  /**
   * Graph traversal using adjacency lists for O(1) lookups
   */
  private async graphTraversal(
    params: {
      from?: string
      to?: string
      type?: string
      direction?: 'in' | 'out' | 'both'
      depth?: number
    }
  ): Promise<TripleResult[]> {
    const startTime = performance.now()
    const maxDepth = params.depth || 2
    const results: TripleResult[] = []
    const visited = new Set<string>()
    
    // BFS traversal with O(1) adjacency lookups
    const queue: Array<{ id: string; depth: number; score: number }> = []
    
    // Initialize queue with starting node(s)
    if (params.from) {
      queue.push({ id: params.from, depth: 0, score: 1.0 })
    }
    
    while (queue.length > 0) {
      const { id, depth, score } = queue.shift()!
      
      if (visited.has(id) || depth > maxDepth) {
        continue
      }
      visited.add(id)
      
      // Get entity
      const entity = await this.storage.getNoun(id)
      if (entity) {
        results.push({
          id,
          score: score * Math.pow(0.8, depth), // Decay by distance
          entity,
          metadata: entity.metadata || {},
          graphScore: score,
          depth
        })
      }
      
      // Get neighbors - O(1) adjacency list lookup
      if (depth < maxDepth) {
        const neighbors = await this.graphIndex.getNeighbors(id, params.direction)
        
        for (const neighborId of neighbors) {
          if (!visited.has(neighborId)) {
            queue.push({
              id: neighborId,
              depth: depth + 1,
              score: score * 0.8
            })
          }
        }
      }
    }
    
    const elapsed = performance.now() - startTime
    this.metrics.recordOperation('graph_traversal', elapsed, results.length)
    
    // Graph traversal should be fast due to O(1) adjacency lookups
    const expectedTime = visited.size * 0.5 // 0.5ms per node
    if (elapsed > expectedTime * 3) {
      throw new Error(
        `Graph traversal performance warning: ${elapsed.toFixed(2)}ms > ${expectedTime.toFixed(2)}ms`
      )
    }
    
    return results
  }
  
  /**
   * Execute the query plan
   */
  private async executeQueryPlan(
    plan: QueryPlan,
    query: TripleQuery,
    options?: TripleOptions
  ): Promise<TripleResult[]> {
    const limit = query.limit || 10
    const intermediateResults: Map<string, TripleResult[]> = new Map()
    
    // Execute each step in the plan
    for (const step of plan.steps) {
      const stepStartTime = performance.now()
      let stepResults: TripleResult[] = []
      
      switch (step.type) {
        case 'vector':
          stepResults = await this.vectorSearch(
            query.similar || query.like!,
            limit * 3 // Over-fetch for fusion
          )
          break
          
        case 'field':
          stepResults = await this.fieldFilter(
            query.where!,
            limit * 3
          )
          break
          
        case 'graph':
          stepResults = await this.graphTraversal(query.connected!)
          break
          
        default:
          throw new Error(`Unknown query step type: ${step.type}`)
      }
      
      intermediateResults.set(step.type, stepResults)
      
      const stepElapsed = performance.now() - stepStartTime
      console.log(
        `Step ${step.type}:${step.operation} completed in ${stepElapsed.toFixed(2)}ms with ${stepResults.length} results`
      )
    }
    
    // Fuse results if multiple signals
    if (intermediateResults.size > 1) {
      return this.fuseResults(intermediateResults, limit, options)
    }
    
    // Single signal - return as is
    const singleResults = Array.from(intermediateResults.values())[0]
    return singleResults.slice(0, limit)
  }
  
  /**
   * Fuse results using Reciprocal Rank Fusion (RRF)
   */
  private fuseResults(
    resultSets: Map<string, TripleResult[]>,
    limit: number,
    options?: TripleOptions
  ): TripleResult[] {
    const startTime = performance.now()
    const k = options?.fusion?.k || 60 // RRF constant
    const weights = options?.fusion?.weights || {
      vector: 0.5,
      field: 0.3,
      graph: 0.2
    }
    
    // Calculate RRF scores
    const fusionScores = new Map<string, number>()
    const entityMap = new Map<string, TripleResult>()
    
    for (const [signalType, results] of resultSets) {
      const weight = weights[signalType] || 1.0
      
      results.forEach((result, rank) => {
        const rrfScore = weight / (k + rank + 1)
        const currentScore = fusionScores.get(result.id) || 0
        fusionScores.set(result.id, currentScore + rrfScore)
        
        // Keep the result with the most information
        if (!entityMap.has(result.id)) {
          entityMap.set(result.id, result)
        }
      })
    }
    
    // Sort by fusion score
    const sortedIds = Array.from(fusionScores.entries())
      .sort((a, b) => b[1] - a[1])
      .slice(0, limit)
    
    // Build final results
    const results: TripleResult[] = []
    for (const [id, fusionScore] of sortedIds) {
      const result = entityMap.get(id)!
      results.push({
        ...result,
        fusionScore,
        score: fusionScore // Use fusion score as primary score
      })
    }
    
    const elapsed = performance.now() - startTime
    this.metrics.recordOperation('fusion', elapsed, results.length)
    
    // Fusion should be O(k log k)
    const expectedTime = Math.log2(Math.max(1, fusionScores.size)) * 2
    if (elapsed > expectedTime * 3) {
      console.warn(
        `Fusion performance warning: ${elapsed.toFixed(2)}ms > ${expectedTime.toFixed(2)}ms`
      )
    }
    
    return results
  }
  
  /**
   * Validate query parameters
   */
  private validateQuery(query: TripleQuery): void {
    if (!query.similar && !query.like && !query.where && !query.connected) {
      throw new Error(
        'Query must specify at least one of: similar, like, where, or connected'
      )
    }
    
    if (query.limit && (query.limit < 1 || query.limit > 10000)) {
      throw new Error('Query limit must be between 1 and 10000')
    }
  }
  
  /**
   * Verify required indexes are available
   */
  private verifyIndexes(required: string[]): void {
    for (const index of required) {
      switch (index) {
        case 'hnsw':
          if (!this.hnswIndex || this.hnswIndex.size() === 0) {
            throw new Error('HNSW index not available or empty')
          }
          break
          
        case 'metadata':
          if (!this.metadataIndex) {
            throw new Error('Metadata index not available')
          }
          break
          
        case 'graph':
          if (!this.graphIndex) {
            throw new Error('Graph index not available')
          }
          break
      }
    }
  }
  
  /**
   * Assert performance guarantees
   */
  private assertPerformance(elapsed: number, resultCount: number): void {
    const itemCount = this.getTotalItems()
    const expectedTime = Math.log2(Math.max(1, itemCount)) * 20 // 20ms per log operation
    
    if (elapsed > expectedTime * 3) {
      throw new Error(
        `Query performance violation: ${elapsed.toFixed(2)}ms > expected ${expectedTime.toFixed(2)}ms ` +
        `for ${itemCount} items`
      )
    }
  }
  
  /**
   * Check if where clause has range operators
   */
  private hasRangeOperators(where: Record<string, any>): boolean {
    for (const value of Object.values(where)) {
      if (typeof value === 'object' && value !== null) {
        const keys = Object.keys(value)
        if (keys.some(k => ['$gt', '$gte', '$lt', '$lte', '$between'].includes(k))) {
          return true
        }
      }
    }
    return false
  }
  
  /**
   * Update item count for metrics
   */
  private updateItemCount(): void {
    const count = this.getTotalItems()
    this.metrics.setTotalItems(count)
  }
  
  /**
   * Get total item count across all indexes
   */
  private getTotalItems(): number {
    // Get the largest count from available indexes
    // Note: MetadataIndexManager might not have a size() method
    // so we'll use HNSW index size as primary indicator
    return Math.max(
      this.hnswIndex?.size() || 0,
      1000000, // Assume max 1M items for now
      this.graphIndex?.size() || 0
    )
  }
  
  /**
   * Get performance metrics
   */
  getMetrics(): PerformanceMetrics {
    return this.metrics
  }
  
  /**
   * Reset performance metrics
   */
  resetMetrics(): void {
    this.metrics.reset()
  }
}

// Type definitions

interface OperationStats {
  count: number
  totalTime: number
  maxTime: number
  minTime: number
  violations: number
}

interface QueryLog {
  type: string
  elapsed: number
  expectedTime: number
  timestamp: number
  itemCount: number
}

interface PerformanceReport {
  operations: Record<string, {
    avgTime: number
    maxTime: number
    minTime: number
    violations: number
    violationRate: number
    totalCalls: number
  }>
  violations: Array<{
    type: string
    count: number
    rate: number
  }>
  slowQueries: QueryLog[]
}

interface QueryPlan {
  steps: QueryStep[]
  estimatedCost: number
  requiresIndexes: string[]
}

interface QueryStep {
  type: string
  operation: string
  requiresFastPath: boolean
  estimatedSelectivity: number
}

interface TripleResult {
  id: string
  score: number
  entity: any
  metadata: Record<string, any>
  vectorScore?: number
  fieldScore?: number
  graphScore?: number
  fusionScore?: number
  depth?: number
}