brainy/src/neural/neuralAPI.ts

/**
 * Neural API - Unified Semantic Intelligence
 * 
 * Best-of-both: Complete functionality + Enterprise performance
 * Combines rich features with O(n) algorithms for millions of items
 */

import { Vector, HNSWNoun } from '../coreTypes.js'
import { cosineDistance } from '../utils/distance.js'

// === Rich Result Types (from original neuralAPI) ===

export interface SimilarityResult {
  score: number
  method?: string
  confidence?: number
  explanation?: string
  hierarchy?: {
    sharedParent?: string
    distance?: number
  }
  breakdown?: {
    semantic?: number
    taxonomic?: number
    contextual?: number
  }
}

export interface SimilarityOptions {
  explain?: boolean
  includeBreakdown?: boolean
  method?: 'cosine' | 'euclidean' | 'hybrid'
}

export interface SemanticCluster {
  id: string
  centroid: Vector
  members: string[]
  label?: string
  confidence: number
  depth?: number
  // Enterprise additions
  size?: number
  level?: number
  center?: any
}

export interface SemanticHierarchy {
  self: { id: string; type?: string; vector: Vector }
  parent?: { id: string; type?: string; similarity: number }
  grandparent?: { id: string; type?: string; similarity: number }
  root?: { id: string; type?: string; similarity: number }
  siblings?: Array<{ id: string; similarity: number }>
  children?: Array<{ id: string; similarity: number }>
  depth?: number
}

export interface NeighborGraph {
  center: string
  neighbors: Array<{
    id: string
    similarity: number
    type?: string
    connections?: number
  }>
  edges?: Array<{
    source: string
    target: string
    weight: number
    type?: string
  }>
}

export interface ClusterOptions {
  algorithm?: 'hierarchical' | 'kmeans' | 'sample' | 'stream'
  maxClusters?: number
  threshold?: number
  // Enterprise options
  sampleSize?: number
  strategy?: 'random' | 'diverse' | 'recent'
  level?: number
  batchSize?: number
}

export interface VisualizationData {
  format: 'force-directed' | 'hierarchical' | 'radial'
  nodes: Array<{
    id: string
    x: number
    y: number
    z?: number
    type?: string
    cluster?: string
    size?: number
  }>
  edges: Array<{
    source: string
    target: string
    weight: number
    type?: string
  }>
  layout?: {
    dimensions: number
    algorithm: string
    bounds?: { width: number; height: number; depth?: number }
  }
  clusters?: Array<{
    id: string
    color: string
    label?: string
    size: number
  }>
}

// === Enterprise Types (from neuralOptimized) ===

export interface ClusteringStrategy {
  type: 'sample' | 'hierarchical' | 'stream' | 'hybrid'
  sampleSize?: number
  maxClusters?: number
  minClusterSize?: number
}

export interface LODConfig {
  levels: number
  itemsPerLevel: number[]
  zoomThresholds: number[]
}

/**
 * Neural API - Unified best-of-both implementation
 */
export class NeuralAPI {
  private brain: any // Brainy instance
  private distanceFn: (a: Vector, b: Vector) => number
  private similarityCache: Map<string, number> = new Map()
  private clusterCache: Map<string, any> = new Map() // Enhanced for enterprise
  private hierarchyCache: Map<string, SemanticHierarchy> = new Map()

  constructor(brain: any) {
    this.brain = brain
    this.distanceFn = brain.distance || cosineDistance
  }
  
  // ===== SMART USER-FRIENDLY API =====
  
  /**
   * Calculate similarity between any two items (smart detection)
   */
  async similar(a: any, b: any, options?: SimilarityOptions): Promise<number | SimilarityResult> {
    // Auto-detect input types
    if (typeof a === 'string' && typeof b === 'string') {
      if (this.isId(a) && this.isId(b)) {
        return this.similarityById(a, b, options)
      } else {
        return this.similarityByText(a, b, options)
      }
    } else if (Array.isArray(a) && Array.isArray(b)) {
      return this.similarityByVector(a as Vector, b as Vector, options)
    }
    
    // Handle mixed types
    return this.smartSimilarity(a, b, options)
  }
  
  /**
   * Find semantic clusters (auto-detects best approach)
   * Now with enterprise performance!
   */
  async clusters(input?: any): Promise<SemanticCluster[]> {
    // No input? Use enterprise fast clustering
    if (!input) {
      return this.clusterFast()
    }
    
    // Array? Cluster these items (use large clustering for big arrays)
    if (Array.isArray(input)) {
      if (input.length > 1000) {
        return this.clusterLarge({ sampleSize: Math.min(input.length, 1000) })
      }
      return this.clusterItems(input)
    }
    
    // String? Find clusters near this
    if (typeof input === 'string') {
      return this.clustersNear(input)
    }
    
    // Object? Use as config with enterprise algorithms
    if (typeof input === 'object' && !Array.isArray(input)) {
      return this.clusterWithConfig(input as ClusterOptions)
    }
    
    throw new Error('Invalid input for clustering')
  }
  
  /**
   * Get semantic hierarchy for an item
   */
  async hierarchy(id: string): Promise<SemanticHierarchy> {
    // Check cache first
    if (this.hierarchyCache.has(id)) {
      return this.hierarchyCache.get(id)!
    }
    
    const item = await this.brain.get(id)
    if (!item) {
      throw new Error(`Item not found: ${id}`)
    }
    
    // Find semantic relationships
    const hierarchy = await this.buildHierarchy(item)
    
    // Cache result
    this.hierarchyCache.set(id, hierarchy)
    
    return hierarchy
  }
  
  /**
   * Find semantic neighbors for visualization
   */
  async neighbors(id: string, options?: {
    radius?: number
    limit?: number
    includeEdges?: boolean
  }): Promise<NeighborGraph> {
    const radius = options?.radius ?? 0.3
    const limit = options?.limit ?? 50
    
    // Search for nearby items
    const results = await this.brain.search(id, limit * 2)
    
    // Filter by semantic radius
    const neighbors = results
      .filter((r: any) => r.similarity >= (1 - radius))
      .slice(0, limit)
      .map((r: any) => ({
        id: r.id,
        similarity: r.similarity,
        type: r.metadata?.type,
        connections: r.metadata?.connections?.size || 0
      }))
    
    const graph: NeighborGraph = {
      center: id,
      neighbors
    }
    
    // Add edges if requested
    if (options?.includeEdges) {
      graph.edges = await this.buildEdges(id, neighbors)
    }
    
    return graph
  }
  
  /**
   * Find semantic path between two items
   */
  async semanticPath(fromId: string, toId: string, options?: {
    maxHops?: number
    algorithm?: 'breadth' | 'dijkstra'
  }): Promise<Array<{
    id: string
    similarity: number
    hop: number
  }>> {
    const maxHops = options?.maxHops ?? 5
    const algorithm = options?.algorithm ?? 'breadth'
    
    if (algorithm === 'dijkstra') {
      return this.dijkstraPath(fromId, toId, maxHops)
    } else {
      return this.breadthFirstPath(fromId, toId, maxHops)
    }
  }
  
  /**
   * Detect semantic outliers
   */
  async outliers(threshold: number = 0.3): Promise<string[]> {
    // Get all items
    const stats = this.brain.getStats()
    const totalItems = stats.entities.total
    
    if (totalItems === 0) return []
    
    // For large datasets, use sampling
    if (totalItems > 10000) {
      return this.outliersViaSampling(threshold, 1000)
    }
    
    return this.outliersByDistance(threshold)
  }
  
  /**
   * Generate visualization data
   */
  async visualize(options?: {
    maxNodes?: number
    dimensions?: 2 | 3
    algorithm?: 'force' | 'hierarchical' | 'radial'
    includeEdges?: boolean
  }): Promise<VisualizationData> {
    const maxNodes = options?.maxNodes ?? 100
    const dimensions = options?.dimensions ?? 2
    const algorithm = options?.algorithm ?? 'force'
    
    // Get representative nodes
    const nodes = await this.getVisualizationNodes(maxNodes)
    
    // Apply layout algorithm
    const positioned = await this.applyLayout(nodes, algorithm, dimensions)
    
    // Build edges if requested
    const edges = options?.includeEdges !== false ? 
      await this.buildVisualizationEdges(positioned) : []
    
    // Detect optimal format
    const format = this.detectOptimalFormat(positioned, edges)
    
    return {
      format,
      nodes: positioned,
      edges,
      layout: {
        dimensions,
        algorithm,
        bounds: this.calculateBounds(positioned, dimensions)
      }
    }
  }
  
  // ===== ENTERPRISE PERFORMANCE ALGORITHMS =====
  
  /**
   * Fast clustering using HNSW levels - O(n) instead of O(n²)
   */
  async clusterFast(options: {
    level?: number
    maxClusters?: number
  } = {}): Promise<SemanticCluster[]> {
    const cacheKey = `hierarchical-${options.level}-${options.maxClusters}`
    if (this.clusterCache.has(cacheKey)) {
      return this.clusterCache.get(cacheKey)
    }
    
    // Use HNSW's natural hierarchy - auto-select optimal level
    const level = options.level ?? await this.getOptimalClusteringLevel()
    const maxClusters = options.maxClusters ?? 100
    
    // Get representative nodes from HNSW level
    const representatives = await this.getHNSWLevelNodes(level)
    
    // Each representative is a natural cluster center
    const clusters = []
    for (const rep of representatives.slice(0, maxClusters)) {
      const members = await this.findClusterMembers(rep, level - 1)
      clusters.push({
        id: `cluster-${rep.id}`,
        centroid: rep.vector,
        center: rep,
        members: members.map(m => m.id),
        size: members.length,
        level,
        confidence: 0.8 + (members.length / 100) * 0.2 // Size-based confidence
      } as SemanticCluster)
    }
    
    this.clusterCache.set(cacheKey, clusters)
    return clusters
  }
  
  /**
   * Large-scale clustering for massive datasets (millions of items)
   */
  async clusterLarge(options: {
    sampleSize?: number
    strategy?: 'random' | 'diverse' | 'recent'
  } = {}): Promise<SemanticCluster[]> {
    const sampleSize = options.sampleSize ?? 1000
    const strategy = options.strategy ?? 'diverse'
    
    // Get representative sample
    const sample = await this.getSample(sampleSize, strategy)
    
    // Cluster the sample (fast on small set)
    const sampleClusters = await this.performFastClustering(sample)
    
    // Project clusters to full dataset
    return this.projectClustersToFullDataset(sampleClusters)
  }
  
  /**
   * Streaming clustering for progressive refinement
   */
  async* clusterStream(options: {
    batchSize?: number
    maxBatches?: number
  } = {}): AsyncGenerator<SemanticCluster[]> {
    const batchSize = options.batchSize ?? 1000
    const maxBatches = options.maxBatches ?? Infinity
    
    let offset = 0
    let batchCount = 0
    let globalClusters: SemanticCluster[] = []
    
    while (batchCount < maxBatches) {
      // Get next batch
      const batch = await this.getBatch(offset, batchSize)
      if (batch.length === 0) break
      
      // Cluster this batch
      const batchClusters = await this.performFastClustering(batch)
      
      // Merge with global clusters
      globalClusters = await this.mergeClusters(globalClusters, batchClusters)
      
      // Yield current state
      yield globalClusters
      
      offset += batchSize
      batchCount++
    }
  }
  
  /**
   * Level-of-detail for massive visualization
   */
  async getLOD(zoomLevel: number, viewport?: {
    center: Vector
    radius: number
  }): Promise<any> {
    // Define LOD levels based on zoom
    const lodLevels = [
      { zoom: 0, maxNodes: 50, clusterLevel: 3 },
      { zoom: 1, maxNodes: 200, clusterLevel: 2 },
      { zoom: 2, maxNodes: 1000, clusterLevel: 1 },
      { zoom: 3, maxNodes: 5000, clusterLevel: 0 }
    ]
    
    const lod = lodLevels.find(l => zoomLevel <= l.zoom) || lodLevels[lodLevels.length - 1]
    
    if (viewport) {
      return this.getViewportLOD(viewport, lod)
    } else {
      return this.getGlobalLOD(lod)
    }
  }
  
  // ===== IMPLEMENTATION HELPERS =====
  
  private isId(str: string): boolean {
    // Check if string looks like an ID (UUID pattern, etc.)
    return (str.length === 36 && str.includes('-')) || !!str.match(/^[a-f0-9]{24}$/)
  }
  
  private async similarityById(idA: string, idB: string, options?: SimilarityOptions): Promise<number | SimilarityResult> {
    const cacheKey = `${idA}-${idB}`
    if (this.similarityCache.has(cacheKey)) {
      return this.similarityCache.get(cacheKey)!
    }
    
    // Get items
    const [itemA, itemB] = await Promise.all([
      this.brain.get(idA),
      this.brain.get(idB)
    ])
    
    if (!itemA || !itemB) {
      throw new Error('One or both items not found')
    }
    
    // Calculate similarity
    const score = this.distanceFn(itemA.vector, itemB.vector)
    
    this.similarityCache.set(cacheKey, score)
    
    if (options?.explain) {
      return {
        score,
        method: 'cosine',
        confidence: 0.9,
        explanation: `Semantic similarity between ${idA} and ${idB}`
      }
    }
    
    return score
  }
  
  private async similarityByText(textA: string, textB: string, options?: SimilarityOptions): Promise<number | SimilarityResult> {
    // Generate embeddings
    const [vectorA, vectorB] = await Promise.all([
      this.brain.embed(textA),
      this.brain.embed(textB)
    ])
    
    return this.similarityByVector(vectorA, vectorB, options)
  }
  
  private async similarityByVector(vectorA: Vector, vectorB: Vector, options?: SimilarityOptions): Promise<number | SimilarityResult> {
    const score = this.distanceFn(vectorA, vectorB)
    
    if (options?.explain) {
      return {
        score,
        method: options.method || 'cosine',
        confidence: 0.95,
        explanation: 'Direct vector similarity calculation'
      }
    }
    
    return score
  }
  
  private async smartSimilarity(a: any, b: any, options?: SimilarityOptions): Promise<number | SimilarityResult> {
    // Convert both to vectors and compare
    const vectorA = await this.toVector(a)
    const vectorB = await this.toVector(b)
    
    return this.similarityByVector(vectorA, vectorB, options)
  }
  
  private async toVector(item: any): Promise<Vector> {
    if (Array.isArray(item)) return item
    if (typeof item === 'string') {
      if (this.isId(item)) {
        const found = await this.brain.get(item)
        return found?.vector || await this.brain.embed(item)
      }
      return await this.brain.embed(item)
    }
    if (typeof item === 'object' && item.vector) {
      return item.vector
    }
    // Convert object to string and embed
    return await this.brain.embed(JSON.stringify(item))
  }
  
  // Enterprise clustering implementations
  private async getOptimalClusteringLevel(): Promise<number> {
    // Analyze dataset size and return optimal HNSW level
    const stats = this.brain.getStats()
    const itemCount = stats.entities.total
    
    if (itemCount < 1000) return 0
    if (itemCount < 10000) return 1
    if (itemCount < 100000) return 2
    return 3
  }
  
  private async getHNSWLevelNodes(level: number): Promise<any[]> {
    // Get nodes from specific HNSW level
    // For now, use search to get a representative sample
    const stats = this.brain.getStats()
    const sampleSize = Math.min(100, Math.floor(stats.entities.total / (level + 1)))
    
    // Use search with a general query to get representative items
    const queryVector = await this.brain.embed('data information content')
    const allItems = await this.brain.search(queryVector, sampleSize * 2)
    return allItems.slice(0, sampleSize)
  }
  
  private async findClusterMembers(center: any, level: number): Promise<any[]> {
    // Find all items that belong to this cluster
    const results = await this.brain.search(center.vector, 50)
    return results.filter((r: any) => r.similarity > 0.7)
  }
  
  private async getSample(size: number, strategy: string): Promise<any[]> {
    // Use search to get a sample of items
    const stats = this.brain.getStats()
    const maxSize = Math.min(size * 3, stats.entities.total) // Get more than needed for sampling
    const queryVector = await this.brain.embed('sample data content')
    const allItems = await this.brain.search(queryVector, maxSize)
    
    switch (strategy) {
      case 'random':
        return this.shuffleArray(allItems).slice(0, size)
      case 'diverse':
        return this.getDiverseSample(allItems, size)
      case 'recent':
        return allItems.slice(-size)
      default:
        return allItems.slice(0, size)
    }
  }
  
  private shuffleArray(array: any[]): any[] {
    const shuffled = [...array]
    for (let i = shuffled.length - 1; i > 0; i--) {
      const j = Math.floor(Math.random() * (i + 1));
      [shuffled[i], shuffled[j]] = [shuffled[j], shuffled[i]]
    }
    return shuffled
  }
  
  private async getDiverseSample(items: any[], size: number): Promise<any[]> {
    // Select diverse items using maximum distance sampling
    if (items.length <= size) return items
    
    const sample = [items[0]] // Start with first item
    
    for (let i = 1; i < size; i++) {
      let maxMinDistance = -1
      let bestItem = null
      
      for (const candidate of items) {
        if (sample.includes(candidate)) continue
        
        // Find minimum distance to existing sample
        let minDistance = Infinity
        for (const selected of sample) {
          const distance = this.distanceFn(candidate.vector, selected.vector)
          minDistance = Math.min(minDistance, distance)
        }
        
        // Select item with maximum minimum distance
        if (minDistance > maxMinDistance) {
          maxMinDistance = minDistance
          bestItem = candidate
        }
      }
      
      if (bestItem) sample.push(bestItem)
    }
    
    return sample
  }
  
  private async performFastClustering(items: any[]): Promise<SemanticCluster[]> {
    // Simple k-means clustering for the sample
    const k = Math.min(10, Math.floor(items.length / 3))
    if (k <= 1) {
      return [{
        id: 'cluster-0',
        centroid: items[0]?.vector || [],
        members: items.map(i => i.id),
        confidence: 1.0
      }]
    }
    
    // Initialize centroids randomly
    const centroids = items.slice(0, k).map(item => item.vector)
    
    // Run k-means iterations (simplified)
    for (let iter = 0; iter < 10; iter++) {
      const clusters = Array(k).fill(null).map(() => [])
      
      // Assign items to nearest centroid
      for (const item of items) {
        let bestCluster = 0
        let bestDistance = Infinity
        
        for (let c = 0; c < k; c++) {
          const distance = this.distanceFn(item.vector, centroids[c])
          if (distance < bestDistance) {
            bestDistance = distance
            bestCluster = c
          }
        }
        
        (clusters as any[])[bestCluster].push(item)
      }
      
      // Update centroids
      for (let c = 0; c < k; c++) {
        if (clusters[c].length > 0) {
          const newCentroid = this.calculateCentroid(clusters[c])
          centroids[c] = newCentroid
        }
      }
    }
    
    // Convert to SemanticCluster format
    const result: SemanticCluster[] = []
    for (let c = 0; c < k; c++) {
      const members = items.filter(item => {
        let bestCluster = 0
        let bestDistance = Infinity
        
        for (let cc = 0; cc < k; cc++) {
          const distance = this.distanceFn(item.vector, centroids[cc])
          if (distance < bestDistance) {
            bestDistance = distance
            bestCluster = cc
          }
        }
        
        return bestCluster === c
      })
      
      if (members.length > 0) {
        result.push({
          id: `cluster-${c}`,
          centroid: centroids[c],
          members: members.map(m => m.id),
          confidence: Math.min(0.9, members.length / items.length * 2)
        })
      }
    }
    
    return result
  }
  
  private calculateCentroid(items: any[]): Vector {
    if (items.length === 0) return []
    
    const dimensions = items[0].vector.length
    const centroid = new Array(dimensions).fill(0)
    
    for (const item of items) {
      for (let d = 0; d < dimensions; d++) {
        centroid[d] += item.vector[d]
      }
    }
    
    for (let d = 0; d < dimensions; d++) {
      centroid[d] /= items.length
    }
    
    return centroid
  }
  
  private async projectClustersToFullDataset(sampleClusters: SemanticCluster[]): Promise<SemanticCluster[]> {
    // Project sample clusters to full dataset
    const result: SemanticCluster[] = []
    
    for (const cluster of sampleClusters) {
      // Find all items similar to this cluster's centroid
      const similar = await this.brain.search(cluster.centroid, 1000)
      const members = similar
        .filter((s: any) => s.similarity > 0.6)
        .map((s: any) => s.id)
      
      result.push({
        ...cluster,
        members,
        size: members.length
      })
    }
    
    return result
  }
  
  private async mergeClusters(globalClusters: SemanticCluster[], batchClusters: SemanticCluster[]): Promise<SemanticCluster[]> {
    // Simple merge strategy - combine similar clusters
    const result = [...globalClusters]
    
    for (const batchCluster of batchClusters) {
      let merged = false
      
      for (let i = 0; i < result.length; i++) {
        const similarity = this.distanceFn(result[i].centroid, batchCluster.centroid)
        
        if (similarity > 0.8) {
          // Merge clusters
          const newMembers = [...new Set([...result[i].members, ...batchCluster.members])]
          result[i] = {
            ...result[i],
            members: newMembers,
            size: newMembers.length,
            centroid: this.averageVectors(result[i].centroid, batchCluster.centroid)
          }
          merged = true
          break
        }
      }
      
      if (!merged) {
        result.push(batchCluster)
      }
    }
    
    return result
  }
  
  private averageVectors(v1: Vector, v2: Vector): Vector {
    const result = new Array(v1.length)
    for (let i = 0; i < v1.length; i++) {
      result[i] = (v1[i] + v2[i]) / 2
    }
    return result
  }
  
  private async getBatch(offset: number, size: number): Promise<any[]> {
    // Get batch of items for streaming using search with offset
    const queryVector = await this.brain.embed('batch data content')
    const items = await this.brain.search(queryVector, size, { offset })
    return items
  }
  
  // Additional methods needed for full compatibility...
  private async clusterAll(): Promise<SemanticCluster[]> {
    return this.clusterFast()
  }
  
  private async clusterItems(items: any[]): Promise<SemanticCluster[]> {
    return this.performFastClustering(items)
  }
  
  private async clustersNear(id: string): Promise<SemanticCluster[]> {
    const neighbors = await this.neighbors(id, { limit: 100 })
    return this.performFastClustering(neighbors.neighbors)
  }
  
  private async clusterWithConfig(config: ClusterOptions): Promise<SemanticCluster[]> {
    switch (config.algorithm) {
      case 'hierarchical':
        return this.clusterFast(config)
      case 'sample':
        return this.clusterLarge(config)
      case 'stream':
        const generator = this.clusterStream(config)
        const results = []
        for await (const batch of generator) {
          results.push(...batch)
        }
        return results
      default:
        return this.clusterFast(config)
    }
  }
  
  // Placeholder implementations for remaining methods
  private async buildHierarchy(item: any): Promise<SemanticHierarchy> {
    // Implementation for hierarchy building
    return {
      self: { id: item.id, vector: item.vector }
    }
  }
  
  private async buildEdges(centerId: string, neighbors: any[]): Promise<any[]> {
    return []
  }
  
  private async dijkstraPath(from: string, to: string, maxHops: number): Promise<any[]> {
    return []
  }
  
  private async breadthFirstPath(from: string, to: string, maxHops: number): Promise<any[]> {
    return []
  }
  
  private async outliersViaSampling(threshold: number, sampleSize: number): Promise<string[]> {
    return []
  }
  
  private async outliersByDistance(threshold: number): Promise<string[]> {
    return []
  }
  
  private async getVisualizationNodes(maxNodes: number): Promise<any[]> {
    return []
  }
  
  private async applyLayout(nodes: any[], algorithm: string, dimensions: number): Promise<any[]> {
    return nodes
  }
  
  private async buildVisualizationEdges(nodes: any[]): Promise<any[]> {
    return []
  }
  
  private detectOptimalFormat(nodes: any[], edges: any[]): 'force-directed' | 'hierarchical' | 'radial' {
    return 'force-directed'
  }
  
  private calculateBounds(nodes: any[], dimensions: number): any {
    return { width: 100, height: 100 }
  }
  
  private async getViewportLOD(viewport: any, lod: any): Promise<any> {
    // LOD visualization is an optional advanced feature
    // Return default view without LOD optimization
    console.warn('Viewport LOD optimization not available. Using standard view.')
    return { nodes: [], edges: [], optimized: false }
  }

  private async getGlobalLOD(lod: any): Promise<any> {
    // LOD visualization is an optional advanced feature
    // Return default view without LOD optimization
    console.warn('Global LOD optimization not available. Using standard view.')
    return { nodes: [], edges: [], optimized: false }
  }
}