brainy/src/query/semanticTypeInference.ts

/**
 * Semantic Type Inference - THE ONE unified function for all type inference
 *
 * Single source of truth using semantic similarity against pre-computed keyword embeddings.
 *
 * Used by:
 * - TypeAwareQueryPlanner (query routing to specific HNSW graphs)
 * - Import pipeline (entity extraction during indexing)
 * - Neural operations (concept extraction)
 * - Public API (developer integrations)
 *
 * Performance: 1-2ms (uncached embedding), 0.2-0.5ms (cached embedding)
 * Accuracy: 95%+ (handles exact matches, synonyms, typos, semantic similarity)
 */

import { NounType, VerbType } from '../types/graphTypes.js'
import { Vector } from '../coreTypes.js'
import { getKeywordEmbeddings, type KeywordEmbedding } from '../neural/embeddedKeywordEmbeddings.js'
import { HNSWIndex } from '../hnsw/hnswIndex.js'
import { TransformerEmbedding } from '../utils/embedding.js'
import { prodLog } from '../utils/logger.js'

/**
 * Type inference result (unified nouns + verbs)
 */
export interface TypeInference {
  type: NounType | VerbType
  typeCategory: 'noun' | 'verb'
  confidence: number          // 0-1 (cosine similarity * base confidence)
  matchedKeywords: string[]   // Keywords that triggered this inference
  similarity: number          // Cosine similarity to matched keyword (0-1)
  baseConfidence: number      // Keyword's base confidence (0.7-0.95)
}

/**
 * Options for semantic type inference
 */
export interface SemanticTypeInferenceOptions {
  /** Maximum number of results to return (default: 5) */
  maxResults?: number

  /** Minimum confidence threshold (default: 0.5) */
  minConfidence?: number

  /** Filter by specific types (default: all types) */
  filterTypes?: (NounType | VerbType)[]

  /** Filter by type category (default: both) */
  filterCategory?: 'noun' | 'verb'

  /** Use embedding cache (default: true) */
  useCache?: boolean
}

/**
 * Semantic Type Inference - THE ONE unified system
 *
 * Infers entity types using semantic similarity against 700+ pre-computed keyword embeddings.
 */
export class SemanticTypeInference {
  private keywordEmbeddings: KeywordEmbedding[]
  private keywordHNSW: HNSWIndex
  private embedder: TransformerEmbedding | null = null
  private embeddingCache: Map<string, Vector>
  private readonly CACHE_MAX_SIZE = 1000
  private initPromise: Promise<void>

  constructor() {
    // Load pre-computed keyword embeddings
    this.keywordEmbeddings = getKeywordEmbeddings()

    prodLog.info(`SemanticTypeInference: Loading ${this.keywordEmbeddings.length} keyword embeddings...`)

    // Build HNSW index for O(log n) semantic search
    this.keywordHNSW = new HNSWIndex({
      M: 16,                    // Number of bi-directional links per node
      efConstruction: 200,      // Higher = better quality, slower build
      efSearch: 50,             // Search quality parameter
      ml: 1.0 / Math.log(16)    // Level generation factor
    })

    // Initialize embedding cache (LRU-style with size limit)
    this.embeddingCache = new Map()

    // Async initialization of HNSW index
    this.initPromise = this.initializeHNSW()
  }

  /**
   * Initialize HNSW index with keyword embeddings
   */
  private async initializeHNSW(): Promise<void> {
    const vectors = this.keywordEmbeddings.map(k => k.embedding)

    // Add all keyword vectors to HNSW
    for (let i = 0; i < vectors.length; i++) {
      await this.keywordHNSW.addItem({
        id: i.toString(),
        vector: vectors[i]
      })
    }

    prodLog.info(
      `SemanticTypeInference initialized: ${this.keywordEmbeddings.length} keywords, ` +
      `HNSW index built (M=16, efConstruction=200)`
    )
  }

  /**
   * THE ONE FUNCTION - Infer entity types from natural language text
   *
   * Uses semantic similarity to match text against 700+ keyword embeddings.
   *
   * @example
   * ```typescript
   * // Query routing
   * const types = await inferTypes("Find cardiologists")
   * // → [{type: Person, confidence: 0.92, keyword: "cardiologist"}]
   *
   * // Entity extraction
   * const entities = await inferTypes("Dr. Sarah Chen")
   * // → [{type: Person, confidence: 0.90, keyword: "doctor"}]
   *
   * // Concept extraction
   * const concepts = await inferTypes("machine learning")
   * // → [{type: Concept, confidence: 0.95, keyword: "machine learning"}]
   * ```
   */
  async inferTypes(
    text: string,
    options: SemanticTypeInferenceOptions = {}
  ): Promise<TypeInference[]> {
    const startTime = performance.now()

    // Ensure HNSW index is initialized
    await this.initPromise

    // Normalize text
    const normalized = text.toLowerCase().trim()

    if (!normalized) {
      return []
    }

    try {
      // Get or compute embedding
      const embedding = options.useCache !== false
        ? await this.getOrComputeEmbedding(normalized)
        : await this.computeEmbedding(normalized)

      // Search HNSW index (O(log n) semantic search)
      const k = options.maxResults ?? 5
      const candidates = await this.keywordHNSW.search(embedding, k * 3)  // Fetch extra for filtering

      // Convert to TypeInference results
      const results: TypeInference[] = []

      for (const [idStr, distance] of candidates) {
        const id = parseInt(idStr, 10)
        const keyword = this.keywordEmbeddings[id]

        // Apply category filter
        if (options.filterCategory && keyword.typeCategory !== options.filterCategory) {
          continue
        }

        // Apply type filter
        if (options.filterTypes && !options.filterTypes.includes(keyword.type)) {
          continue
        }

        // Calculate combined confidence (similarity * base confidence)
        const confidence = distance * keyword.confidence

        // Apply confidence threshold
        if (confidence < (options.minConfidence ?? 0.5)) {
          continue
        }

        results.push({
          type: keyword.type,
          typeCategory: keyword.typeCategory,
          confidence,
          matchedKeywords: [keyword.keyword],
          similarity: distance,
          baseConfidence: keyword.confidence
        })

        // Stop once we have enough results
        if (results.length >= k) break
      }

      const elapsed = performance.now() - startTime
      const cacheHit = this.embeddingCache.has(normalized)

      if (elapsed > 10) {
        prodLog.debug(
          `Semantic type inference: ${results.length} types in ${elapsed.toFixed(2)}ms ` +
          `(${cacheHit ? 'cached' : 'computed'} embedding)`
        )
      }

      return results
    } catch (error: any) {
      prodLog.error(`Semantic type inference failed: ${error.message}`)
      return []
    }
  }

  /**
   * Get embedding from cache or compute
   */
  private async getOrComputeEmbedding(text: string): Promise<Vector> {
    // Check cache
    const cached = this.embeddingCache.get(text)
    if (cached) {
      return cached
    }

    // Compute embedding
    const embedding = await this.computeEmbedding(text)

    // Add to cache (with size limit)
    if (this.embeddingCache.size >= this.CACHE_MAX_SIZE) {
      // Remove oldest entry (first entry in Map)
      const firstKey = this.embeddingCache.keys().next().value
      if (firstKey !== undefined) {
        this.embeddingCache.delete(firstKey)
      }
    }
    this.embeddingCache.set(text, embedding)

    return embedding
  }

  /**
   * Compute text embedding using TransformerEmbedding
   */
  private async computeEmbedding(text: string): Promise<Vector> {
    // Lazy-load embedder
    if (!this.embedder) {
      this.embedder = new TransformerEmbedding({ verbose: false })
      await this.embedder.init()
    }

    return await this.embedder.embed(text)
  }

  /**
   * Get statistics about the inference system
   */
  getStats() {
    const canonical = this.keywordEmbeddings.filter(k => k.isCanonical).length
    const synonyms = this.keywordEmbeddings.filter(k => !k.isCanonical).length

    return {
      totalKeywords: this.keywordEmbeddings.length,
      canonicalKeywords: canonical,
      synonymKeywords: synonyms,
      cacheSize: this.embeddingCache.size,
      cacheMaxSize: this.CACHE_MAX_SIZE
    }
  }

  /**
   * Clear embedding cache
   */
  clearCache() {
    this.embeddingCache.clear()
  }
}

/**
 * Global singleton instance
 */
let globalInstance: SemanticTypeInference | null = null

/**
 * Get or create the global SemanticTypeInference instance
 */
export function getSemanticTypeInference(): SemanticTypeInference {
  if (!globalInstance) {
    globalInstance = new SemanticTypeInference()
  }
  return globalInstance
}

/**
 * THE ONE FUNCTION - Public API for semantic type inference
 *
 * Infer entity types from natural language text using semantic similarity.
 *
 * @param text - Natural language text (query, entity name, concept)
 * @param options - Configuration options
 * @returns Array of type inferences sorted by confidence (highest first)
 *
 * @example
 * ```typescript
 * import { inferTypes } from '@soulcraft/brainy'
 *
 * // Query routing
 * const types = await inferTypes("Find cardiologists in San Francisco")
 * // → [
 * //   {type: "person", confidence: 0.92, keyword: "cardiologist"},
 * //   {type: "location", confidence: 0.88, keyword: "san francisco"}
 * // ]
 *
 * // Entity extraction
 * const entities = await inferTypes("Dr. Sarah Chen works at UCSF")
 * // → [
 * //   {type: "person", confidence: 0.90, keyword: "doctor"},
 * //   {type: "organization", confidence: 0.82, keyword: "ucsf"}
 * // ]
 *
 * // Concept extraction
 * const concepts = await inferTypes("machine learning algorithms")
 * // → [{type: "concept", confidence: 0.95, keyword: "machine learning"}]
 *
 * // Filter by specific types
 * const people = await inferTypes("Find doctors", {
 *   filterTypes: [NounType.Person],
 *   maxResults: 3
 * })
 * ```
 */
export async function inferTypes(
  text: string,
  options?: SemanticTypeInferenceOptions
): Promise<TypeInference[]> {
  return getSemanticTypeInference().inferTypes(text, options)
}

/**
 * Convenience function - Infer noun types only
 *
 * Filters results to noun types (Person, Organization, Location, etc.)
 *
 * @param text - Natural language text
 * @param options - Configuration options
 * @returns Array of noun type inferences
 *
 * @example
 * ```typescript
 * import { inferNouns } from '@soulcraft/brainy'
 *
 * const entities = await inferNouns("Dr. Sarah Chen works at UCSF")
 * // → [
 * //   {type: "person", typeCategory: "noun", confidence: 0.90},
 * //   {type: "organization", typeCategory: "noun", confidence: 0.82}
 * // ]
 * ```
 */
export async function inferNouns(
  text: string,
  options?: Omit<SemanticTypeInferenceOptions, 'filterCategory'>
): Promise<TypeInference[]> {
  return getSemanticTypeInference().inferTypes(text, {
    ...options,
    filterCategory: 'noun'
  })
}

/**
 * Convenience function - Infer verb types only
 *
 * Filters results to verb types (Creates, Transforms, MemberOf, etc.)
 *
 * @param text - Natural language text
 * @param options - Configuration options
 * @returns Array of verb type inferences
 *
 * @example
 * ```typescript
 * import { inferVerbs } from '@soulcraft/brainy'
 *
 * const actions = await inferVerbs("creates and transforms data")
 * // → [
 * //   {type: "creates", typeCategory: "verb", confidence: 0.95},
 * //   {type: "transforms", typeCategory: "verb", confidence: 0.93}
 * // ]
 * ```
 */
export async function inferVerbs(
  text: string,
  options?: Omit<SemanticTypeInferenceOptions, 'filterCategory'>
): Promise<TypeInference[]> {
  return getSemanticTypeInference().inferTypes(text, {
    ...options,
    filterCategory: 'verb'
  })
}

/**
 * Infer query intent - Returns both nouns AND verbs separately
 *
 * Best for complete query understanding. Returns structured intent with
 * entities (nouns) and actions (verbs) identified separately.
 *
 * @param text - Natural language query
 * @param options - Configuration options
 * @returns Structured intent with separate noun and verb inferences
 *
 * @example
 * ```typescript
 * import { inferIntent } from '@soulcraft/brainy'
 *
 * const intent = await inferIntent("Find doctors who work at UCSF")
 * // → {
 * //   nouns: [
 * //     {type: "person", confidence: 0.92, matchedKeywords: ["doctors"]},
 * //     {type: "organization", confidence: 0.85, matchedKeywords: ["ucsf"]}
 * //   ],
 * //   verbs: [
 * //     {type: "memberOf", confidence: 0.88, matchedKeywords: ["work at"]}
 * //   ]
 * // }
 * ```
 */
export async function inferIntent(
  text: string,
  options?: Omit<SemanticTypeInferenceOptions, 'filterCategory'>
): Promise<{ nouns: TypeInference[]; verbs: TypeInference[] }> {
  // Run inference once to get all types
  const allTypes = await getSemanticTypeInference().inferTypes(text, {
    ...options,
    maxResults: (options?.maxResults ?? 5) * 2  // Get more results since we're splitting
  })

  // Split into nouns and verbs
  const nouns = allTypes.filter(t => t.typeCategory === 'noun')
  const verbs = allTypes.filter(t => t.typeCategory === 'verb')

  // Limit each category to maxResults
  const limit = options?.maxResults ?? 5

  return {
    nouns: nouns.slice(0, limit),
    verbs: verbs.slice(0, limit)
  }
}