open-brainy/src/neural/signals/ExactMatchSignal.ts

/**
 * ExactMatchSignal - O(1) exact match entity type classification
 *
 * HIGHEST WEIGHT: 40% (most reliable signal)
 *
 * Uses:
 * 1. O(1) term index lookup (exact string match)
 * 2. O(1) metadata hints (column names, file structure)
 * 3. Format-specific intelligence (Excel, CSV, PDF, YAML, DOCX)
 *
 * Finds explicit relationships via exact matching — added after a consumer
 * report of extraction missing explicitly-named relationships.
 */

import type { Brainy } from '../../brainy.js'
import { NounType } from '../../types/graphTypes.js'
import type { Vector } from '../../coreTypes.js'

/**
 * Signal result with classification details
 */
export interface TypeSignal {
  source: 'exact-term' | 'exact-metadata' | 'exact-format'
  type: NounType
  confidence: number
  evidence: string
  metadata?: {
    matchedTerm?: string
    columnHint?: string
    formatHint?: string
  }
}

/**
 * Options for exact match signal
 */
export interface ExactMatchSignalOptions {
  minConfidence?: number // Minimum confidence threshold (default: 0.85)
  cacheSize?: number     // LRU cache size (default: 5000)

  // Format-specific detection
  enableFormatHints?: boolean  // Use format-specific intelligence (default: true)

  // Metadata column detection patterns
  columnPatterns?: {
    term?: string[]         // ["Term", "Name", "Title", "Entity"]
    type?: string[]         // ["Type", "Category", "Kind"]
    definition?: string[]   // ["Definition", "Description", "Text"]
    related?: string[]      // ["Related", "See Also", "References"]
  }
}

/**
 * Term index entry with type information
 */
interface TermEntry {
  term: string          // Original term text
  type: NounType        // Classified type
  confidence: number    // Classification confidence
  source: string        // Where it came from
}

/**
 * Format-specific hint from file structure
 */
interface FormatHint {
  type: NounType
  confidence: number
  evidence: string
}

/**
 * ExactMatchSignal - Instant O(1) type classification via exact matching
 *
 * Production features:
 * - O(1) hash table lookups (fastest possible)
 * - Format-specific intelligence (Excel columns, CSV headers, etc.)
 * - Metadata hints (column names reveal entity types)
 * - LRU cache for hot paths
 * - Highest confidence (0.95-0.99) - most reliable signal
 */
export class ExactMatchSignal {
  private brain: Brainy
  private options: Required<ExactMatchSignalOptions>

  // O(1) term lookup index (key: normalized term → value: type info)
  private termIndex: Map<string, TermEntry> = new Map()

  // LRU cache for hot lookups
  private cache: Map<string, TypeSignal | null> = new Map()
  private cacheOrder: string[] = []

  // Statistics
  private stats = {
    calls: 0,
    cacheHits: 0,
    termMatches: 0,
    metadataMatches: 0,
    formatMatches: 0
  }

  constructor(brain: Brainy, options?: ExactMatchSignalOptions) {
    this.brain = brain
    this.options = {
      minConfidence: options?.minConfidence ?? 0.85,
      cacheSize: options?.cacheSize ?? 5000,
      enableFormatHints: options?.enableFormatHints ?? true,
      columnPatterns: {
        term: options?.columnPatterns?.term ?? ['term', 'name', 'title', 'entity', 'concept'],
        type: options?.columnPatterns?.type ?? ['type', 'category', 'kind', 'class'],
        definition: options?.columnPatterns?.definition ?? ['definition', 'description', 'text', 'content'],
        related: options?.columnPatterns?.related ?? ['related', 'see also', 'references', 'links']
      }
    }
  }

  /**
   * Build term index from import data (call once per import)
   *
   * This is O(n) upfront cost, then O(1) lookups forever
   *
   * @param terms Array of terms with their types
   */
  buildIndex(terms: Array<{ text: string, type: NounType, confidence?: number }>): void {
    this.termIndex.clear()

    for (const term of terms) {
      const normalized = this.normalize(term.text)

      // Index full term
      this.termIndex.set(normalized, {
        term: term.text,
        type: term.type,
        confidence: term.confidence ?? 1.0,
        source: 'index'
      })

      // Also index individual tokens for multi-word terms
      const tokens = this.tokenize(normalized)
      for (const token of tokens) {
        if (token.length >= 3 && !this.termIndex.has(token)) {
          this.termIndex.set(token, {
            term: term.text,
            type: term.type,
            confidence: (term.confidence ?? 1.0) * 0.8, // Slight discount for partial match
            source: 'token'
          })
        }
      }
    }
  }

  /**
   * Classify entity type using exact matching
   *
   * Main entry point - checks term index, metadata, and format hints
   *
   * @param candidate Entity text to classify
   * @param context Optional context for better matching
   * @returns TypeSignal with classification result or null
   */
  async classify(
    candidate: string,
    context?: {
      definition?: string
      metadata?: Record<string, any>
      columnName?: string
      fileFormat?: 'excel' | 'csv' | 'pdf' | 'json' | 'markdown' | 'yaml' | 'docx'
      rowData?: Record<string, any>
    }
  ): Promise<TypeSignal | null> {
    this.stats.calls++

    // Check cache first (O(1))
    const cacheKey = this.getCacheKey(candidate, context)
    const cached = this.getFromCache(cacheKey)
    if (cached !== undefined) {
      this.stats.cacheHits++
      return cached
    }

    // Try exact term match (O(1))
    const termMatch = this.matchTerm(candidate)
    if (termMatch && termMatch.confidence >= this.options.minConfidence) {
      this.stats.termMatches++
      this.addToCache(cacheKey, termMatch)
      return termMatch
    }

    // Try metadata hints (O(1))
    if (context?.metadata || context?.columnName) {
      const metadataMatch = this.matchMetadata(candidate, context)
      if (metadataMatch && metadataMatch.confidence >= this.options.minConfidence) {
        this.stats.metadataMatches++
        this.addToCache(cacheKey, metadataMatch)
        return metadataMatch
      }
    }

    // Try format-specific hints
    if (this.options.enableFormatHints && context?.fileFormat) {
      const formatMatch = this.matchFormat(candidate, context)
      if (formatMatch && formatMatch.confidence >= this.options.minConfidence) {
        this.stats.formatMatches++
        this.addToCache(cacheKey, formatMatch)
        return formatMatch
      }
    }

    // No match found - cache null to avoid recomputation
    this.addToCache(cacheKey, null)
    return null
  }

  /**
   * Match against term index (O(1))
   *
   * Highest confidence - exact string match
   */
  private matchTerm(candidate: string): TypeSignal | null {
    const normalized = this.normalize(candidate)
    const entry = this.termIndex.get(normalized)

    if (!entry) return null

    return {
      source: 'exact-term',
      type: entry.type,
      confidence: entry.confidence * 0.99, // 0.99 for exact term match
      evidence: `Exact match in term index: "${entry.term}"`,
      metadata: {
        matchedTerm: entry.term
      }
    }
  }

  /**
   * Match using metadata hints (column names, file structure)
   *
   * High confidence - structural clues reveal entity types
   */
  private matchMetadata(
    candidate: string,
    context: {
      metadata?: Record<string, any>
      columnName?: string
      rowData?: Record<string, any>
    }
  ): TypeSignal | null {
    // Check column name patterns
    if (context.columnName) {
      const hint = this.detectColumnType(context.columnName, context.rowData)
      if (hint) {
        return {
          source: 'exact-metadata',
          type: hint.type,
          confidence: hint.confidence * 0.95, // 0.95 for metadata hints
          evidence: hint.evidence,
          metadata: {
            columnHint: context.columnName
          }
        }
      }
    }

    // Check explicit type metadata
    if (context.metadata?.type) {
      const hint = this.inferTypeFromMetadata(context.metadata.type)
      if (hint) {
        return {
          source: 'exact-metadata',
          type: hint.type,
          confidence: hint.confidence * 0.98, // 0.98 for explicit type
          evidence: hint.evidence,
          metadata: {
            columnHint: 'type'
          }
        }
      }
    }

    return null
  }

  /**
   * Match using format-specific intelligence
   *
   * Excel, CSV, PDF, YAML, DOCX each have unique structural patterns
   */
  private matchFormat(
    candidate: string,
    context: {
      fileFormat?: string
      rowData?: Record<string, any>
      definition?: string
      metadata?: Record<string, any>
    }
  ): TypeSignal | null {
    if (!context.fileFormat) return null

    switch (context.fileFormat) {
      case 'excel':
        return this.detectExcelPatterns(candidate, context)

      case 'csv':
        return this.detectCSVPatterns(candidate, context)

      case 'pdf':
        return this.detectPDFPatterns(candidate, context)

      case 'yaml':
        return this.detectYAMLPatterns(candidate, context)

      case 'docx':
        return this.detectDOCXPatterns(candidate, context)

      default:
        return null
    }
  }

  /**
   * Detect Excel-specific patterns
   *
   * - Cell formats (dates, currencies)
   * - Named ranges
   * - Column headers reveal entity types
   * - Sheet names as categories
   */
  private detectExcelPatterns(
    candidate: string,
    context: { rowData?: Record<string, any>, metadata?: Record<string, any> }
  ): TypeSignal | null {
    // Sheet name hints
    if (context.metadata?.sheetName) {
      const sheetHint = this.inferTypeFromSheetName(context.metadata.sheetName)
      if (sheetHint) {
        return {
          source: 'exact-format',
          type: sheetHint.type,
          confidence: sheetHint.confidence * 0.90,
          evidence: `Excel sheet name: "${context.metadata.sheetName}"`,
          metadata: { formatHint: 'excel-sheet' }
        }
      }
    }

    // Column position hints (first column often = entity name)
    if (context.metadata?.columnIndex === 0) {
      // First column is often the primary entity
      // But don't return a type without more evidence
    }

    return null
  }

  /**
   * Detect CSV-specific patterns
   *
   * - Relationship columns (parent_id, created_by)
   * - Nested delimiters (semicolons, pipes)
   * - URL columns indicate external references
   */
  private detectCSVPatterns(
    candidate: string,
    context: { rowData?: Record<string, any> }
  ): TypeSignal | null {
    if (!context.rowData) return null

    // Check for relationship columns
    const keys = Object.keys(context.rowData)

    // parent_id → indicates hierarchical structure
    if (keys.some(k => k.toLowerCase().includes('parent'))) {
      // This entity is part of a hierarchy
    }

    // URL column → external reference
    const urlPattern = /^https?:\/\//
    if (typeof candidate === 'string' && urlPattern.test(candidate)) {
      // Don't classify URLs as entities - they're references
      return null
    }

    return null
  }

  /**
   * Detect PDF-specific patterns
   *
   * - Table of contents entries
   * - Section headings
   * - Citation references
   * - Figure captions
   */
  private detectPDFPatterns(
    candidate: string,
    context: { metadata?: Record<string, any> }
  ): TypeSignal | null {
    // TOC entry → likely a concept or topic
    if (context.metadata?.isTOCEntry) {
      return {
        source: 'exact-format',
        type: NounType.Concept,
        confidence: 0.88,
        evidence: 'PDF table of contents entry',
        metadata: { formatHint: 'pdf-toc' }
      }
    }

    return null
  }

  /**
   * Detect YAML-specific patterns
   *
   * - Key names reveal entity types
   * - Nested structure indicates relationships
   * - Lists indicate collections
   */
  private detectYAMLPatterns(
    candidate: string,
    context: { metadata?: Record<string, any> }
  ): TypeSignal | null {
    if (!context.metadata?.yamlKey) return null

    const key = context.metadata.yamlKey.toLowerCase()

    // Common YAML patterns
    if (key.includes('user') || key.includes('author')) {
      return {
        source: 'exact-format',
        type: NounType.Person,
        confidence: 0.90,
        evidence: `YAML key indicates person: "${context.metadata.yamlKey}"`,
        metadata: { formatHint: 'yaml-key' }
      }
    }

    if (key.includes('organization') || key.includes('company')) {
      return {
        source: 'exact-format',
        type: NounType.Organization,
        confidence: 0.92,
        evidence: `YAML key indicates organization: "${context.metadata.yamlKey}"`,
        metadata: { formatHint: 'yaml-key' }
      }
    }

    return null
  }

  /**
   * Detect DOCX-specific patterns
   *
   * - Heading levels indicate hierarchy
   * - List items indicate collections
   * - Comments indicate relationships
   * - Track changes reveal authorship
   */
  private detectDOCXPatterns(
    candidate: string,
    context: { metadata?: Record<string, any> }
  ): TypeSignal | null {
    // Heading level → concept hierarchy
    if (context.metadata?.headingLevel) {
      return {
        source: 'exact-format',
        type: NounType.Concept,
        confidence: 0.87,
        evidence: `DOCX heading (level ${context.metadata.headingLevel})`,
        metadata: { formatHint: 'docx-heading' }
      }
    }

    return null
  }

  /**
   * Detect entity type from column name patterns
   */
  private detectColumnType(
    columnName: string,
    rowData?: Record<string, any>
  ): FormatHint | null {
    const lower = columnName.toLowerCase()

    // Location indicators
    if (lower.includes('location') || lower.includes('place') ||
        lower.includes('city') || lower.includes('country')) {
      return {
        type: NounType.Location,
        confidence: 0.92,
        evidence: `Column name indicates location: "${columnName}"`
      }
    }

    // Person indicators
    if (lower.includes('person') || lower.includes('author') ||
        lower.includes('user') || lower.includes('name') &&
        (lower.includes('first') || lower.includes('last'))) {
      return {
        type: NounType.Person,
        confidence: 0.90,
        evidence: `Column name indicates person: "${columnName}"`
      }
    }

    // Organization indicators
    if (lower.includes('organization') || lower.includes('company') ||
        lower.includes('institution') || lower.includes('org')) {
      return {
        type: NounType.Organization,
        confidence: 0.91,
        evidence: `Column name indicates organization: "${columnName}"`
      }
    }

    return null
  }

  /**
   * Infer type from explicit type metadata
   */
  private inferTypeFromMetadata(typeValue: any): FormatHint | null {
    if (typeof typeValue !== 'string') return null

    const lower = typeValue.toLowerCase()

    // Direct mapping
    const typeMap: Record<string, NounType> = {
      'person': NounType.Person,
      'people': NounType.Person,
      'location': NounType.Location,
      'place': NounType.Location,
      'organization': NounType.Organization,
      'company': NounType.Organization,
      'concept': NounType.Concept,
      'idea': NounType.Concept,
      'event': NounType.Event,
      'document': NounType.Document,
      'file': NounType.File,
      'product': NounType.Product,
      'service': NounType.Service
    }

    const type = typeMap[lower]
    if (type) {
      return {
        type,
        confidence: 0.98,
        evidence: `Explicit type metadata: "${typeValue}"`
      }
    }

    return null
  }

  /**
   * Infer type from Excel sheet name
   */
  private inferTypeFromSheetName(sheetName: string): FormatHint | null {
    const lower = sheetName.toLowerCase()

    if (lower.includes('character') || lower.includes('people') || lower.includes('person')) {
      return {
        type: NounType.Person,
        confidence: 0.88,
        evidence: `Sheet name suggests people: "${sheetName}"`
      }
    }

    if (lower.includes('location') || lower.includes('place') || lower.includes('map')) {
      return {
        type: NounType.Location,
        confidence: 0.87,
        evidence: `Sheet name suggests locations: "${sheetName}"`
      }
    }

    if (lower.includes('concept') || lower.includes('glossary') || lower.includes('term')) {
      return {
        type: NounType.Concept,
        confidence: 0.85,
        evidence: `Sheet name suggests concepts: "${sheetName}"`
      }
    }

    return null
  }

  /**
   * Get index size
   */
  getIndexSize(): number {
    return this.termIndex.size
  }

  /**
   * Get statistics
   */
  getStats() {
    return {
      ...this.stats,
      indexSize: this.termIndex.size,
      cacheSize: this.cache.size,
      cacheHitRate: this.stats.calls > 0 ? this.stats.cacheHits / this.stats.calls : 0,
      termMatchRate: this.stats.calls > 0 ? this.stats.termMatches / this.stats.calls : 0,
      metadataMatchRate: this.stats.calls > 0 ? this.stats.metadataMatches / this.stats.calls : 0,
      formatMatchRate: this.stats.calls > 0 ? this.stats.formatMatches / this.stats.calls : 0
    }
  }

  /**
   * Reset statistics
   */
  resetStats(): void {
    this.stats = {
      calls: 0,
      cacheHits: 0,
      termMatches: 0,
      metadataMatches: 0,
      formatMatches: 0
    }
  }

  /**
   * Clear cache
   */
  clearCache(): void {
    this.cache.clear()
    this.cacheOrder = []
  }

  /**
   * Clear index
   */
  clearIndex(): void {
    this.termIndex.clear()
  }

  // ========== Private Helper Methods ==========

  /**
   * Normalize text for matching
   */
  private normalize(text: string): string {
    return text.toLowerCase().trim()
  }

  /**
   * Tokenize text into words
   */
  private tokenize(text: string): string[] {
    return text.toLowerCase().split(/\W+/).filter(t => t.length >= 3)
  }

  /**
   * Generate cache key
   */
  private getCacheKey(candidate: string, context?: any): string {
    const normalized = this.normalize(candidate)
    if (!context) return normalized

    const parts = [normalized]
    if (context.columnName) parts.push(context.columnName)
    if (context.fileFormat) parts.push(context.fileFormat)

    return parts.join(':')
  }

  /**
   * Get from LRU cache
   */
  private getFromCache(key: string): TypeSignal | null | undefined {
    if (!this.cache.has(key)) return undefined

    const cached = this.cache.get(key)

    // Move to end (most recently used)
    this.cacheOrder = this.cacheOrder.filter(k => k !== key)
    this.cacheOrder.push(key)

    return cached ?? null
  }

  /**
   * Add to LRU cache with eviction
   */
  private addToCache(key: string, value: TypeSignal | null): void {
    this.cache.set(key, value)
    this.cacheOrder.push(key)

    // Evict oldest if over limit
    if (this.cache.size > this.options.cacheSize) {
      const oldest = this.cacheOrder.shift()
      if (oldest) {
        this.cache.delete(oldest)
      }
    }
  }
}

/**
 * Create a new ExactMatchSignal instance
 */
export function createExactMatchSignal(
  brain: Brainy,
  options?: ExactMatchSignalOptions
): ExactMatchSignal {
  return new ExactMatchSignal(brain, options)
}