/** * ExactMatchSignal - O(1) exact match entity type classification * * HIGHEST WEIGHT: 40% (most reliable signal) * * Uses: * 1. O(1) term index lookup (exact string match) * 2. O(1) metadata hints (column names, file structure) * 3. Format-specific intelligence (Excel, CSV, PDF, YAML, DOCX) * * Finds explicit relationships via exact matching — added after a consumer * report of extraction missing explicitly-named relationships. */ import type { Brainy } from '../../brainy.js' import { NounType } from '../../types/graphTypes.js' import type { Vector } from '../../coreTypes.js' /** * Signal result with classification details */ export interface TypeSignal { source: 'exact-term' | 'exact-metadata' | 'exact-format' type: NounType confidence: number evidence: string metadata?: { matchedTerm?: string columnHint?: string formatHint?: string } } /** * Options for exact match signal */ export interface ExactMatchSignalOptions { minConfidence?: number // Minimum confidence threshold (default: 0.85) cacheSize?: number // LRU cache size (default: 5000) // Format-specific detection enableFormatHints?: boolean // Use format-specific intelligence (default: true) // Metadata column detection patterns columnPatterns?: { term?: string[] // ["Term", "Name", "Title", "Entity"] type?: string[] // ["Type", "Category", "Kind"] definition?: string[] // ["Definition", "Description", "Text"] related?: string[] // ["Related", "See Also", "References"] } } /** * Term index entry with type information */ interface TermEntry { term: string // Original term text type: NounType // Classified type confidence: number // Classification confidence source: string // Where it came from } /** * Format-specific hint from file structure */ interface FormatHint { type: NounType confidence: number evidence: string } /** * ExactMatchSignal - Instant O(1) type classification via exact matching * * Production features: * - O(1) hash table lookups (fastest possible) * - Format-specific intelligence (Excel columns, CSV headers, etc.) * - Metadata hints (column names reveal entity types) * - LRU cache for hot paths * - Highest confidence (0.95-0.99) - most reliable signal */ export class ExactMatchSignal { private brain: Brainy private options: Required // O(1) term lookup index (key: normalized term → value: type info) private termIndex: Map = new Map() // LRU cache for hot lookups private cache: Map = new Map() private cacheOrder: string[] = [] // Statistics private stats = { calls: 0, cacheHits: 0, termMatches: 0, metadataMatches: 0, formatMatches: 0 } constructor(brain: Brainy, options?: ExactMatchSignalOptions) { this.brain = brain this.options = { minConfidence: options?.minConfidence ?? 0.85, cacheSize: options?.cacheSize ?? 5000, enableFormatHints: options?.enableFormatHints ?? true, columnPatterns: { term: options?.columnPatterns?.term ?? ['term', 'name', 'title', 'entity', 'concept'], type: options?.columnPatterns?.type ?? ['type', 'category', 'kind', 'class'], definition: options?.columnPatterns?.definition ?? ['definition', 'description', 'text', 'content'], related: options?.columnPatterns?.related ?? ['related', 'see also', 'references', 'links'] } } } /** * Build term index from import data (call once per import) * * This is O(n) upfront cost, then O(1) lookups forever * * @param terms Array of terms with their types */ buildIndex(terms: Array<{ text: string, type: NounType, confidence?: number }>): void { this.termIndex.clear() for (const term of terms) { const normalized = this.normalize(term.text) // Index full term this.termIndex.set(normalized, { term: term.text, type: term.type, confidence: term.confidence ?? 1.0, source: 'index' }) // Also index individual tokens for multi-word terms const tokens = this.tokenize(normalized) for (const token of tokens) { if (token.length >= 3 && !this.termIndex.has(token)) { this.termIndex.set(token, { term: term.text, type: term.type, confidence: (term.confidence ?? 1.0) * 0.8, // Slight discount for partial match source: 'token' }) } } } } /** * Classify entity type using exact matching * * Main entry point - checks term index, metadata, and format hints * * @param candidate Entity text to classify * @param context Optional context for better matching * @returns TypeSignal with classification result or null */ async classify( candidate: string, context?: { definition?: string metadata?: Record columnName?: string fileFormat?: 'excel' | 'csv' | 'pdf' | 'json' | 'markdown' | 'yaml' | 'docx' rowData?: Record } ): Promise { this.stats.calls++ // Check cache first (O(1)) const cacheKey = this.getCacheKey(candidate, context) const cached = this.getFromCache(cacheKey) if (cached !== undefined) { this.stats.cacheHits++ return cached } // Try exact term match (O(1)) const termMatch = this.matchTerm(candidate) if (termMatch && termMatch.confidence >= this.options.minConfidence) { this.stats.termMatches++ this.addToCache(cacheKey, termMatch) return termMatch } // Try metadata hints (O(1)) if (context?.metadata || context?.columnName) { const metadataMatch = this.matchMetadata(candidate, context) if (metadataMatch && metadataMatch.confidence >= this.options.minConfidence) { this.stats.metadataMatches++ this.addToCache(cacheKey, metadataMatch) return metadataMatch } } // Try format-specific hints if (this.options.enableFormatHints && context?.fileFormat) { const formatMatch = this.matchFormat(candidate, context) if (formatMatch && formatMatch.confidence >= this.options.minConfidence) { this.stats.formatMatches++ this.addToCache(cacheKey, formatMatch) return formatMatch } } // No match found - cache null to avoid recomputation this.addToCache(cacheKey, null) return null } /** * Match against term index (O(1)) * * Highest confidence - exact string match */ private matchTerm(candidate: string): TypeSignal | null { const normalized = this.normalize(candidate) const entry = this.termIndex.get(normalized) if (!entry) return null return { source: 'exact-term', type: entry.type, confidence: entry.confidence * 0.99, // 0.99 for exact term match evidence: `Exact match in term index: "${entry.term}"`, metadata: { matchedTerm: entry.term } } } /** * Match using metadata hints (column names, file structure) * * High confidence - structural clues reveal entity types */ private matchMetadata( candidate: string, context: { metadata?: Record columnName?: string rowData?: Record } ): TypeSignal | null { // Check column name patterns if (context.columnName) { const hint = this.detectColumnType(context.columnName, context.rowData) if (hint) { return { source: 'exact-metadata', type: hint.type, confidence: hint.confidence * 0.95, // 0.95 for metadata hints evidence: hint.evidence, metadata: { columnHint: context.columnName } } } } // Check explicit type metadata if (context.metadata?.type) { const hint = this.inferTypeFromMetadata(context.metadata.type) if (hint) { return { source: 'exact-metadata', type: hint.type, confidence: hint.confidence * 0.98, // 0.98 for explicit type evidence: hint.evidence, metadata: { columnHint: 'type' } } } } return null } /** * Match using format-specific intelligence * * Excel, CSV, PDF, YAML, DOCX each have unique structural patterns */ private matchFormat( candidate: string, context: { fileFormat?: string rowData?: Record definition?: string metadata?: Record } ): TypeSignal | null { if (!context.fileFormat) return null switch (context.fileFormat) { case 'excel': return this.detectExcelPatterns(candidate, context) case 'csv': return this.detectCSVPatterns(candidate, context) case 'pdf': return this.detectPDFPatterns(candidate, context) case 'yaml': return this.detectYAMLPatterns(candidate, context) case 'docx': return this.detectDOCXPatterns(candidate, context) default: return null } } /** * Detect Excel-specific patterns * * - Cell formats (dates, currencies) * - Named ranges * - Column headers reveal entity types * - Sheet names as categories */ private detectExcelPatterns( candidate: string, context: { rowData?: Record, metadata?: Record } ): TypeSignal | null { // Sheet name hints if (context.metadata?.sheetName) { const sheetHint = this.inferTypeFromSheetName(context.metadata.sheetName) if (sheetHint) { return { source: 'exact-format', type: sheetHint.type, confidence: sheetHint.confidence * 0.90, evidence: `Excel sheet name: "${context.metadata.sheetName}"`, metadata: { formatHint: 'excel-sheet' } } } } // Column position hints (first column often = entity name) if (context.metadata?.columnIndex === 0) { // First column is often the primary entity // But don't return a type without more evidence } return null } /** * Detect CSV-specific patterns * * - Relationship columns (parent_id, created_by) * - Nested delimiters (semicolons, pipes) * - URL columns indicate external references */ private detectCSVPatterns( candidate: string, context: { rowData?: Record } ): TypeSignal | null { if (!context.rowData) return null // Check for relationship columns const keys = Object.keys(context.rowData) // parent_id → indicates hierarchical structure if (keys.some(k => k.toLowerCase().includes('parent'))) { // This entity is part of a hierarchy } // URL column → external reference const urlPattern = /^https?:\/\// if (typeof candidate === 'string' && urlPattern.test(candidate)) { // Don't classify URLs as entities - they're references return null } return null } /** * Detect PDF-specific patterns * * - Table of contents entries * - Section headings * - Citation references * - Figure captions */ private detectPDFPatterns( candidate: string, context: { metadata?: Record } ): TypeSignal | null { // TOC entry → likely a concept or topic if (context.metadata?.isTOCEntry) { return { source: 'exact-format', type: NounType.Concept, confidence: 0.88, evidence: 'PDF table of contents entry', metadata: { formatHint: 'pdf-toc' } } } return null } /** * Detect YAML-specific patterns * * - Key names reveal entity types * - Nested structure indicates relationships * - Lists indicate collections */ private detectYAMLPatterns( candidate: string, context: { metadata?: Record } ): TypeSignal | null { if (!context.metadata?.yamlKey) return null const key = context.metadata.yamlKey.toLowerCase() // Common YAML patterns if (key.includes('user') || key.includes('author')) { return { source: 'exact-format', type: NounType.Person, confidence: 0.90, evidence: `YAML key indicates person: "${context.metadata.yamlKey}"`, metadata: { formatHint: 'yaml-key' } } } if (key.includes('organization') || key.includes('company')) { return { source: 'exact-format', type: NounType.Organization, confidence: 0.92, evidence: `YAML key indicates organization: "${context.metadata.yamlKey}"`, metadata: { formatHint: 'yaml-key' } } } return null } /** * Detect DOCX-specific patterns * * - Heading levels indicate hierarchy * - List items indicate collections * - Comments indicate relationships * - Track changes reveal authorship */ private detectDOCXPatterns( candidate: string, context: { metadata?: Record } ): TypeSignal | null { // Heading level → concept hierarchy if (context.metadata?.headingLevel) { return { source: 'exact-format', type: NounType.Concept, confidence: 0.87, evidence: `DOCX heading (level ${context.metadata.headingLevel})`, metadata: { formatHint: 'docx-heading' } } } return null } /** * Detect entity type from column name patterns */ private detectColumnType( columnName: string, rowData?: Record ): FormatHint | null { const lower = columnName.toLowerCase() // Location indicators if (lower.includes('location') || lower.includes('place') || lower.includes('city') || lower.includes('country')) { return { type: NounType.Location, confidence: 0.92, evidence: `Column name indicates location: "${columnName}"` } } // Person indicators if (lower.includes('person') || lower.includes('author') || lower.includes('user') || lower.includes('name') && (lower.includes('first') || lower.includes('last'))) { return { type: NounType.Person, confidence: 0.90, evidence: `Column name indicates person: "${columnName}"` } } // Organization indicators if (lower.includes('organization') || lower.includes('company') || lower.includes('institution') || lower.includes('org')) { return { type: NounType.Organization, confidence: 0.91, evidence: `Column name indicates organization: "${columnName}"` } } return null } /** * Infer type from explicit type metadata */ private inferTypeFromMetadata(typeValue: any): FormatHint | null { if (typeof typeValue !== 'string') return null const lower = typeValue.toLowerCase() // Direct mapping const typeMap: Record = { 'person': NounType.Person, 'people': NounType.Person, 'location': NounType.Location, 'place': NounType.Location, 'organization': NounType.Organization, 'company': NounType.Organization, 'concept': NounType.Concept, 'idea': NounType.Concept, 'event': NounType.Event, 'document': NounType.Document, 'file': NounType.File, 'product': NounType.Product, 'service': NounType.Service } const type = typeMap[lower] if (type) { return { type, confidence: 0.98, evidence: `Explicit type metadata: "${typeValue}"` } } return null } /** * Infer type from Excel sheet name */ private inferTypeFromSheetName(sheetName: string): FormatHint | null { const lower = sheetName.toLowerCase() if (lower.includes('character') || lower.includes('people') || lower.includes('person')) { return { type: NounType.Person, confidence: 0.88, evidence: `Sheet name suggests people: "${sheetName}"` } } if (lower.includes('location') || lower.includes('place') || lower.includes('map')) { return { type: NounType.Location, confidence: 0.87, evidence: `Sheet name suggests locations: "${sheetName}"` } } if (lower.includes('concept') || lower.includes('glossary') || lower.includes('term')) { return { type: NounType.Concept, confidence: 0.85, evidence: `Sheet name suggests concepts: "${sheetName}"` } } return null } /** * Get index size */ getIndexSize(): number { return this.termIndex.size } /** * Get statistics */ getStats() { return { ...this.stats, indexSize: this.termIndex.size, cacheSize: this.cache.size, cacheHitRate: this.stats.calls > 0 ? this.stats.cacheHits / this.stats.calls : 0, termMatchRate: this.stats.calls > 0 ? this.stats.termMatches / this.stats.calls : 0, metadataMatchRate: this.stats.calls > 0 ? this.stats.metadataMatches / this.stats.calls : 0, formatMatchRate: this.stats.calls > 0 ? this.stats.formatMatches / this.stats.calls : 0 } } /** * Reset statistics */ resetStats(): void { this.stats = { calls: 0, cacheHits: 0, termMatches: 0, metadataMatches: 0, formatMatches: 0 } } /** * Clear cache */ clearCache(): void { this.cache.clear() this.cacheOrder = [] } /** * Clear index */ clearIndex(): void { this.termIndex.clear() } // ========== Private Helper Methods ========== /** * Normalize text for matching */ private normalize(text: string): string { return text.toLowerCase().trim() } /** * Tokenize text into words */ private tokenize(text: string): string[] { return text.toLowerCase().split(/\W+/).filter(t => t.length >= 3) } /** * Generate cache key */ private getCacheKey(candidate: string, context?: any): string { const normalized = this.normalize(candidate) if (!context) return normalized const parts = [normalized] if (context.columnName) parts.push(context.columnName) if (context.fileFormat) parts.push(context.fileFormat) return parts.join(':') } /** * Get from LRU cache */ private getFromCache(key: string): TypeSignal | null | undefined { if (!this.cache.has(key)) return undefined const cached = this.cache.get(key) // Move to end (most recently used) this.cacheOrder = this.cacheOrder.filter(k => k !== key) this.cacheOrder.push(key) return cached ?? null } /** * Add to LRU cache with eviction */ private addToCache(key: string, value: TypeSignal | null): void { this.cache.set(key, value) this.cacheOrder.push(key) // Evict oldest if over limit if (this.cache.size > this.options.cacheSize) { const oldest = this.cacheOrder.shift() if (oldest) { this.cache.delete(oldest) } } } } /** * Create a new ExactMatchSignal instance */ export function createExactMatchSignal( brain: Brainy, options?: ExactMatchSignalOptions ): ExactMatchSignal { return new ExactMatchSignal(brain, options) }