David Snelling
0d649b8a79
Major optimization - all type embeddings now built into package: Build-time generation: - Created scripts/buildTypeEmbeddings.ts to generate all type embeddings - Generates embeddings for 31 NounTypes + 40 VerbTypes at build time - Stores as base64-encoded binary data in embeddedTypeEmbeddings.ts - Added check script to rebuild only when needed Updated all consumers: - NeuralEntityExtractor: loads pre-computed embeddings (instant) - BrainyTypes: loads pre-computed embeddings (instant init) - NaturalLanguageProcessor: loads pre-computed embeddings (instant init) Build process: - Added npm run build:types to generate embeddings - Added npm run build:types:if-needed for conditional rebuild - Integrated into main build pipeline - Auto-rebuilds only when types or build script change Benefits: - Zero runtime cost - embeddings loaded instantly - Survives all container restarts - All 71 types always available (31 nouns + 40 verbs) - ~100KB memory overhead for permanent performance gain - Eliminates 5-10 second initialization delay This completes the type embedding optimization started in v3.32.5
1482 lines
No EOL
48 KiB
TypeScript
1482 lines
No EOL
48 KiB
TypeScript
/**
|
|
* 🧠 Natural Language Query Processor
|
|
* Auto-breaks down natural language into structured Triple Intelligence queries
|
|
*
|
|
* Uses all of Brainy's sophisticated features:
|
|
* - Embedding model for semantic understanding
|
|
* - Pattern library with 100+ research-based patterns
|
|
* - Entity Registry for concept mapping
|
|
* - Progressive learning from usage
|
|
*/
|
|
|
|
import { Vector } from '../coreTypes.js'
|
|
import { TripleQuery } from '../triple/TripleIntelligence.js'
|
|
import { Brainy } from '../brainy.js'
|
|
import { PatternLibrary } from './patternLibrary.js'
|
|
import { NounType, VerbType } from '../types/graphTypes.js'
|
|
import { getNounTypeEmbeddings, getVerbTypeEmbeddings } from './embeddedTypeEmbeddings.js'
|
|
|
|
export interface NaturalQueryIntent {
|
|
type: 'vector' | 'field' | 'graph' | 'combined'
|
|
primaryIntent: 'search' | 'filter' | 'aggregate' | 'navigate' | 'compare' | 'explain'
|
|
confidence: number
|
|
extractedTerms: {
|
|
searchTerms?: string[]
|
|
fields?: Record<string, any>
|
|
connections?: {
|
|
entities: string[]
|
|
relationships: string[]
|
|
}
|
|
filters?: Record<string, any>
|
|
modifiers?: {
|
|
recent?: boolean
|
|
popular?: boolean
|
|
limit?: number
|
|
boost?: string
|
|
sortBy?: string
|
|
groupBy?: string
|
|
}
|
|
}
|
|
context?: {
|
|
domain?: string // e.g., 'technical', 'business', 'academic'
|
|
temporalScope?: 'past' | 'present' | 'future' | 'all'
|
|
complexity?: 'simple' | 'moderate' | 'complex'
|
|
}
|
|
}
|
|
|
|
export class NaturalLanguageProcessor {
|
|
private brain: Brainy
|
|
private patternLibrary: PatternLibrary
|
|
private queryHistory: Array<{ query: string; result: TripleQuery; success: boolean }>
|
|
private initialized: boolean = false
|
|
private embeddingCache: Map<string, Vector> = new Map()
|
|
|
|
// Field discovery with semantic matching
|
|
private fieldEmbeddings: Map<string, Vector> = new Map()
|
|
private fieldNames: string[] = []
|
|
private lastFieldRefresh: number = 0
|
|
private readonly FIELD_REFRESH_INTERVAL = 60000 // Refresh every minute
|
|
|
|
// Type embeddings for NounType and VerbType matching
|
|
private nounTypeEmbeddings: Map<string, Vector> = new Map()
|
|
private verbTypeEmbeddings: Map<string, Vector> = new Map()
|
|
private typeEmbeddingsInitialized: boolean = false
|
|
|
|
constructor(brain: Brainy) {
|
|
this.brain = brain
|
|
this.patternLibrary = new PatternLibrary(brain)
|
|
this.queryHistory = []
|
|
}
|
|
|
|
/**
|
|
* Get embedding directly using brain's embed method
|
|
*/
|
|
private async getEmbedding(text: string): Promise<Vector> {
|
|
// Check cache first
|
|
if (this.embeddingCache.has(text)) {
|
|
return this.embeddingCache.get(text)!
|
|
}
|
|
|
|
// Use brain's embed method directly to avoid recursion
|
|
const embedding = await (this.brain as any).embed(text)
|
|
|
|
// Cache the embedding
|
|
this.embeddingCache.set(text, embedding)
|
|
return embedding
|
|
}
|
|
|
|
/**
|
|
* Initialize the pattern library (lazy loading)
|
|
*/
|
|
private async ensureInitialized(): Promise<void> {
|
|
if (!this.initialized) {
|
|
await this.patternLibrary.init()
|
|
await this.initializeTypeEmbeddings() // Embed all noun/verb types
|
|
await this.refreshFieldEmbeddings() // Load field embeddings
|
|
this.initialized = true
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Initialize embeddings for all NounTypes and VerbTypes
|
|
* PRODUCTION OPTIMIZATION (v3.33.0): Uses pre-computed type embeddings
|
|
* Zero runtime cost - embeddings are loaded instantly from embedded data
|
|
*/
|
|
private async initializeTypeEmbeddings(): Promise<void> {
|
|
if (this.typeEmbeddingsInitialized) return
|
|
|
|
// Load pre-computed embeddings (instant, no computation)
|
|
const nounEmbeddings = getNounTypeEmbeddings()
|
|
const verbEmbeddings = getVerbTypeEmbeddings()
|
|
|
|
// Store noun type embeddings with all variations for lookup
|
|
for (const [type, embedding] of nounEmbeddings.entries()) {
|
|
this.nounTypeEmbeddings.set(type, embedding)
|
|
// Also store lowercase version for case-insensitive matching
|
|
this.nounTypeEmbeddings.set(type.toLowerCase(), embedding)
|
|
}
|
|
|
|
// Store verb type embeddings with all variations for lookup
|
|
for (const [type, embedding] of verbEmbeddings.entries()) {
|
|
this.verbTypeEmbeddings.set(type, embedding)
|
|
// Also store lowercase version for case-insensitive matching
|
|
this.verbTypeEmbeddings.set(type.toLowerCase(), embedding)
|
|
}
|
|
|
|
this.typeEmbeddingsInitialized = true
|
|
}
|
|
|
|
/**
|
|
* Find best matching NounType using semantic similarity
|
|
*/
|
|
private async findBestNounType(term: string): Promise<{
|
|
type: string | null
|
|
confidence: number
|
|
}> {
|
|
const termEmbedding = await this.getEmbedding(term)
|
|
|
|
let bestMatch: string | null = null
|
|
let bestScore = 0
|
|
|
|
for (const [typeName, typeEmbedding] of this.nounTypeEmbeddings) {
|
|
const similarity = this.cosineSimilarity(termEmbedding, typeEmbedding)
|
|
if (similarity > bestScore && similarity > 0.75) { // Higher threshold for types
|
|
bestScore = similarity
|
|
bestMatch = typeName
|
|
}
|
|
}
|
|
|
|
// Map back to the actual NounType value
|
|
if (bestMatch) {
|
|
for (const [key, value] of Object.entries(NounType)) {
|
|
if (key === bestMatch || value === bestMatch ||
|
|
key.toLowerCase() === bestMatch.toLowerCase()) {
|
|
return { type: value as string, confidence: bestScore }
|
|
}
|
|
}
|
|
}
|
|
|
|
return { type: null, confidence: 0 }
|
|
}
|
|
|
|
/**
|
|
* Find best matching VerbType using semantic similarity
|
|
*/
|
|
private async findBestVerbType(term: string): Promise<{
|
|
type: string | null
|
|
confidence: number
|
|
}> {
|
|
const termEmbedding = await this.getEmbedding(term)
|
|
|
|
let bestMatch: string | null = null
|
|
let bestScore = 0
|
|
|
|
for (const [typeName, typeEmbedding] of this.verbTypeEmbeddings) {
|
|
const similarity = this.cosineSimilarity(termEmbedding, typeEmbedding)
|
|
if (similarity > bestScore && similarity > 0.75) {
|
|
bestScore = similarity
|
|
bestMatch = typeName
|
|
}
|
|
}
|
|
|
|
// Map back to the actual VerbType value
|
|
if (bestMatch) {
|
|
for (const [key, value] of Object.entries(VerbType)) {
|
|
if (key === bestMatch || value === bestMatch ||
|
|
key.toLowerCase() === bestMatch.toLowerCase()) {
|
|
return { type: value as string, confidence: bestScore }
|
|
}
|
|
}
|
|
}
|
|
|
|
return { type: null, confidence: 0 }
|
|
}
|
|
|
|
/**
|
|
* Refresh field embeddings from metadata index
|
|
* Creates embeddings for all indexed fields for semantic matching
|
|
*/
|
|
private async refreshFieldEmbeddings(): Promise<void> {
|
|
const now = Date.now()
|
|
if (now - this.lastFieldRefresh < this.FIELD_REFRESH_INTERVAL) {
|
|
return // Skip if recently refreshed
|
|
}
|
|
|
|
try {
|
|
// Get actual indexed fields from metadata
|
|
this.fieldNames = await this.brain.getAvailableFields()
|
|
|
|
// Create embeddings for each field name for semantic matching
|
|
for (const field of this.fieldNames) {
|
|
if (!this.fieldEmbeddings.has(field)) {
|
|
// Embed the field name itself
|
|
const fieldEmbedding = await this.getEmbedding(field)
|
|
this.fieldEmbeddings.set(field, fieldEmbedding)
|
|
|
|
// Also embed common variations
|
|
const variations = this.getFieldVariations(field)
|
|
for (const variant of variations) {
|
|
const variantEmbedding = await this.getEmbedding(variant)
|
|
this.fieldEmbeddings.set(variant, variantEmbedding)
|
|
}
|
|
}
|
|
}
|
|
|
|
this.lastFieldRefresh = now
|
|
} catch (error) {
|
|
console.warn('Failed to refresh field embeddings:', error)
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Generate linguistic variations of field names - NO HARDCODED TERMS
|
|
* Uses algorithmic patterns to create natural variations
|
|
*/
|
|
private getFieldVariations(field: string): string[] {
|
|
const variations: string[] = []
|
|
|
|
// camelCase to space separated: publishDate -> publish date
|
|
const spaceSeparated = field.replace(/([A-Z])/g, ' $1').toLowerCase().trim()
|
|
if (spaceSeparated !== field.toLowerCase()) {
|
|
variations.push(spaceSeparated)
|
|
}
|
|
|
|
// snake_case to space separated: created_at -> created at
|
|
const underscoreRemoved = field.replace(/_/g, ' ').toLowerCase()
|
|
if (underscoreRemoved !== field.toLowerCase()) {
|
|
variations.push(underscoreRemoved)
|
|
}
|
|
|
|
// kebab-case to space separated: publish-date -> publish date
|
|
const dashRemoved = field.replace(/-/g, ' ').toLowerCase()
|
|
if (dashRemoved !== field.toLowerCase()) {
|
|
variations.push(dashRemoved)
|
|
}
|
|
|
|
// Generate suffix variations (remove common suffixes)
|
|
const suffixes = ['At', 'Date', 'Time', 'Id', 'Ref', 'Name', 'Value', 'Count', 'Number']
|
|
for (const suffix of suffixes) {
|
|
if (field.endsWith(suffix) && field.length > suffix.length) {
|
|
const withoutSuffix = field.slice(0, -suffix.length).toLowerCase()
|
|
variations.push(withoutSuffix)
|
|
}
|
|
}
|
|
|
|
// Generate prefix variations (remove common prefixes)
|
|
const prefixes = ['is', 'has', 'can', 'get', 'set']
|
|
for (const prefix of prefixes) {
|
|
if (field.toLowerCase().startsWith(prefix) && field.length > prefix.length) {
|
|
const withoutPrefix = field.slice(prefix.length).toLowerCase()
|
|
variations.push(withoutPrefix)
|
|
}
|
|
}
|
|
|
|
return [...new Set(variations)] // Remove duplicates
|
|
}
|
|
|
|
/**
|
|
* Find best matching field using semantic similarity
|
|
* Returns field name and confidence score
|
|
*/
|
|
private async findBestMatchingField(term: string): Promise<{
|
|
field: string | null
|
|
confidence: number
|
|
}> {
|
|
// Ensure fields are loaded
|
|
await this.refreshFieldEmbeddings()
|
|
|
|
if (this.fieldNames.length === 0) {
|
|
return { field: null, confidence: 0 }
|
|
}
|
|
|
|
// Get embedding for the search term
|
|
const termEmbedding = await this.getEmbedding(term)
|
|
|
|
// Find most similar field using cosine similarity
|
|
let bestMatch: string | null = null
|
|
let bestScore = 0
|
|
|
|
for (const [fieldName, fieldEmbedding] of this.fieldEmbeddings) {
|
|
const similarity = this.cosineSimilarity(termEmbedding, fieldEmbedding)
|
|
if (similarity > bestScore && similarity > 0.7) { // 0.7 threshold for semantic match
|
|
bestScore = similarity
|
|
bestMatch = fieldName
|
|
}
|
|
}
|
|
|
|
// Map back to actual field name if it was a variation
|
|
if (bestMatch && !this.fieldNames.includes(bestMatch)) {
|
|
// Find the original field this variation belongs to
|
|
for (const field of this.fieldNames) {
|
|
const variations = this.getFieldVariations(field)
|
|
if (variations.includes(bestMatch)) {
|
|
bestMatch = field
|
|
break
|
|
}
|
|
}
|
|
}
|
|
|
|
return { field: bestMatch, confidence: bestScore }
|
|
}
|
|
|
|
/**
|
|
* Find best matching field with type context prioritization
|
|
* Fields with high type affinity get boosted scores
|
|
*/
|
|
private async findBestMatchingFieldWithTypeContext(
|
|
term: string,
|
|
typeSpecificFields: Array<{field: string; affinity: number}>
|
|
): Promise<{ field: string | null; confidence: number }> {
|
|
// First do normal field matching
|
|
const normalMatch = await this.findBestMatchingField(term)
|
|
|
|
if (!normalMatch.field || typeSpecificFields.length === 0) {
|
|
return normalMatch
|
|
}
|
|
|
|
// Check if the matched field has type affinity
|
|
const typeField = typeSpecificFields.find(tf => tf.field === normalMatch.field)
|
|
|
|
if (typeField) {
|
|
// Boost confidence based on type affinity
|
|
// High affinity (0.8+) gets 20% boost, medium affinity (0.5+) gets 10% boost
|
|
let boost = 0
|
|
if (typeField.affinity >= 0.8) {
|
|
boost = 0.2
|
|
} else if (typeField.affinity >= 0.5) {
|
|
boost = 0.1
|
|
}
|
|
|
|
const boostedConfidence = Math.min(1.0, normalMatch.confidence + boost)
|
|
return { field: normalMatch.field, confidence: boostedConfidence }
|
|
}
|
|
|
|
return normalMatch
|
|
}
|
|
|
|
/**
|
|
* Validate field-type compatibility
|
|
* Returns validation result with suggestions for invalid combinations
|
|
*/
|
|
private async validateFieldForType(
|
|
field: string,
|
|
nounType: string
|
|
): Promise<{
|
|
isValid: boolean
|
|
affinity: number
|
|
suggestions?: string[]
|
|
reason?: string
|
|
}> {
|
|
// Get fields that actually appear with this type
|
|
const typeFields = await this.brain.getFieldsForType(nounType as NounType)
|
|
|
|
// Check if this field appears with this type
|
|
const fieldInfo = typeFields.find(tf => tf.field === field)
|
|
|
|
if (fieldInfo) {
|
|
return {
|
|
isValid: true,
|
|
affinity: fieldInfo.affinity
|
|
}
|
|
}
|
|
|
|
// Field doesn't appear with this type - provide suggestions
|
|
const suggestions = typeFields
|
|
.filter(tf => tf.affinity > 0.1) // Only suggest common fields
|
|
.slice(0, 3) // Top 3 suggestions
|
|
.map(tf => tf.field)
|
|
|
|
return {
|
|
isValid: false,
|
|
affinity: 0,
|
|
suggestions,
|
|
reason: `Field '${field}' rarely appears with ${nounType} entities. Common fields: ${suggestions.join(', ')}`
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Enhanced intelligent parse with validation
|
|
*/
|
|
private async validateAndOptimizeQuery(
|
|
tripleQuery: TripleQuery,
|
|
detectedNounType: string | null,
|
|
fieldMatches: any[]
|
|
): Promise<TripleQuery> {
|
|
if (!detectedNounType || !tripleQuery.where) {
|
|
return tripleQuery
|
|
}
|
|
|
|
const validationErrors: string[] = []
|
|
const optimizedWhere: Record<string, any> = {}
|
|
|
|
// Validate each field in the where clause
|
|
for (const [field, value] of Object.entries(tripleQuery.where)) {
|
|
if (field === 'noun') {
|
|
optimizedWhere[field] = value // Always valid
|
|
continue
|
|
}
|
|
|
|
const validation = await this.validateFieldForType(field, detectedNounType)
|
|
|
|
if (validation.isValid || validation.affinity > 0.05) {
|
|
// Valid or has some affinity - include in query
|
|
optimizedWhere[field] = value
|
|
} else {
|
|
// Invalid field for this type
|
|
validationErrors.push(validation.reason || `Invalid field: ${field}`)
|
|
|
|
// Try to find a better field match from suggestions
|
|
if (validation.suggestions && validation.suggestions.length > 0) {
|
|
const bestSuggestion = validation.suggestions[0]
|
|
optimizedWhere[bestSuggestion] = value
|
|
}
|
|
}
|
|
}
|
|
|
|
// Log validation errors for debugging (in production, could throw or return errors)
|
|
if (validationErrors.length > 0) {
|
|
console.warn('Field validation warnings:', validationErrors)
|
|
}
|
|
|
|
return {
|
|
...tripleQuery,
|
|
where: optimizedWhere
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Calculate cosine similarity between two vectors
|
|
*/
|
|
private cosineSimilarity(a: Vector, b: Vector): number {
|
|
if (a.length !== b.length) return 0
|
|
|
|
let dotProduct = 0
|
|
let normA = 0
|
|
let normB = 0
|
|
|
|
for (let i = 0; i < a.length; i++) {
|
|
dotProduct += a[i] * b[i]
|
|
normA += a[i] * a[i]
|
|
normB += b[i] * b[i]
|
|
}
|
|
|
|
if (normA === 0 || normB === 0) return 0
|
|
return dotProduct / (Math.sqrt(normA) * Math.sqrt(normB))
|
|
}
|
|
|
|
/**
|
|
* Public initialization method for external callers
|
|
*/
|
|
async init(): Promise<void> {
|
|
await this.ensureInitialized()
|
|
}
|
|
|
|
/**
|
|
* 🎯 MAIN METHOD: Convert natural language to Triple Intelligence query
|
|
*/
|
|
async processNaturalQuery(naturalQuery: string): Promise<TripleQuery> {
|
|
await this.ensureInitialized()
|
|
|
|
// Step 1: Get embedding via add/get/delete pattern
|
|
const queryEmbedding = await this.getEmbedding(naturalQuery)
|
|
|
|
// Step 2: Find best matching patterns from our library
|
|
const matches = await this.patternLibrary.findBestPatterns(queryEmbedding, 3)
|
|
|
|
// Step 3: Try each pattern until we get a good match
|
|
for (const { pattern, similarity } of matches) {
|
|
if (similarity < 0.5) break // Too low similarity, skip
|
|
|
|
// Extract slots from the query based on pattern
|
|
const extraction = this.patternLibrary.extractSlots(naturalQuery, pattern)
|
|
|
|
if (extraction.confidence > 0.6) {
|
|
// Fill the template with extracted slots
|
|
const query = this.patternLibrary.fillTemplate(pattern.template, extraction.slots)
|
|
|
|
// Track this query for learning
|
|
this.queryHistory.push({
|
|
query: naturalQuery,
|
|
result: query,
|
|
success: true // Will be updated based on user behavior
|
|
})
|
|
|
|
// Update pattern success metric
|
|
this.patternLibrary.updateSuccessMetric(pattern.id, true)
|
|
|
|
return query
|
|
}
|
|
}
|
|
|
|
// Step 4: Use intelligent field-aware parsing instead of fallback
|
|
return this.intelligentParse(naturalQuery, queryEmbedding)
|
|
}
|
|
|
|
/**
|
|
* Intelligent parse using type-aware field discovery and semantic matching
|
|
* NO FALLBACKS - uses actual indexed fields, entities, and type context
|
|
*/
|
|
private async intelligentParse(query: string, queryEmbedding: Vector): Promise<TripleQuery> {
|
|
// Step 1: Analyze intent and extract structure
|
|
const intent = await this.analyzeIntent(query)
|
|
|
|
// Step 2: Extract query terms
|
|
const queryTerms = query.split(/\s+/).filter(term => term.length > 2)
|
|
|
|
// Step 3: Detect NounType first for context
|
|
let detectedNounType: string | null = null
|
|
let typeConfidence = 0
|
|
for (const term of queryTerms) {
|
|
const nounTypeMatch = await this.findBestNounType(term)
|
|
if (nounTypeMatch.type && nounTypeMatch.confidence > typeConfidence) {
|
|
detectedNounType = nounTypeMatch.type
|
|
typeConfidence = nounTypeMatch.confidence
|
|
}
|
|
}
|
|
|
|
// Step 4: Get type-specific fields if we detected a type
|
|
let typeSpecificFields: Array<{field: string; affinity: number}> = []
|
|
if (detectedNounType && typeConfidence > 0.75) {
|
|
const fieldsForType = await this.brain.getFieldsForType(detectedNounType as NounType)
|
|
typeSpecificFields = fieldsForType.map(f => ({field: f.field, affinity: f.affinity}))
|
|
}
|
|
|
|
// Step 5: Find matching fields and entities with type context
|
|
const entityMatches = await this.findEntityMatchesWithTypeContext(queryTerms, typeSpecificFields)
|
|
|
|
// Step 6: Build structured query from matches
|
|
const tripleQuery: TripleQuery = {}
|
|
|
|
// Separate fields from entities
|
|
const fieldMatches = entityMatches.filter(m => m.type === 'field')
|
|
const entityRefs = entityMatches.filter(m => m.type === 'entity')
|
|
|
|
// Add detected type constraint if confident
|
|
if (detectedNounType && typeConfidence > 0.75) {
|
|
tripleQuery.where = { ...tripleQuery.where, noun: detectedNounType }
|
|
}
|
|
|
|
// Build metadata filters from field matches
|
|
if (fieldMatches.length > 0) {
|
|
// Use field cardinality to optimize query order
|
|
fieldMatches.sort((a, b) => (a.cardinality || 0) - (b.cardinality || 0))
|
|
|
|
tripleQuery.where = {}
|
|
for (const match of fieldMatches) {
|
|
// Extract value for this field from query
|
|
// Escape special regex characters in the term
|
|
const escapedTerm = match.term.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')
|
|
const valuePattern = new RegExp(`${escapedTerm}\\s*(?:is|=|:)?\\s*(\\S+)`, 'i')
|
|
const valueMatch = query.match(valuePattern)
|
|
|
|
if (valueMatch) {
|
|
tripleQuery.where[match.field] = valueMatch[1]
|
|
}
|
|
}
|
|
}
|
|
|
|
// Build graph connections from entity references
|
|
if (entityRefs.length > 0) {
|
|
tripleQuery.connected = {
|
|
to: entityRefs[0].id
|
|
}
|
|
}
|
|
|
|
// Use remaining terms for vector search
|
|
const usedTerms = new Set([...fieldMatches.map(m => m.term), ...entityRefs.map(m => m.term)])
|
|
const searchTerms = queryTerms.filter(term => !usedTerms.has(term))
|
|
|
|
if (searchTerms.length > 0) {
|
|
tripleQuery.like = searchTerms.join(' ')
|
|
} else if (!tripleQuery.where || Object.keys(tripleQuery.where).length === 0) {
|
|
// If no specific filters, use the full query for vector search
|
|
tripleQuery.like = query
|
|
}
|
|
|
|
// Validate and optimize query based on type context
|
|
const validatedQuery = await this.validateAndOptimizeQuery(
|
|
tripleQuery,
|
|
detectedNounType,
|
|
fieldMatches
|
|
)
|
|
|
|
// Add query optimization hints based on field statistics
|
|
if (fieldMatches.length > 0 && validatedQuery.where) {
|
|
const queryPlan = await this.brain.getOptimalQueryPlan(validatedQuery.where)
|
|
|
|
// Attach optimization hints as a separate property
|
|
const hints: any = validatedQuery
|
|
hints.optimizationHints = {
|
|
detectedType: detectedNounType,
|
|
typeConfidence,
|
|
fieldOrder: queryPlan.fieldOrder,
|
|
strategy: queryPlan.strategy,
|
|
estimatedCost: queryPlan.estimatedCost,
|
|
typeSpecificFieldCount: typeSpecificFields.length
|
|
}
|
|
}
|
|
|
|
return validatedQuery
|
|
}
|
|
|
|
/**
|
|
* Analyze intent using keywords and structure with enhanced classification
|
|
*/
|
|
private async analyzeIntent(query: string): Promise<NaturalQueryIntent> {
|
|
// Analyze query structure patterns
|
|
const lowerQuery = query.toLowerCase()
|
|
|
|
// Determine primary intent
|
|
let primaryIntent: NaturalQueryIntent['primaryIntent'] = 'search'
|
|
let confidence = 0.7 // Base confidence
|
|
let type: NaturalQueryIntent['type'] = 'vector' // Default
|
|
|
|
// Intent detection patterns
|
|
if (lowerQuery.match(/\b(filter|where|with|having)\b/)) {
|
|
primaryIntent = 'filter'
|
|
confidence += 0.15
|
|
} else if (lowerQuery.match(/\b(count|sum|average|total|group by)\b/)) {
|
|
primaryIntent = 'aggregate'
|
|
confidence += 0.2
|
|
} else if (lowerQuery.match(/\b(compare|versus|vs|difference|between)\b/)) {
|
|
primaryIntent = 'compare'
|
|
confidence += 0.15
|
|
} else if (lowerQuery.match(/\b(explain|why|how|what causes)\b/)) {
|
|
primaryIntent = 'explain'
|
|
confidence += 0.1
|
|
} else if (lowerQuery.match(/\b(connected|related|linked|from.*to)\b/)) {
|
|
primaryIntent = 'navigate'
|
|
type = 'graph'
|
|
confidence += 0.15
|
|
}
|
|
|
|
// Detect field queries
|
|
if (this.hasFieldPatterns(lowerQuery)) {
|
|
type = type === 'graph' ? 'combined' : 'field'
|
|
confidence += 0.1
|
|
}
|
|
|
|
// Detect connection queries
|
|
if (this.hasConnectionPatterns(lowerQuery)) {
|
|
type = type === 'field' ? 'combined' : 'graph'
|
|
confidence += 0.1
|
|
}
|
|
|
|
// Extract context
|
|
const context: NaturalQueryIntent['context'] = {
|
|
domain: this.detectDomain(query),
|
|
temporalScope: this.detectTemporalScope(query),
|
|
complexity: this.assessComplexity(query)
|
|
}
|
|
|
|
// Extract basic terms with enhanced modifiers
|
|
const extractedTerms = this.extractTerms(query)
|
|
|
|
return {
|
|
type,
|
|
primaryIntent,
|
|
confidence,
|
|
extractedTerms,
|
|
context
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Detect the domain of the query
|
|
*/
|
|
private detectDomain(query: string): string {
|
|
const lowerQuery = query.toLowerCase()
|
|
|
|
if (lowerQuery.match(/\b(code|function|api|bug|error|debug)\b/)) {
|
|
return 'technical'
|
|
} else if (lowerQuery.match(/\b(revenue|sales|profit|customer|market)\b/)) {
|
|
return 'business'
|
|
} else if (lowerQuery.match(/\b(research|study|paper|theory|hypothesis)\b/)) {
|
|
return 'academic'
|
|
}
|
|
|
|
return 'general'
|
|
}
|
|
|
|
/**
|
|
* Detect temporal scope in query
|
|
*/
|
|
private detectTemporalScope(query: string): 'past' | 'present' | 'future' | 'all' {
|
|
const lowerQuery = query.toLowerCase()
|
|
|
|
if (lowerQuery.match(/\b(was|were|did|had|yesterday|last|previous|ago)\b/)) {
|
|
return 'past'
|
|
} else if (lowerQuery.match(/\b(will|going to|tomorrow|next|future|upcoming)\b/)) {
|
|
return 'future'
|
|
} else if (lowerQuery.match(/\b(is|are|currently|now|today|present)\b/)) {
|
|
return 'present'
|
|
}
|
|
|
|
return 'all'
|
|
}
|
|
|
|
/**
|
|
* Assess query complexity
|
|
*/
|
|
private assessComplexity(query: string): 'simple' | 'moderate' | 'complex' {
|
|
const words = query.split(/\s+/).length
|
|
const hasMultipleClauses = query.match(/\b(and|or|but|with|where)\b/g)?.length || 0
|
|
const hasNesting = query.includes('(') || query.includes('[')
|
|
|
|
if (words < 5 && hasMultipleClauses === 0) {
|
|
return 'simple'
|
|
} else if (words > 15 || hasMultipleClauses > 2 || hasNesting) {
|
|
return 'complex'
|
|
}
|
|
|
|
return 'moderate'
|
|
}
|
|
|
|
/**
|
|
* Step 2: Use neural analysis to decompose complex queries
|
|
*/
|
|
private async decomposeQuery(query: string, intent: NaturalQueryIntent): Promise<any> {
|
|
// Use Brainy's neural clustering to find similar patterns
|
|
const queryTerms = query.split(/\\s+/).filter(term => term.length > 2)
|
|
|
|
// Try to find existing entities that match query terms
|
|
const entityMatches = await this.findEntityMatches(queryTerms)
|
|
|
|
return {
|
|
originalQuery: query,
|
|
intent,
|
|
entityMatches,
|
|
queryTerms
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Step 3: Map concepts using Entity Registry and taxonomy
|
|
*/
|
|
private async mapConcepts(decomposition: any): Promise<any> {
|
|
const mappedFields: Record<string, any> = {}
|
|
const searchTerms: string[] = []
|
|
const connections: any = {}
|
|
|
|
// Use Entity Registry to map known entities
|
|
for (const term of decomposition.queryTerms) {
|
|
const entityMatch = decomposition.entityMatches.find((m: any) =>
|
|
m.term.toLowerCase() === term.toLowerCase()
|
|
)
|
|
|
|
if (entityMatch) {
|
|
if (entityMatch.type === 'field') {
|
|
mappedFields[entityMatch.field] = entityMatch.value
|
|
} else if (entityMatch.type === 'entity') {
|
|
connections[entityMatch.id] = entityMatch
|
|
}
|
|
} else {
|
|
searchTerms.push(term)
|
|
}
|
|
}
|
|
|
|
return {
|
|
searchTerms,
|
|
mappedFields,
|
|
connections
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Step 4: Construct final Triple Intelligence query
|
|
*/
|
|
private constructTripleQuery(
|
|
originalQuery: string,
|
|
intent: NaturalQueryIntent,
|
|
mapped: any
|
|
): TripleQuery {
|
|
const query: TripleQuery = {}
|
|
|
|
// Set vector search if we have search terms
|
|
if (mapped.searchTerms.length > 0) {
|
|
query.like = mapped.searchTerms.join(' ')
|
|
} else if (intent.type === 'vector') {
|
|
query.like = originalQuery
|
|
}
|
|
|
|
// Set field filters if we found field mappings
|
|
if (Object.keys(mapped.mappedFields).length > 0) {
|
|
query.where = mapped.mappedFields
|
|
}
|
|
|
|
// Set connection searches if we found entity connections
|
|
if (Object.keys(mapped.connections).length > 0) {
|
|
const entities = Object.keys(mapped.connections)
|
|
if (entities.length > 0) {
|
|
query.connected = { to: entities }
|
|
}
|
|
}
|
|
|
|
// Apply extracted modifiers
|
|
if (intent.extractedTerms.modifiers) {
|
|
const mods = intent.extractedTerms.modifiers
|
|
if (mods.limit) query.limit = mods.limit
|
|
// Convert string boost to proper boost object
|
|
if (mods.boost) {
|
|
if (mods.boost === 'recent') {
|
|
query.boost = { field: 2.0, vector: 1.0, graph: 1.0 }
|
|
} else if (mods.boost === 'popular') {
|
|
query.boost = { graph: 2.0, vector: 1.0, field: 1.0 }
|
|
}
|
|
}
|
|
}
|
|
|
|
return query
|
|
}
|
|
|
|
/**
|
|
* Initialize pattern recognition for common query types
|
|
*/
|
|
private initializePatterns(): Map<RegExp, (match: RegExpMatchArray) => Partial<TripleQuery>> {
|
|
const patterns = new Map<RegExp, (match: RegExpMatchArray) => Partial<TripleQuery>>()
|
|
|
|
// "Find papers about AI from 2023"
|
|
patterns.set(
|
|
/find\\s+(.+?)\\s+about\\s+(.+?)\\s+from\\s+(\\d{4})/i,
|
|
(match) => ({
|
|
like: match[2],
|
|
where: { year: parseInt(match[3]) }
|
|
})
|
|
)
|
|
|
|
// "Show me recent posts by John"
|
|
patterns.set(
|
|
/show\\s+me\\s+recent\\s+(.+?)\\s+by\\s+(.+)/i,
|
|
(match) => ({
|
|
like: match[1],
|
|
boost: { field: 2.0, vector: 1.0, graph: 1.0 },
|
|
connected: { from: match[2] }
|
|
})
|
|
)
|
|
|
|
// "Papers with more than 100 citations"
|
|
patterns.set(
|
|
/(.+?)\\s+with\\s+more\\s+than\\s+(\\d+)\\s+(.+)/i,
|
|
(match) => ({
|
|
like: match[1],
|
|
where: { [match[3]]: { greaterThan: parseInt(match[2]) } }
|
|
})
|
|
)
|
|
|
|
// "Documents related to Stanford"
|
|
patterns.set(
|
|
/(.+?)\\s+related\\s+to\\s+(.+)/i,
|
|
(match) => ({
|
|
like: match[1],
|
|
connected: { to: match[2] }
|
|
})
|
|
)
|
|
|
|
return patterns
|
|
}
|
|
|
|
/**
|
|
* Detect field query patterns
|
|
*/
|
|
private hasFieldPatterns(query: string): boolean {
|
|
const fieldIndicators = [
|
|
'from', 'after', 'before', 'with more than', 'with less than',
|
|
'published', 'created', 'year', 'date', 'citations', 'score'
|
|
]
|
|
|
|
return fieldIndicators.some(indicator => query.includes(indicator))
|
|
}
|
|
|
|
/**
|
|
* Detect connection query patterns
|
|
*/
|
|
private hasConnectionPatterns(query: string): boolean {
|
|
const connectionIndicators = [
|
|
'by', 'from', 'connected to', 'related to', 'authored by',
|
|
'created by', 'associated with', 'linked to'
|
|
]
|
|
|
|
return connectionIndicators.some(indicator => query.includes(indicator))
|
|
}
|
|
|
|
/**
|
|
* Extract terms and modifiers from query
|
|
*/
|
|
private extractTerms(query: string): NaturalQueryIntent['extractedTerms'] {
|
|
const extracted: NaturalQueryIntent['extractedTerms'] = {}
|
|
|
|
// Extract limit numbers
|
|
const limitMatch = query.match(/(?:top|first|limit)\\s+(\\d+)/i)
|
|
if (limitMatch) {
|
|
extracted.modifiers = { limit: parseInt(limitMatch[1]) }
|
|
}
|
|
|
|
// Extract boost indicators
|
|
if (query.toLowerCase().includes('recent')) {
|
|
extracted.modifiers = { ...extracted.modifiers, boost: 'recent' }
|
|
}
|
|
if (query.toLowerCase().includes('popular')) {
|
|
extracted.modifiers = { ...extracted.modifiers, boost: 'popular' }
|
|
}
|
|
|
|
return extracted
|
|
}
|
|
|
|
/**
|
|
* Find entity matches using type context and semantic similarity
|
|
*/
|
|
private async findEntityMatches(terms: string[]): Promise<any[]> {
|
|
return this.findEntityMatchesWithTypeContext(terms, [])
|
|
}
|
|
|
|
/**
|
|
* Find entity matches with type context for better field prioritization
|
|
*/
|
|
private async findEntityMatchesWithTypeContext(
|
|
terms: string[],
|
|
typeSpecificFields: Array<{field: string; affinity: number}>
|
|
): Promise<any[]> {
|
|
const matches: any[] = []
|
|
|
|
// Get field statistics for optimization hints
|
|
const fieldStats = await this.brain.getFieldStatistics()
|
|
|
|
// Create field priority map based on type affinity
|
|
const fieldPriorityMap = new Map<string, number>()
|
|
for (const {field, affinity} of typeSpecificFields) {
|
|
fieldPriorityMap.set(field, affinity)
|
|
}
|
|
|
|
for (const term of terms) {
|
|
try {
|
|
// First, check if term matches a field using semantic similarity
|
|
const fieldMatch = await this.findBestMatchingFieldWithTypeContext(term, typeSpecificFields)
|
|
|
|
if (fieldMatch.field && fieldMatch.confidence > 0.7) {
|
|
const stats = fieldStats.get(fieldMatch.field)
|
|
const typeAffinity = fieldPriorityMap.get(fieldMatch.field) || 0
|
|
|
|
matches.push({
|
|
term,
|
|
type: 'field',
|
|
field: fieldMatch.field,
|
|
confidence: fieldMatch.confidence,
|
|
typeAffinity, // NEW: How likely this field appears with this type
|
|
cardinality: stats?.cardinality.uniqueValues,
|
|
distribution: stats?.cardinality.distribution,
|
|
indexType: stats?.indexType
|
|
})
|
|
// Skip entity search if we found a field match
|
|
continue
|
|
}
|
|
|
|
// Search for similar entities in the knowledge base
|
|
const results = await this.brain.find({
|
|
query: term,
|
|
limit: 5
|
|
})
|
|
|
|
for (const result of results) {
|
|
if (result.score > 0.8) { // High similarity threshold
|
|
matches.push({
|
|
term,
|
|
id: result.id,
|
|
type: 'entity',
|
|
confidence: result.score,
|
|
metadata: result.entity?.metadata
|
|
})
|
|
}
|
|
}
|
|
} catch (error) {
|
|
// If search fails, continue with other terms
|
|
console.debug(`Failed to search for term: ${term}`, error)
|
|
}
|
|
}
|
|
|
|
return matches
|
|
}
|
|
|
|
// REMOVED: isKnownField and mapToFieldName - now using semantic field matching
|
|
// The findBestMatchingField method with embeddings replaces these hardcoded approaches
|
|
|
|
/**
|
|
* Find similar successful queries from history
|
|
* Uses Brainy's vector search to find semantically similar previous queries
|
|
*/
|
|
private async findSimilarQueries(queryEmbedding: Vector): Promise<any[]> {
|
|
try {
|
|
// Search for similar queries in a hypothetical query history
|
|
// For now, return empty array since we don't have query history storage yet
|
|
// This would integrate with Brainy's search to find similar query patterns
|
|
|
|
// Future implementation could search a query_history noun type:
|
|
// const similarQueries = await this.brainy.search(queryEmbedding, {
|
|
// limit: 5,
|
|
// metadata: { type: 'successful_query' },
|
|
// nounTypes: ['query_history']
|
|
// })
|
|
|
|
return []
|
|
} catch (error) {
|
|
console.debug('Failed to find similar queries:', error)
|
|
return []
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Extract entities from query using Brainy's semantic search
|
|
* Identifies known entities, concepts, and relationships in the query text
|
|
*/
|
|
private async extractEntities(query: string): Promise<any[]> {
|
|
try {
|
|
// Split query into potential entity terms
|
|
const terms = query.toLowerCase()
|
|
.split(/[\s,\.;!?]+/)
|
|
.filter(term => term.length > 2)
|
|
|
|
const entities: any[] = []
|
|
|
|
// Search for each term in Brainy to see if it matches known entities
|
|
for (const term of terms) {
|
|
try {
|
|
const results = await this.brain.find(term)
|
|
|
|
if (results && results.length > 0) {
|
|
// Found matching entities
|
|
entities.push({
|
|
term,
|
|
matches: results,
|
|
confidence: results[0].score || 0.7
|
|
})
|
|
}
|
|
} catch (searchError) {
|
|
// Continue if individual term search fails
|
|
console.debug(`Entity search failed for term: ${term}`, searchError)
|
|
}
|
|
}
|
|
|
|
return entities
|
|
} catch (error) {
|
|
console.debug('Failed to extract entities:', error)
|
|
return []
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Build final TripleQuery based on intent, entities, and query analysis
|
|
* Constructs optimized query combining vector, graph, and field searches
|
|
*/
|
|
private async buildQuery(query: string, intent: any, entities: any[]): Promise<TripleQuery> {
|
|
try {
|
|
const tripleQuery: TripleQuery = {
|
|
like: query, // Default to semantic search
|
|
limit: 10
|
|
}
|
|
|
|
// Add field filters based on intent
|
|
if (intent.hasFieldPatterns) {
|
|
// Extract field-based constraints from the query
|
|
const whereClause: Record<string, any> = {}
|
|
|
|
// Look for date/year patterns
|
|
const yearMatch = query.match(/(\d{4})/g)
|
|
if (yearMatch) {
|
|
whereClause.year = parseInt(yearMatch[0])
|
|
}
|
|
|
|
// Look for numeric constraints
|
|
const moreThanMatch = query.match(/more than (\d+)/i)
|
|
if (moreThanMatch) {
|
|
whereClause.count = { greaterThan: parseInt(moreThanMatch[1]) }
|
|
}
|
|
|
|
if (Object.keys(whereClause).length > 0) {
|
|
tripleQuery.where = whereClause
|
|
}
|
|
}
|
|
|
|
// Add connection-based searches
|
|
if (intent.hasConnectionPatterns) {
|
|
// Look for relationship patterns in the query
|
|
const connectedMatch = query.match(/connected to (.+?)$/i) ||
|
|
query.match(/related to (.+?)$/i)
|
|
|
|
if (connectedMatch) {
|
|
tripleQuery.connected = {
|
|
to: connectedMatch[1].trim()
|
|
}
|
|
}
|
|
}
|
|
|
|
// Add entity-specific filters
|
|
if (entities && entities.length > 0) {
|
|
const highConfidenceEntities = entities.filter(e => e.confidence > 0.8)
|
|
|
|
if (highConfidenceEntities.length > 0) {
|
|
// Use the highest confidence entity to refine search
|
|
const topEntity = highConfidenceEntities[0]
|
|
if (topEntity.matches && topEntity.matches.length > 0) {
|
|
// Add entity-specific metadata or connection
|
|
const entityData = topEntity.matches[0].metadata
|
|
if (entityData && entityData.category) {
|
|
tripleQuery.where = {
|
|
...tripleQuery.where,
|
|
category: entityData.category
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
return tripleQuery
|
|
} catch (error) {
|
|
console.debug('Failed to build query:', error)
|
|
// Return simple query as fallback
|
|
return {
|
|
like: query,
|
|
limit: 10
|
|
}
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Extract entities from text using NEURAL matching to strict NounTypes
|
|
* ALWAYS uses neural matching, NEVER falls back to patterns
|
|
*/
|
|
async extract(text: string, options?: {
|
|
types?: string[]
|
|
includeMetadata?: boolean
|
|
confidence?: number
|
|
}): Promise<Array<{
|
|
text: string
|
|
type: string
|
|
position: { start: number; end: number }
|
|
confidence: number
|
|
metadata?: any
|
|
}>> {
|
|
await this.ensureInitialized()
|
|
|
|
// ALWAYS use NeuralEntityExtractor for proper type matching
|
|
const { NeuralEntityExtractor } = await import('./entityExtractor.js')
|
|
const extractor = new NeuralEntityExtractor(this.brain)
|
|
|
|
// Convert string types to NounTypes if provided
|
|
const nounTypes = options?.types ?
|
|
options.types.map(t => t as any) :
|
|
undefined
|
|
|
|
// Extract using neural matching
|
|
const entities = await extractor.extract(text, {
|
|
types: nounTypes,
|
|
confidence: options?.confidence || 0.0, // Accept ALL matches
|
|
includeVectors: false,
|
|
neuralMatching: true // ALWAYS use neural matching
|
|
})
|
|
|
|
// Convert to expected format
|
|
return entities.map(entity => ({
|
|
text: entity.text,
|
|
type: entity.type,
|
|
position: entity.position,
|
|
confidence: entity.confidence,
|
|
metadata: options?.includeMetadata ? {
|
|
...entity.metadata,
|
|
neuralMatch: true,
|
|
extractedAt: Date.now()
|
|
} : undefined
|
|
}))
|
|
}
|
|
|
|
/**
|
|
* DEPRECATED - Old pattern-based extraction
|
|
* This should NEVER be used - kept only for reference
|
|
*/
|
|
private async extractWithPatterns_DEPRECATED(text: string, options?: {
|
|
types?: string[]
|
|
includeMetadata?: boolean
|
|
confidence?: number
|
|
}): Promise<Array<{
|
|
text: string
|
|
type: string
|
|
position: { start: number; end: number }
|
|
confidence: number
|
|
metadata?: any
|
|
}>> {
|
|
const extracted: Array<{
|
|
text: string
|
|
type: string
|
|
position: { start: number; end: number }
|
|
confidence: number
|
|
metadata?: any
|
|
}> = []
|
|
|
|
// Common entity patterns
|
|
const patterns = {
|
|
// People (names with capitals)
|
|
person: /\b([A-Z][a-z]+ [A-Z][a-z]+)\b/g,
|
|
// Organizations (capitals, Inc, LLC, etc)
|
|
organization: /\b([A-Z][a-zA-Z&]+(?: [A-Z][a-zA-Z&]+)*(?:,? (?:Inc|LLC|Corp|Ltd|Co|Group|Foundation|Institute|University|College|School|Hospital|Bank|Agency)\.?))\b/g,
|
|
// Locations (capitals, common place words)
|
|
location: /\b([A-Z][a-z]+(?: [A-Z][a-z]+)*(?:,? (?:[A-Z][a-z]+))?)(?= (?:City|County|State|Country|Street|Road|Avenue|Boulevard|Drive|Park|Square|Place|Island|Mountain|River|Lake|Ocean|Sea))\b/g,
|
|
// Dates
|
|
date: /\b(\d{1,2}[\/\-]\d{1,2}[\/\-]\d{2,4}|\d{4}[\/\-]\d{1,2}[\/\-]\d{1,2}|(?:Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)[a-z]* \d{1,2},? \d{4}|\d{1,2} (?:Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)[a-z]* \d{4})\b/gi,
|
|
// Times
|
|
time: /\b(\d{1,2}:\d{2}(?::\d{2})?(?:\s?[AP]M)?)\b/gi,
|
|
// Emails
|
|
email: /\b([a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,})\b/g,
|
|
// URLs
|
|
url: /\b(https?:\/\/[^\s]+)\b/g,
|
|
// Phone numbers
|
|
phone: /\b(\+?\d{1,3}?[- .]?\(?\d{1,4}\)?[- .]?\d{1,4}[- .]?\d{1,4})\b/g,
|
|
// Money
|
|
money: /\b(\$[\d,]+(?:\.\d{2})?|[\d,]+(?:\.\d{2})?\s*(?:USD|EUR|GBP|JPY|CNY))\b/gi,
|
|
// Percentages
|
|
percentage: /\b(\d+(?:\.\d+)?%)\b/g,
|
|
// Products/versions
|
|
product: /\b([A-Z][a-zA-Z0-9]*(?: [A-Z][a-zA-Z0-9]*)*\s+v?\d+(?:\.\d+)*)\b/g,
|
|
// Hashtags
|
|
hashtag: /#[a-zA-Z0-9_]+/g,
|
|
// Mentions
|
|
mention: /@[a-zA-Z0-9_]+/g
|
|
}
|
|
|
|
const minConfidence = options?.confidence || 0.5
|
|
const targetTypes = options?.types || Object.keys(patterns)
|
|
|
|
// Apply each pattern
|
|
for (const [type, pattern] of Object.entries(patterns)) {
|
|
if (!targetTypes.includes(type)) continue
|
|
|
|
let match
|
|
while ((match = pattern.exec(text)) !== null) {
|
|
const extractedText = match[1] || match[0]
|
|
const confidence = this.calculateConfidence(extractedText, type)
|
|
|
|
if (confidence >= minConfidence) {
|
|
const entity = {
|
|
text: extractedText,
|
|
type,
|
|
position: {
|
|
start: match.index,
|
|
end: match.index + match[0].length
|
|
},
|
|
confidence
|
|
}
|
|
|
|
if (options?.includeMetadata) {
|
|
(entity as any).metadata = {
|
|
pattern: pattern.source,
|
|
contextBefore: text.substring(Math.max(0, match.index - 20), match.index),
|
|
contextAfter: text.substring(match.index + match[0].length, Math.min(text.length, match.index + match[0].length + 20))
|
|
}
|
|
}
|
|
|
|
extracted.push(entity)
|
|
}
|
|
}
|
|
}
|
|
|
|
// Sort by position
|
|
extracted.sort((a, b) => a.position.start - b.position.start)
|
|
|
|
// Remove overlapping entities (keep higher confidence)
|
|
const filtered: typeof extracted = []
|
|
for (const entity of extracted) {
|
|
const overlapping = filtered.find(e =>
|
|
(entity.position.start >= e.position.start && entity.position.start < e.position.end) ||
|
|
(entity.position.end > e.position.start && entity.position.end <= e.position.end)
|
|
)
|
|
|
|
if (!overlapping) {
|
|
filtered.push(entity)
|
|
} else if (entity.confidence > overlapping.confidence) {
|
|
const index = filtered.indexOf(overlapping)
|
|
filtered[index] = entity
|
|
}
|
|
}
|
|
|
|
return filtered
|
|
}
|
|
|
|
/**
|
|
* Analyze sentiment of text
|
|
*/
|
|
async sentiment(text: string, options?: {
|
|
granularity?: 'document' | 'sentence' | 'aspect'
|
|
aspects?: string[]
|
|
}): Promise<{
|
|
overall: {
|
|
score: number // -1 to 1
|
|
magnitude: number // 0 to 1
|
|
label: 'positive' | 'negative' | 'neutral' | 'mixed'
|
|
}
|
|
sentences?: Array<{
|
|
text: string
|
|
score: number
|
|
magnitude: number
|
|
label: string
|
|
}>
|
|
aspects?: Record<string, {
|
|
score: number
|
|
magnitude: number
|
|
mentions: number
|
|
}>
|
|
}> {
|
|
// Sentiment words with scores
|
|
const positiveWords = new Set(['good', 'great', 'excellent', 'amazing', 'wonderful', 'fantastic', 'love', 'like', 'best', 'happy', 'joy', 'brilliant', 'outstanding', 'perfect', 'beautiful', 'awesome', 'super', 'nice', 'fun', 'exciting', 'impressive', 'incredible', 'remarkable', 'delightful', 'pleased', 'satisfied', 'successful', 'effective', 'helpful'])
|
|
const negativeWords = new Set(['bad', 'terrible', 'awful', 'horrible', 'hate', 'dislike', 'worst', 'sad', 'angry', 'poor', 'disappointing', 'failed', 'broken', 'useless', 'waste', 'sucks', 'disgusting', 'ugly', 'boring', 'annoying', 'frustrating', 'difficult', 'complicated', 'confusing', 'slow', 'expensive', 'unfair', 'wrong', 'mistake', 'problem', 'issue'])
|
|
const intensifiers = new Set(['very', 'extremely', 'really', 'absolutely', 'completely', 'totally', 'quite', 'rather', 'so'])
|
|
const negations = new Set(['not', 'no', 'never', 'neither', 'none', 'nobody', 'nothing', 'nowhere', 'hardly', 'barely', 'scarcely'])
|
|
|
|
const normalizedText = text.toLowerCase()
|
|
const words = normalizedText.split(/\s+/)
|
|
|
|
// Calculate overall sentiment
|
|
let positiveCount = 0
|
|
let negativeCount = 0
|
|
let intensifierBoost = 1
|
|
|
|
for (let i = 0; i < words.length; i++) {
|
|
const word = words[i].replace(/[^a-z]/g, '')
|
|
const prevWord = i > 0 ? words[i - 1].replace(/[^a-z]/g, '') : ''
|
|
|
|
// Check for intensifiers
|
|
if (intensifiers.has(prevWord)) {
|
|
intensifierBoost = 1.5
|
|
} else {
|
|
intensifierBoost = 1
|
|
}
|
|
|
|
// Check for negation
|
|
const isNegated = negations.has(prevWord)
|
|
|
|
if (positiveWords.has(word)) {
|
|
if (isNegated) {
|
|
negativeCount += intensifierBoost
|
|
} else {
|
|
positiveCount += intensifierBoost
|
|
}
|
|
} else if (negativeWords.has(word)) {
|
|
if (isNegated) {
|
|
positiveCount += intensifierBoost
|
|
} else {
|
|
negativeCount += intensifierBoost
|
|
}
|
|
}
|
|
}
|
|
|
|
const total = positiveCount + negativeCount
|
|
const score = total > 0 ? (positiveCount - negativeCount) / total : 0
|
|
const magnitude = Math.min(1, total / words.length)
|
|
|
|
let label: 'positive' | 'negative' | 'neutral' | 'mixed'
|
|
if (score > 0.2) label = 'positive'
|
|
else if (score < -0.2) label = 'negative'
|
|
else if (magnitude > 0.3) label = 'mixed'
|
|
else label = 'neutral'
|
|
|
|
const result: any = {
|
|
overall: {
|
|
score,
|
|
magnitude,
|
|
label
|
|
}
|
|
}
|
|
|
|
// Sentence-level analysis
|
|
if (options?.granularity === 'sentence' || options?.granularity === 'aspect') {
|
|
const sentences = text.match(/[^.!?]+[.!?]+/g) || [text]
|
|
result.sentences = []
|
|
|
|
for (const sentence of sentences) {
|
|
const sentenceResult = await this.sentiment(sentence)
|
|
result.sentences.push({
|
|
text: sentence.trim(),
|
|
score: sentenceResult.overall.score,
|
|
magnitude: sentenceResult.overall.magnitude,
|
|
label: sentenceResult.overall.label
|
|
})
|
|
}
|
|
}
|
|
|
|
// Aspect-based analysis
|
|
if (options?.granularity === 'aspect' && options?.aspects) {
|
|
result.aspects = {}
|
|
|
|
for (const aspect of options.aspects) {
|
|
const aspectRegex = new RegExp(`[^.!?]*\\b${aspect}\\b[^.!?]*[.!?]?`, 'gi')
|
|
const aspectSentences = text.match(aspectRegex) || []
|
|
|
|
if (aspectSentences.length > 0) {
|
|
let aspectScore = 0
|
|
let aspectMagnitude = 0
|
|
|
|
for (const sentence of aspectSentences) {
|
|
const sentimentResult = await this.sentiment(sentence)
|
|
aspectScore += sentimentResult.overall.score
|
|
aspectMagnitude += sentimentResult.overall.magnitude
|
|
}
|
|
|
|
result.aspects[aspect] = {
|
|
score: aspectScore / aspectSentences.length,
|
|
magnitude: aspectMagnitude / aspectSentences.length,
|
|
mentions: aspectSentences.length
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
return result
|
|
}
|
|
|
|
/**
|
|
* Calculate confidence for entity extraction
|
|
*/
|
|
private calculateConfidence(text: string, type: string): number {
|
|
let confidence = 0.5 // Base confidence
|
|
|
|
// Adjust based on type-specific rules
|
|
switch (type) {
|
|
case 'person':
|
|
// Names with 2-3 capitalized words are more confident
|
|
const nameWords = text.split(' ')
|
|
if (nameWords.length >= 2 && nameWords.length <= 3) {
|
|
confidence += 0.3
|
|
}
|
|
if (nameWords.every(w => /^[A-Z]/.test(w))) {
|
|
confidence += 0.2
|
|
}
|
|
break
|
|
|
|
case 'organization':
|
|
// Presence of corporate suffixes increases confidence
|
|
if (/\b(Inc|LLC|Corp|Ltd|Co|Group)\.?$/.test(text)) {
|
|
confidence += 0.4
|
|
}
|
|
break
|
|
|
|
case 'email':
|
|
case 'url':
|
|
// These patterns are very specific, high confidence
|
|
confidence = 0.95
|
|
break
|
|
|
|
case 'date':
|
|
case 'time':
|
|
case 'money':
|
|
case 'percentage':
|
|
// Numeric patterns are reliable
|
|
confidence = 0.9
|
|
break
|
|
|
|
case 'location':
|
|
// Geographic terms increase confidence
|
|
if (/\b(City|State|Country|Street|Road|Avenue)$/.test(text)) {
|
|
confidence += 0.3
|
|
}
|
|
break
|
|
}
|
|
|
|
return Math.min(1, confidence)
|
|
}
|
|
} |