David Snelling
33c6b06649
- Add incremental sorted index updates during CRUD operations for consistent <5ms range queries - Implement parallel search optimization with vector, metadata, and graph intelligence fusion - Fix metadata-only query handling to properly return results without vector search - Fix NLP recursive call issue by using embed() instead of add() - Add cardinality tracking for smart index optimization - Store entity data in metadata for proper retrieval - Add comprehensive performance documentation This improves query performance from O(n) to O(log n) for range queries and ensures consistent fast performance without lazy loading delays.
789 lines
No EOL
24 KiB
TypeScript
789 lines
No EOL
24 KiB
TypeScript
/**
|
|
* 🧠 Pattern Library for Natural Language Processing
|
|
* Manages pre-computed pattern embeddings and smart matching
|
|
*
|
|
* Uses Brainy's own features for self-leveraging intelligence:
|
|
* - Embeddings for semantic similarity
|
|
* - Pattern caching for performance
|
|
* - Progressive learning from usage
|
|
*/
|
|
|
|
import { Vector } from '../coreTypes.js'
|
|
import { Brainy } from '../brainy.js'
|
|
import { EMBEDDED_PATTERNS, getPatternEmbeddings, PATTERNS_METADATA } from './embeddedPatterns.js'
|
|
|
|
export interface Pattern {
|
|
id: string
|
|
category: string
|
|
examples: string[]
|
|
pattern: string
|
|
template: any
|
|
confidence: number
|
|
embedding?: Vector
|
|
domain?: string
|
|
frequency?: number | string
|
|
slots?: SlotDefinition[] // Named slot definitions
|
|
}
|
|
|
|
export interface SlotDefinition {
|
|
name: string
|
|
type: 'text' | 'number' | 'date' | 'entity' | 'location' | 'person' | 'any'
|
|
required?: boolean
|
|
default?: any
|
|
pattern?: string // Optional regex for validation
|
|
transform?: (value: string) => any // Optional transformation function
|
|
}
|
|
|
|
export interface SlotExtraction {
|
|
slots: Record<string, any>
|
|
confidence: number
|
|
errors?: string[] // Validation errors
|
|
}
|
|
|
|
export class PatternLibrary {
|
|
private patterns: Map<string, Pattern>
|
|
private patternEmbeddings: Map<string, Vector>
|
|
private brain: Brainy
|
|
private embeddingCache: Map<string, Vector>
|
|
private successMetrics: Map<string, number>
|
|
|
|
constructor(brain: Brainy) {
|
|
this.brain = brain
|
|
this.patterns = new Map()
|
|
this.patternEmbeddings = new Map()
|
|
this.embeddingCache = new Map()
|
|
this.successMetrics = new Map()
|
|
}
|
|
|
|
/**
|
|
* Initialize pattern library with pre-computed embeddings
|
|
*/
|
|
async init(): Promise<void> {
|
|
// Try to load pre-computed embeddings first
|
|
const precomputedEmbeddings = getPatternEmbeddings()
|
|
|
|
if (precomputedEmbeddings.size > 0) {
|
|
// Use pre-computed embeddings (instant!)
|
|
console.debug(`Loading ${precomputedEmbeddings.size} pre-computed pattern embeddings`)
|
|
|
|
for (const pattern of EMBEDDED_PATTERNS) {
|
|
this.patterns.set(pattern.id, pattern)
|
|
this.successMetrics.set(pattern.id, pattern.confidence)
|
|
|
|
const embedding = precomputedEmbeddings.get(pattern.id)
|
|
if (embedding) {
|
|
this.patternEmbeddings.set(pattern.id, Array.from(embedding))
|
|
}
|
|
}
|
|
|
|
console.debug(`Pattern library ready: ${PATTERNS_METADATA.totalPatterns} patterns loaded instantly`)
|
|
} else {
|
|
// Fall back to runtime computation
|
|
console.debug('No pre-computed embeddings found, computing at runtime...')
|
|
|
|
for (const pattern of EMBEDDED_PATTERNS) {
|
|
this.patterns.set(pattern.id, pattern)
|
|
this.successMetrics.set(pattern.id, pattern.confidence)
|
|
}
|
|
|
|
// Compute embeddings for all patterns
|
|
await this.precomputeEmbeddings()
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Pre-compute embeddings for all patterns for fast matching
|
|
*/
|
|
private async precomputeEmbeddings(): Promise<void> {
|
|
for (const [id, pattern] of this.patterns) {
|
|
// Average embeddings of all examples for robust representation
|
|
const embeddings: Vector[] = []
|
|
|
|
for (const example of pattern.examples) {
|
|
const embedding = await this.getEmbedding(example)
|
|
embeddings.push(embedding)
|
|
}
|
|
|
|
// Average the embeddings
|
|
const avgEmbedding = this.averageVectors(embeddings)
|
|
this.patternEmbeddings.set(id, avgEmbedding)
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Get embedding with caching
|
|
*/
|
|
private async getEmbedding(text: string): Promise<Vector> {
|
|
if (this.embeddingCache.has(text)) {
|
|
return this.embeddingCache.get(text)!
|
|
}
|
|
|
|
// Use brain's embed method directly to avoid recursion
|
|
const embedding = await (this.brain as any).embed(text)
|
|
|
|
this.embeddingCache.set(text, embedding)
|
|
return embedding
|
|
}
|
|
|
|
/**
|
|
* Find best matching patterns for a query
|
|
*/
|
|
async findBestPatterns(queryEmbedding: Vector, k: number = 3): Promise<Array<{
|
|
pattern: Pattern
|
|
similarity: number
|
|
}>> {
|
|
const matches: Array<{ pattern: Pattern; similarity: number }> = []
|
|
|
|
// Calculate similarity with all patterns
|
|
for (const [id, patternEmbedding] of this.patternEmbeddings) {
|
|
const similarity = this.cosineSimilarity(queryEmbedding, patternEmbedding)
|
|
const pattern = this.patterns.get(id)!
|
|
|
|
// Apply success metric boost
|
|
const successBoost = this.successMetrics.get(id) || 0.5
|
|
const adjustedSimilarity = similarity * (0.7 + 0.3 * successBoost)
|
|
|
|
matches.push({
|
|
pattern,
|
|
similarity: adjustedSimilarity
|
|
})
|
|
}
|
|
|
|
// Sort by similarity and return top k
|
|
matches.sort((a, b) => b.similarity - a.similarity)
|
|
return matches.slice(0, k)
|
|
}
|
|
|
|
/**
|
|
* Extract slots from query based on pattern with enhanced fuzzy matching
|
|
*/
|
|
extractSlots(query: string, pattern: Pattern): SlotExtraction {
|
|
const slots: Record<string, any> = {}
|
|
const errors: string[] = []
|
|
let confidence = pattern.confidence
|
|
|
|
// If pattern has named slot definitions, use them
|
|
if (pattern.slots && pattern.slots.length > 0) {
|
|
return this.extractNamedSlots(query, pattern)
|
|
}
|
|
|
|
// Try regex extraction first
|
|
const regex = new RegExp(pattern.pattern, 'i')
|
|
const match = query.match(regex)
|
|
|
|
if (match) {
|
|
// Extract captured groups as slots
|
|
for (let i = 1; i < match.length; i++) {
|
|
slots[`$${i}`] = match[i]
|
|
}
|
|
|
|
// High confidence if regex matches
|
|
confidence = Math.min(confidence * 1.2, 1.0)
|
|
} else {
|
|
// Enhanced fuzzy matching with Levenshtein distance
|
|
const fuzzyResult = this.fuzzyExtractSlots(query, pattern)
|
|
Object.assign(slots, fuzzyResult.slots)
|
|
confidence = fuzzyResult.confidence
|
|
|
|
if (fuzzyResult.errors) {
|
|
errors.push(...fuzzyResult.errors)
|
|
}
|
|
}
|
|
|
|
// Post-process slots
|
|
this.postProcessSlots(slots, pattern)
|
|
|
|
return { slots, confidence, errors: errors.length > 0 ? errors : undefined }
|
|
}
|
|
|
|
/**
|
|
* Extract named slots with type validation
|
|
*/
|
|
private extractNamedSlots(query: string, pattern: Pattern): SlotExtraction {
|
|
const slots: Record<string, any> = {}
|
|
const errors: string[] = []
|
|
let confidence = pattern.confidence
|
|
|
|
if (!pattern.slots) {
|
|
return { slots, confidence }
|
|
}
|
|
|
|
// Create a flexible regex from pattern
|
|
let flexiblePattern = pattern.pattern
|
|
const slotPositions: Map<number, SlotDefinition> = new Map()
|
|
|
|
// Replace named slots in pattern with capture groups
|
|
pattern.slots.forEach((slot, index) => {
|
|
const slotPattern = slot.pattern || this.getDefaultPatternForType(slot.type)
|
|
flexiblePattern = flexiblePattern.replace(
|
|
new RegExp(`\\{${slot.name}\\}`, 'g'),
|
|
`(${slotPattern})`
|
|
)
|
|
slotPositions.set(index + 1, slot)
|
|
})
|
|
|
|
const regex = new RegExp(flexiblePattern, 'i')
|
|
const match = query.match(regex)
|
|
|
|
if (match) {
|
|
// Extract and validate each slot
|
|
slotPositions.forEach((slotDef, position) => {
|
|
const value = match[position]
|
|
|
|
if (value) {
|
|
// Apply transformation if defined
|
|
const transformedValue = slotDef.transform
|
|
? slotDef.transform(value)
|
|
: this.transformByType(value, slotDef.type)
|
|
|
|
// Validate the value
|
|
if (this.validateSlotValue(transformedValue, slotDef)) {
|
|
slots[slotDef.name] = transformedValue
|
|
} else {
|
|
errors.push(`Invalid value for slot '${slotDef.name}': expected ${slotDef.type}, got '${value}'`)
|
|
confidence *= 0.8
|
|
}
|
|
} else if (slotDef.required) {
|
|
if (slotDef.default !== undefined) {
|
|
slots[slotDef.name] = slotDef.default
|
|
} else {
|
|
errors.push(`Required slot '${slotDef.name}' not found`)
|
|
confidence *= 0.5
|
|
}
|
|
}
|
|
})
|
|
} else {
|
|
// Try fuzzy matching for named slots
|
|
const fuzzyResult = this.fuzzyExtractNamedSlots(query, pattern)
|
|
Object.assign(slots, fuzzyResult.slots)
|
|
confidence = fuzzyResult.confidence
|
|
if (fuzzyResult.errors) {
|
|
errors.push(...fuzzyResult.errors)
|
|
}
|
|
}
|
|
|
|
return { slots, confidence, errors: errors.length > 0 ? errors : undefined }
|
|
}
|
|
|
|
/**
|
|
* Fuzzy extraction using Levenshtein distance
|
|
*/
|
|
private fuzzyExtractSlots(query: string, pattern: Pattern): SlotExtraction {
|
|
const slots: Record<string, any> = {}
|
|
let bestConfidence = 0
|
|
|
|
// Try each example with fuzzy matching
|
|
for (const example of pattern.examples) {
|
|
const distance = this.levenshteinDistance(query.toLowerCase(), example.toLowerCase())
|
|
const similarity = 1 - (distance / Math.max(query.length, example.length))
|
|
|
|
if (similarity > 0.6) { // 60% similarity threshold
|
|
// Extract slots using alignment
|
|
const aligned = this.alignStrings(query, example)
|
|
const extractedSlots = this.extractSlotsFromAlignment(aligned, pattern)
|
|
|
|
if (Object.keys(extractedSlots).length > 0) {
|
|
const currentConfidence = pattern.confidence * similarity
|
|
if (currentConfidence > bestConfidence) {
|
|
Object.assign(slots, extractedSlots)
|
|
bestConfidence = currentConfidence
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
return {
|
|
slots,
|
|
confidence: bestConfidence,
|
|
errors: bestConfidence < 0.5 ? ['Low confidence fuzzy match'] : undefined
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Fuzzy extraction for named slots
|
|
*/
|
|
private fuzzyExtractNamedSlots(query: string, pattern: Pattern): SlotExtraction {
|
|
const slots: Record<string, any> = {}
|
|
const errors: string[] = []
|
|
let confidence = pattern.confidence * 0.7 // Lower confidence for fuzzy
|
|
|
|
if (!pattern.slots) {
|
|
return { slots, confidence }
|
|
}
|
|
|
|
// Tokenize query for flexible matching
|
|
const tokens = this.tokenize(query)
|
|
|
|
pattern.slots.forEach(slotDef => {
|
|
const value = this.findSlotValueInTokens(tokens, slotDef)
|
|
|
|
if (value) {
|
|
const transformedValue = slotDef.transform
|
|
? slotDef.transform(value)
|
|
: this.transformByType(value, slotDef.type)
|
|
|
|
if (this.validateSlotValue(transformedValue, slotDef)) {
|
|
slots[slotDef.name] = transformedValue
|
|
} else {
|
|
errors.push(`Fuzzy match: uncertain value for '${slotDef.name}'`)
|
|
confidence *= 0.9
|
|
}
|
|
} else if (slotDef.required && slotDef.default !== undefined) {
|
|
slots[slotDef.name] = slotDef.default
|
|
}
|
|
})
|
|
|
|
return { slots, confidence, errors: errors.length > 0 ? errors : undefined }
|
|
}
|
|
|
|
/**
|
|
* Find slot value in tokens based on type
|
|
*/
|
|
private findSlotValueInTokens(tokens: string[], slotDef: SlotDefinition): string | null {
|
|
const joinedTokens = tokens.join(' ')
|
|
|
|
switch (slotDef.type) {
|
|
case 'number':
|
|
const numberMatch = joinedTokens.match(/\d+(\.\d+)?/)
|
|
return numberMatch ? numberMatch[0] : null
|
|
|
|
case 'date':
|
|
const datePatterns = [
|
|
/\d{4}-\d{2}-\d{2}/,
|
|
/\d{1,2}\/\d{1,2}\/\d{2,4}/,
|
|
/(january|february|march|april|may|june|july|august|september|october|november|december)\s+\d{1,2},?\s+\d{4}/i,
|
|
/(today|tomorrow|yesterday)/i
|
|
]
|
|
for (const pattern of datePatterns) {
|
|
const match = joinedTokens.match(pattern)
|
|
if (match) return match[0]
|
|
}
|
|
return null
|
|
|
|
case 'person':
|
|
// Look for capitalized words (proper nouns)
|
|
const personMatch = joinedTokens.match(/\b[A-Z][a-z]+(\s+[A-Z][a-z]+)*\b/)
|
|
return personMatch ? personMatch[0] : null
|
|
|
|
case 'location':
|
|
// Look for location indicators
|
|
const locationPatterns = [
|
|
/\b(in|at|from|to)\s+([A-Z][a-z]+(\s+[A-Z][a-z]+)*)\b/,
|
|
/\b[A-Z][a-z]+,\s+[A-Z]{2}\b/ // City, STATE format
|
|
]
|
|
for (const pattern of locationPatterns) {
|
|
const match = joinedTokens.match(pattern)
|
|
if (match) return match[2] || match[0]
|
|
}
|
|
return null
|
|
|
|
case 'entity':
|
|
case 'text':
|
|
case 'any':
|
|
default:
|
|
// Return first non-common word as potential value
|
|
const commonWords = new Set(['the', 'a', 'an', 'and', 'or', 'but', 'in', 'on', 'at', 'to', 'for'])
|
|
const significantToken = tokens.find(t => !commonWords.has(t.toLowerCase()))
|
|
return significantToken || null
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Get default regex pattern for slot type
|
|
*/
|
|
private getDefaultPatternForType(type: string): string {
|
|
switch (type) {
|
|
case 'number':
|
|
return '\\d+(?:\\.\\d+)?'
|
|
case 'date':
|
|
return '[\\w\\s,/-]+'
|
|
case 'person':
|
|
return '[A-Z][a-z]+(?:\\s+[A-Z][a-z]+)*'
|
|
case 'location':
|
|
return '[A-Z][a-z]+(?:[\\s,]+[A-Z][a-z]+)*'
|
|
case 'entity':
|
|
return '[\\w\\s-]+'
|
|
case 'text':
|
|
case 'any':
|
|
default:
|
|
return '.+'
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Transform value based on type
|
|
*/
|
|
private transformByType(value: string, type: string): any {
|
|
switch (type) {
|
|
case 'number':
|
|
const num = parseFloat(value)
|
|
return isNaN(num) ? value : num
|
|
|
|
case 'date':
|
|
// Simple date parsing
|
|
if (value.toLowerCase() === 'today') {
|
|
return new Date().toISOString().split('T')[0]
|
|
} else if (value.toLowerCase() === 'tomorrow') {
|
|
const tomorrow = new Date()
|
|
tomorrow.setDate(tomorrow.getDate() + 1)
|
|
return tomorrow.toISOString().split('T')[0]
|
|
} else if (value.toLowerCase() === 'yesterday') {
|
|
const yesterday = new Date()
|
|
yesterday.setDate(yesterday.getDate() - 1)
|
|
return yesterday.toISOString().split('T')[0]
|
|
}
|
|
return value
|
|
|
|
case 'person':
|
|
case 'location':
|
|
case 'entity':
|
|
// Capitalize properly
|
|
return value.split(' ')
|
|
.map(word => word.charAt(0).toUpperCase() + word.slice(1).toLowerCase())
|
|
.join(' ')
|
|
|
|
default:
|
|
return value.trim()
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Validate slot value against definition
|
|
*/
|
|
private validateSlotValue(value: any, slotDef: SlotDefinition): boolean {
|
|
if (value === null || value === undefined) {
|
|
return !slotDef.required
|
|
}
|
|
|
|
switch (slotDef.type) {
|
|
case 'number':
|
|
return typeof value === 'number' && !isNaN(value)
|
|
case 'date':
|
|
return typeof value === 'string' && value.length > 0
|
|
case 'text':
|
|
case 'person':
|
|
case 'location':
|
|
case 'entity':
|
|
return typeof value === 'string' && value.length > 0
|
|
case 'any':
|
|
return true
|
|
default:
|
|
return true
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Calculate Levenshtein distance between two strings
|
|
*/
|
|
private levenshteinDistance(s1: string, s2: string): number {
|
|
const len1 = s1.length
|
|
const len2 = s2.length
|
|
const matrix: number[][] = []
|
|
|
|
for (let i = 0; i <= len1; i++) {
|
|
matrix[i] = [i]
|
|
}
|
|
|
|
for (let j = 0; j <= len2; j++) {
|
|
matrix[0][j] = j
|
|
}
|
|
|
|
for (let i = 1; i <= len1; i++) {
|
|
for (let j = 1; j <= len2; j++) {
|
|
const cost = s1[i - 1] === s2[j - 1] ? 0 : 1
|
|
matrix[i][j] = Math.min(
|
|
matrix[i - 1][j] + 1, // deletion
|
|
matrix[i][j - 1] + 1, // insertion
|
|
matrix[i - 1][j - 1] + cost // substitution
|
|
)
|
|
}
|
|
}
|
|
|
|
return matrix[len1][len2]
|
|
}
|
|
|
|
/**
|
|
* Align two strings for slot extraction
|
|
*/
|
|
private alignStrings(query: string, example: string): Array<[string, string]> {
|
|
const queryTokens = this.tokenize(query)
|
|
const exampleTokens = this.tokenize(example)
|
|
const aligned: Array<[string, string]> = []
|
|
|
|
let i = 0, j = 0
|
|
while (i < queryTokens.length && j < exampleTokens.length) {
|
|
if (queryTokens[i] === exampleTokens[j]) {
|
|
aligned.push([queryTokens[i], exampleTokens[j]])
|
|
i++
|
|
j++
|
|
} else {
|
|
// Try to find best match
|
|
const bestMatch = this.findBestTokenMatch(queryTokens[i], exampleTokens.slice(j, j + 3))
|
|
if (bestMatch.index >= 0) {
|
|
j += bestMatch.index
|
|
aligned.push([queryTokens[i], exampleTokens[j]])
|
|
} else {
|
|
aligned.push([queryTokens[i], exampleTokens[j]])
|
|
}
|
|
i++
|
|
j++
|
|
}
|
|
}
|
|
|
|
return aligned
|
|
}
|
|
|
|
/**
|
|
* Find best token match using fuzzy comparison
|
|
*/
|
|
private findBestTokenMatch(token: string, candidates: string[]): { index: number; similarity: number } {
|
|
let bestIndex = -1
|
|
let bestSimilarity = 0
|
|
|
|
candidates.forEach((candidate, index) => {
|
|
const distance = this.levenshteinDistance(token.toLowerCase(), candidate.toLowerCase())
|
|
const similarity = 1 - (distance / Math.max(token.length, candidate.length))
|
|
|
|
if (similarity > bestSimilarity && similarity > 0.6) {
|
|
bestIndex = index
|
|
bestSimilarity = similarity
|
|
}
|
|
})
|
|
|
|
return { index: bestIndex, similarity: bestSimilarity }
|
|
}
|
|
|
|
/**
|
|
* Extract slots from string alignment
|
|
*/
|
|
private extractSlotsFromAlignment(aligned: Array<[string, string]>, _pattern: Pattern): Record<string, any> {
|
|
const slots: Record<string, any> = {}
|
|
let slotIndex = 1
|
|
|
|
aligned.forEach(([queryToken, exampleToken]) => {
|
|
if (exampleToken.startsWith('$')) {
|
|
slots[`$${slotIndex}`] = queryToken
|
|
slotIndex++
|
|
}
|
|
})
|
|
|
|
return slots
|
|
}
|
|
|
|
/**
|
|
* Fill template with extracted slots
|
|
*/
|
|
fillTemplate(template: any, slots: Record<string, any>): any {
|
|
const filled = JSON.parse(JSON.stringify(template))
|
|
|
|
// Recursively replace slot placeholders
|
|
const replacePlaceholders = (obj: any): any => {
|
|
if (typeof obj === 'string') {
|
|
// Replace ${1}, ${2}, etc. with slot values
|
|
return obj.replace(/\$\{(\d+)\}/g, (_, num) => {
|
|
return slots[`$${num}`] || ''
|
|
})
|
|
} else if (Array.isArray(obj)) {
|
|
return obj.map(item => replacePlaceholders(item))
|
|
} else if (typeof obj === 'object' && obj !== null) {
|
|
const result: any = {}
|
|
for (const [key, value] of Object.entries(obj)) {
|
|
const newKey = replacePlaceholders(key)
|
|
result[newKey] = replacePlaceholders(value)
|
|
}
|
|
return result
|
|
}
|
|
return obj
|
|
}
|
|
|
|
return replacePlaceholders(filled)
|
|
}
|
|
|
|
/**
|
|
* Update pattern success metrics based on usage
|
|
*/
|
|
updateSuccessMetric(patternId: string, success: boolean): void {
|
|
const current = this.successMetrics.get(patternId) || 0.5
|
|
|
|
// Exponential moving average
|
|
const alpha = 0.1
|
|
const newMetric = success
|
|
? current + alpha * (1 - current)
|
|
: current - alpha * current
|
|
|
|
this.successMetrics.set(patternId, newMetric)
|
|
}
|
|
|
|
/**
|
|
* Learn new pattern from successful query
|
|
*/
|
|
async learnPattern(query: string, result: any): Promise<void> {
|
|
// Find similar existing patterns
|
|
const queryEmbedding = await this.getEmbedding(query)
|
|
const similar = await this.findBestPatterns(queryEmbedding, 1)
|
|
|
|
if (similar[0]?.similarity < 0.7) {
|
|
// This is a new pattern type - add it
|
|
const newPattern: Pattern = {
|
|
id: `learned_${Date.now()}`,
|
|
category: 'learned',
|
|
examples: [query],
|
|
pattern: this.generateRegexFromQuery(query),
|
|
template: result,
|
|
confidence: 0.6 // Start with moderate confidence
|
|
}
|
|
|
|
this.patterns.set(newPattern.id, newPattern)
|
|
this.patternEmbeddings.set(newPattern.id, queryEmbedding)
|
|
this.successMetrics.set(newPattern.id, 0.6)
|
|
} else {
|
|
// Similar pattern exists - add as example
|
|
const pattern = similar[0].pattern
|
|
if (!pattern.examples.includes(query)) {
|
|
pattern.examples.push(query)
|
|
|
|
// Update pattern embedding with new example
|
|
const embeddings = await Promise.all(
|
|
pattern.examples.map(ex => this.getEmbedding(ex))
|
|
)
|
|
const newEmbedding = this.averageVectors(embeddings)
|
|
this.patternEmbeddings.set(pattern.id, newEmbedding)
|
|
}
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Helper: Average multiple vectors
|
|
*/
|
|
private averageVectors(vectors: Vector[]): Vector {
|
|
if (vectors.length === 0) return []
|
|
|
|
const dim = vectors[0].length
|
|
const avg = new Array(dim).fill(0)
|
|
|
|
for (const vec of vectors) {
|
|
for (let i = 0; i < dim; i++) {
|
|
avg[i] += vec[i]
|
|
}
|
|
}
|
|
|
|
for (let i = 0; i < dim; i++) {
|
|
avg[i] /= vectors.length
|
|
}
|
|
|
|
return avg
|
|
}
|
|
|
|
/**
|
|
* Helper: Calculate cosine similarity
|
|
*/
|
|
private cosineSimilarity(a: Vector, b: Vector): number {
|
|
let dotProduct = 0
|
|
let normA = 0
|
|
let normB = 0
|
|
|
|
for (let i = 0; i < a.length; i++) {
|
|
dotProduct += a[i] * b[i]
|
|
normA += a[i] * a[i]
|
|
normB += b[i] * b[i]
|
|
}
|
|
|
|
normA = Math.sqrt(normA)
|
|
normB = Math.sqrt(normB)
|
|
|
|
if (normA === 0 || normB === 0) return 0
|
|
return dotProduct / (normA * normB)
|
|
}
|
|
|
|
/**
|
|
* Helper: Simple tokenization
|
|
*/
|
|
private tokenize(text: string): string[] {
|
|
return text.toLowerCase().split(/\s+/).filter(t => t.length > 0)
|
|
}
|
|
|
|
/**
|
|
* Helper: Post-process extracted slots
|
|
*/
|
|
private postProcessSlots(slots: Record<string, any>, _pattern: Pattern): void {
|
|
// Convert string numbers to actual numbers
|
|
for (const [key, value] of Object.entries(slots)) {
|
|
if (typeof value === 'string') {
|
|
// Check if it's a number
|
|
const num = parseFloat(value)
|
|
if (!isNaN(num) && value.match(/^\d+(\.\d+)?$/)) {
|
|
slots[key] = num
|
|
}
|
|
|
|
// Parse dates
|
|
if (value.match(/\d{4}/) || value.match(/(january|february|march|april|may|june|july|august|september|october|november|december)/i)) {
|
|
// Simple year extraction
|
|
const year = value.match(/\d{4}/)
|
|
if (year) {
|
|
slots[key] = parseInt(year[0])
|
|
}
|
|
}
|
|
|
|
// Clean up captured values
|
|
slots[key] = value.trim()
|
|
}
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Helper: Generate regex pattern from query
|
|
*/
|
|
private generateRegexFromQuery(query: string): string {
|
|
// Simple pattern generation - replace variable parts with capture groups
|
|
let pattern = query.toLowerCase()
|
|
|
|
// Replace numbers with \d+ capture
|
|
pattern = pattern.replace(/\d+/g, '(\\d+)')
|
|
|
|
// Replace quoted strings with .+ capture
|
|
pattern = pattern.replace(/"[^"]+"/g, '(.+)')
|
|
|
|
// Replace proper nouns (capitalized words) with capture
|
|
pattern = pattern.replace(/\b[A-Z]\w+\b/g, '([A-Z][\\w]+)')
|
|
|
|
return pattern
|
|
}
|
|
|
|
/**
|
|
* Get pattern statistics for monitoring
|
|
*/
|
|
getStatistics(): {
|
|
totalPatterns: number
|
|
categories: Record<string, number>
|
|
averageConfidence: number
|
|
topPatterns: Array<{ id: string; success: number }>
|
|
} {
|
|
const stats = {
|
|
totalPatterns: this.patterns.size,
|
|
categories: {} as Record<string, number>,
|
|
averageConfidence: 0,
|
|
topPatterns: [] as Array<{ id: string; success: number }>
|
|
}
|
|
|
|
// Count by category
|
|
for (const pattern of this.patterns.values()) {
|
|
stats.categories[pattern.category] = (stats.categories[pattern.category] || 0) + 1
|
|
}
|
|
|
|
// Calculate average confidence
|
|
let totalConfidence = 0
|
|
for (const confidence of this.successMetrics.values()) {
|
|
totalConfidence += confidence
|
|
}
|
|
stats.averageConfidence = totalConfidence / this.successMetrics.size
|
|
|
|
// Get top patterns by success
|
|
const sortedPatterns = Array.from(this.successMetrics.entries())
|
|
.sort((a, b) => b[1] - a[1])
|
|
.slice(0, 10)
|
|
|
|
stats.topPatterns = sortedPatterns.map(([id, success]) => ({ id, success }))
|
|
|
|
return stats
|
|
}
|
|
} |