refactor(8.0): remove dead, unreachable, and unwired modules
Pre-GA dead-code sweep. A deterministic import-reachability walk from the
package's real entry points (the exports map, the CLI bin, the conversation
surface) found 34 source modules that nothing reachable imports — they
compile and ship as dist output that no consumer or internal path can ever
reach. All were superseded duplicates, abandoned parallel implementations,
or built-but-never-wired features:
- superseded duplicates of live modules: an older "unified" entry, a
standalone neural-import variant, a static NLP processor + its matcher,
a parallel API-types module, a duplicate progress-types module
- an abandoned import path (orchestrator + entity deduplicator + barrels)
left behind when ingestion moved to the coordinator
- unwired feature modules (instance pool, import presets, cached
embeddings, relationship-confidence scorer) reachable only from tests
- dead leaf utilities (write buffer, deleted-items index, bounded
registry, two crypto shims, a cache manager, a structured logger, a
stale v5 type-migration helper, a browser-only FS type shim) and
several dead re-export barrels
- a CLI catalog command wired into no command
Also removed two tests that only exercised deleted modules, repointed two
VFS tests off a deleted barrel onto the implementation module, and deleted
one example that demoed removed features.
Verified: clean build (both tsc passes), unit 1505 + integration 607 green,
and a re-run of the reachability walk reports zero remaining orphans.
Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
parent
03d654061f
commit
bf0afe8563
39 changed files with 2 additions and 10497 deletions
|
|
@ -1,200 +0,0 @@
|
|||
/**
|
||||
* 🧠 Natural Language Query Processor - STATIC VERSION
|
||||
* No runtime initialization, no memory leaks, patterns pre-built at compile time
|
||||
*
|
||||
* Uses static pattern matching with 220 pre-built patterns
|
||||
*/
|
||||
|
||||
import { Vector } from '../coreTypes.js'
|
||||
import { TripleQuery } from '../triple/TripleIntelligence.js'
|
||||
import { patternMatchQuery, PATTERN_STATS } from './staticPatternMatcher.js'
|
||||
|
||||
export interface NaturalQueryIntent {
|
||||
type: 'vector' | 'field' | 'graph' | 'combined'
|
||||
confidence: number
|
||||
extractedTerms: {
|
||||
entities?: string[]
|
||||
fields?: string[]
|
||||
relationships?: string[]
|
||||
modifiers?: string[]
|
||||
}
|
||||
}
|
||||
|
||||
export class NaturalLanguageProcessor {
|
||||
private queryHistory: Array<{ query: string; result: TripleQuery; success: boolean }>
|
||||
|
||||
constructor() {
|
||||
this.queryHistory = []
|
||||
// Patterns are static - no initialization needed!
|
||||
}
|
||||
|
||||
/**
|
||||
* No initialization needed - patterns are pre-built!
|
||||
*/
|
||||
async init(): Promise<void> {
|
||||
// Nothing to do - patterns are compiled into the code
|
||||
return Promise.resolve()
|
||||
}
|
||||
|
||||
/**
|
||||
* Process natural language query into structured Triple Intelligence query
|
||||
* @param naturalQuery The natural language query string
|
||||
* @param queryEmbedding Pre-computed embedding from Brainy (passed in to avoid circular dependency)
|
||||
*/
|
||||
async processNaturalQuery(naturalQuery: string, queryEmbedding?: Vector): Promise<TripleQuery> {
|
||||
// Use static pattern matcher (no async, no memory allocation!)
|
||||
const structuredQuery = patternMatchQuery(naturalQuery, queryEmbedding)
|
||||
|
||||
// Step 3: Enhance with intent analysis if needed
|
||||
if (!structuredQuery.where && !structuredQuery.connected) {
|
||||
const intent = await this.analyzeIntent(naturalQuery)
|
||||
|
||||
// Add metadata based on intent
|
||||
if (intent.type === 'field' && intent.extractedTerms.fields) {
|
||||
structuredQuery.where = this.buildFieldConstraints(intent.extractedTerms.fields)
|
||||
}
|
||||
}
|
||||
|
||||
// Track for learning (but don't create new Brainy!)
|
||||
this.queryHistory.push({
|
||||
query: naturalQuery,
|
||||
result: structuredQuery,
|
||||
success: false // Will be updated based on user interaction
|
||||
})
|
||||
|
||||
// Keep history limited to prevent memory growth
|
||||
if (this.queryHistory.length > 100) {
|
||||
this.queryHistory.shift()
|
||||
}
|
||||
|
||||
return structuredQuery
|
||||
}
|
||||
|
||||
/**
|
||||
* Analyze query intent using keywords
|
||||
*/
|
||||
private async analyzeIntent(query: string): Promise<NaturalQueryIntent> {
|
||||
const lowerQuery = query.toLowerCase()
|
||||
|
||||
// Check for field-specific keywords
|
||||
const fieldKeywords = ['where', 'filter', 'with', 'has', 'contains', 'equals', 'greater', 'less', 'between']
|
||||
const hasFieldIntent = fieldKeywords.some(kw => lowerQuery.includes(kw))
|
||||
|
||||
// Check for graph keywords
|
||||
const graphKeywords = ['related', 'connected', 'linked', 'associated', 'references']
|
||||
const hasGraphIntent = graphKeywords.some(kw => lowerQuery.includes(kw))
|
||||
|
||||
// Determine type
|
||||
let type: NaturalQueryIntent['type'] = 'vector'
|
||||
if (hasFieldIntent && hasGraphIntent) {
|
||||
type = 'combined'
|
||||
} else if (hasFieldIntent) {
|
||||
type = 'field'
|
||||
} else if (hasGraphIntent) {
|
||||
type = 'graph'
|
||||
}
|
||||
|
||||
return {
|
||||
type,
|
||||
confidence: 0.8,
|
||||
extractedTerms: {
|
||||
fields: hasFieldIntent ? this.extractFieldTerms(query) : undefined,
|
||||
relationships: hasGraphIntent ? this.extractRelationshipTerms(query) : undefined
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Extract field terms from query
|
||||
*/
|
||||
private extractFieldTerms(query: string): string[] {
|
||||
const terms: string[] = []
|
||||
|
||||
// Simple extraction of potential field names
|
||||
const words = query.split(/\s+/)
|
||||
const fieldIndicators = ['year', 'date', 'author', 'type', 'category', 'status', 'price']
|
||||
|
||||
for (const word of words) {
|
||||
if (fieldIndicators.includes(word.toLowerCase())) {
|
||||
terms.push(word.toLowerCase())
|
||||
}
|
||||
}
|
||||
|
||||
return terms
|
||||
}
|
||||
|
||||
/**
|
||||
* Extract relationship terms
|
||||
*/
|
||||
private extractRelationshipTerms(query: string): string[] {
|
||||
const terms: string[] = []
|
||||
const relationshipWords = ['related', 'connected', 'linked', 'references', 'cites']
|
||||
|
||||
const words = query.toLowerCase().split(/\s+/)
|
||||
for (const word of words) {
|
||||
if (relationshipWords.includes(word)) {
|
||||
terms.push(word)
|
||||
}
|
||||
}
|
||||
|
||||
return terms
|
||||
}
|
||||
|
||||
/**
|
||||
* Build field constraints from extracted terms
|
||||
*/
|
||||
private buildFieldConstraints(fields: string[]): Record<string, any> {
|
||||
const constraints: Record<string, any> = {}
|
||||
|
||||
// Simple mapping for common fields
|
||||
for (const field of fields) {
|
||||
// This would be enhanced with actual value extraction
|
||||
constraints[field] = { exists: true }
|
||||
}
|
||||
|
||||
return constraints
|
||||
}
|
||||
|
||||
/**
|
||||
* Find similar queries from history (without using Brainy)
|
||||
* NOTE: Currently unused - reserved for future query caching optimization
|
||||
*/
|
||||
private findSimilarQueries(embedding: Vector): Array<{
|
||||
query: string
|
||||
result: TripleQuery
|
||||
similarity: number
|
||||
}> {
|
||||
// Not implemented - not required for core functionality
|
||||
// Would implement cosine similarity against queryHistory if needed
|
||||
return []
|
||||
}
|
||||
|
||||
/**
|
||||
* Adapt a previous query for new input
|
||||
*/
|
||||
private adaptQuery(newQuery: string, previousResult: TripleQuery): TripleQuery {
|
||||
return previousResult
|
||||
}
|
||||
|
||||
/**
|
||||
* Extract entities from query
|
||||
*/
|
||||
private async extractEntities(query: string): Promise<string[]> {
|
||||
// Could use the Entity Registry here if available
|
||||
return []
|
||||
}
|
||||
|
||||
/**
|
||||
* Build query from components
|
||||
*/
|
||||
private buildQuery(
|
||||
query: string,
|
||||
intent: NaturalQueryIntent,
|
||||
entities: string[]
|
||||
): TripleQuery {
|
||||
return {
|
||||
like: query,
|
||||
limit: 10
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -1,633 +0,0 @@
|
|||
/**
|
||||
* Neural Import - AI-Powered Data Understanding
|
||||
*
|
||||
* Standalone implementation for intelligent data processing.
|
||||
*/
|
||||
|
||||
import { NounType, VerbType } from '../types/graphTypes.js'
|
||||
import * as fs from '../universal/fs.js'
|
||||
import * as path from '../universal/path.js'
|
||||
import { prodLog } from '../utils/logger.js'
|
||||
|
||||
// Neural Import Analysis Types
|
||||
export interface NeuralAnalysisResult {
|
||||
detectedEntities: DetectedEntity[]
|
||||
detectedRelationships: DetectedRelationship[]
|
||||
confidence: number
|
||||
insights: NeuralInsight[]
|
||||
}
|
||||
|
||||
export interface DetectedEntity {
|
||||
originalData: any
|
||||
nounType: string
|
||||
confidence: number
|
||||
suggestedId: string
|
||||
reasoning: string
|
||||
alternativeTypes: Array<{ type: string, confidence: number }>
|
||||
}
|
||||
|
||||
export interface DetectedRelationship {
|
||||
sourceId: string
|
||||
targetId: string
|
||||
verbType: string
|
||||
confidence: number
|
||||
weight: number
|
||||
reasoning: string
|
||||
context: string
|
||||
metadata?: Record<string, any>
|
||||
}
|
||||
|
||||
export interface NeuralInsight {
|
||||
type: 'hierarchy' | 'cluster' | 'pattern' | 'anomaly' | 'opportunity'
|
||||
description: string
|
||||
confidence: number
|
||||
affectedEntities: string[]
|
||||
recommendation?: string
|
||||
}
|
||||
|
||||
export interface NeuralImportConfig {
|
||||
confidenceThreshold: number
|
||||
enableWeights: boolean
|
||||
skipDuplicates: boolean
|
||||
categoryFilter?: string[]
|
||||
dataType?: string
|
||||
}
|
||||
|
||||
/**
|
||||
* Neural Import Augmentation - Unified Implementation
|
||||
* Processes data with AI before storage operations
|
||||
*/
|
||||
export class NeuralImportAugmentation {
|
||||
readonly name = 'neural-import'
|
||||
private operations = ['add', 'addNoun', 'addVerb', 'all']
|
||||
|
||||
protected config: NeuralImportConfig
|
||||
private analysisCache = new Map<string, NeuralAnalysisResult>()
|
||||
private context?: { brain: any }
|
||||
|
||||
constructor(config: Partial<NeuralImportConfig> = {}) {
|
||||
this.config = {
|
||||
confidenceThreshold: 0.7,
|
||||
enableWeights: true,
|
||||
skipDuplicates: true,
|
||||
dataType: 'json',
|
||||
...config
|
||||
}
|
||||
}
|
||||
|
||||
async init(): Promise<void> {
|
||||
// No external dependencies to initialize
|
||||
}
|
||||
|
||||
private log(message: string, _level?: string): void {
|
||||
// Silent by default
|
||||
}
|
||||
|
||||
/**
|
||||
* Execute augmentation - process data with AI before storage
|
||||
*/
|
||||
async execute<T = any>(
|
||||
operation: string,
|
||||
params: any,
|
||||
next: () => Promise<T>
|
||||
): Promise<T> {
|
||||
// Only process on add operations
|
||||
if (!this.operations.includes(operation)) {
|
||||
return next()
|
||||
}
|
||||
|
||||
try {
|
||||
// Extract data from params based on operation
|
||||
const rawData = this.extractRawData(operation, params)
|
||||
if (!rawData) {
|
||||
return next()
|
||||
}
|
||||
|
||||
// Perform neural analysis
|
||||
const analysis = await this.performNeuralAnalysis(rawData, this.config)
|
||||
|
||||
// Enhance params with neural insights
|
||||
if (params.metadata) {
|
||||
params.metadata._neuralProcessed = true
|
||||
params.metadata._neuralConfidence = analysis.confidence
|
||||
params.metadata._detectedEntities = analysis.detectedEntities.length
|
||||
params.metadata._detectedRelationships = analysis.detectedRelationships.length
|
||||
params.metadata._neuralInsights = analysis.insights
|
||||
} else if (typeof params === 'object') {
|
||||
params.metadata = {
|
||||
_neuralProcessed: true,
|
||||
_neuralConfidence: analysis.confidence,
|
||||
_detectedEntities: analysis.detectedEntities.length,
|
||||
_detectedRelationships: analysis.detectedRelationships.length,
|
||||
_neuralInsights: analysis.insights
|
||||
}
|
||||
}
|
||||
|
||||
// Store neural analysis for later retrieval
|
||||
await this.storeNeuralAnalysis(analysis)
|
||||
|
||||
// If we detected entities/relationships, potentially add them
|
||||
if (this.context?.brain && analysis.detectedEntities.length > 0) {
|
||||
// This could automatically create entities/relationships
|
||||
// But for now, just enhance the metadata
|
||||
this.log(`Detected ${analysis.detectedEntities.length} entities and ${analysis.detectedRelationships.length} relationships`)
|
||||
}
|
||||
|
||||
// Continue with enhanced data
|
||||
return next()
|
||||
} catch (error) {
|
||||
this.log(`Neural analysis failed: ${error}`, 'warn')
|
||||
// Continue without neural processing
|
||||
return next()
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Extract raw data from operation params
|
||||
*/
|
||||
private extractRawData(operation: string, params: any): any {
|
||||
switch (operation) {
|
||||
case 'add':
|
||||
return params.content || params.data || params
|
||||
case 'addNoun':
|
||||
return params.noun || params.data || params
|
||||
case 'addVerb':
|
||||
return params.verb || params
|
||||
case 'addBatch':
|
||||
return params.items || params.batch || params
|
||||
default:
|
||||
return null
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the full neural analysis result (for external use)
|
||||
*/
|
||||
async getNeuralAnalysis(rawData: Buffer | string, dataType?: string): Promise<NeuralAnalysisResult> {
|
||||
const parsedData = await this.parseRawData(rawData, dataType || this.config.dataType || 'json')
|
||||
return await this.performNeuralAnalysis(parsedData, this.config)
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse raw data based on type
|
||||
*/
|
||||
private async parseRawData(rawData: Buffer | string, dataType: string): Promise<any[]> {
|
||||
const content = typeof rawData === 'string' ? rawData : rawData.toString('utf8')
|
||||
|
||||
switch (dataType.toLowerCase()) {
|
||||
case 'json':
|
||||
try {
|
||||
const jsonData = JSON.parse(content)
|
||||
return Array.isArray(jsonData) ? jsonData : [jsonData]
|
||||
} catch {
|
||||
// If JSON parse fails, treat as text
|
||||
return [{ text: content }]
|
||||
}
|
||||
|
||||
case 'csv':
|
||||
return this.parseCSV(content)
|
||||
|
||||
case 'yaml':
|
||||
case 'yml':
|
||||
return this.parseYAML(content)
|
||||
|
||||
case 'txt':
|
||||
case 'text':
|
||||
// Split text into sentences/paragraphs for analysis
|
||||
return content.split(/\n+/).filter(line => line.trim()).map(line => ({ text: line }))
|
||||
|
||||
default:
|
||||
// Unknown type, treat as text
|
||||
return [{ text: content }]
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse CSV data - handles quoted values, escaped quotes, and edge cases
|
||||
*/
|
||||
private parseCSV(content: string): any[] {
|
||||
const lines = content.split('\n')
|
||||
if (lines.length === 0) return []
|
||||
|
||||
// Parse a CSV line handling quotes
|
||||
const parseLine = (line: string): string[] => {
|
||||
const result: string[] = []
|
||||
let current = ''
|
||||
let inQuotes = false
|
||||
let i = 0
|
||||
|
||||
while (i < line.length) {
|
||||
const char = line[i]
|
||||
const nextChar = line[i + 1]
|
||||
|
||||
if (char === '"') {
|
||||
if (inQuotes && nextChar === '"') {
|
||||
// Escaped quote
|
||||
current += '"'
|
||||
i += 2
|
||||
} else {
|
||||
// Toggle quote mode
|
||||
inQuotes = !inQuotes
|
||||
i++
|
||||
}
|
||||
} else if (char === ',' && !inQuotes) {
|
||||
// Field separator
|
||||
result.push(current.trim())
|
||||
current = ''
|
||||
i++
|
||||
} else {
|
||||
current += char
|
||||
i++
|
||||
}
|
||||
}
|
||||
|
||||
// Add last field
|
||||
result.push(current.trim())
|
||||
return result
|
||||
}
|
||||
|
||||
// Parse headers
|
||||
const headers = parseLine(lines[0])
|
||||
const data = []
|
||||
|
||||
// Parse data rows
|
||||
for (let i = 1; i < lines.length; i++) {
|
||||
const line = lines[i].trim()
|
||||
if (!line) continue // Skip empty lines
|
||||
|
||||
const values = parseLine(line)
|
||||
const row: any = {}
|
||||
|
||||
headers.forEach((header, index) => {
|
||||
const value = values[index] || ''
|
||||
// Try to parse numbers
|
||||
const num = Number(value)
|
||||
row[header] = !isNaN(num) && value !== '' ? num : value
|
||||
})
|
||||
|
||||
data.push(row)
|
||||
}
|
||||
|
||||
return data
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse YAML data
|
||||
*/
|
||||
private parseYAML(content: string): any[] {
|
||||
try {
|
||||
// Simple YAML parser for basic structures
|
||||
// For full YAML support, we'd use js-yaml library
|
||||
const lines = content.split('\n')
|
||||
const result: any[] = []
|
||||
let currentObject: any = null
|
||||
let currentIndent = 0
|
||||
|
||||
for (const line of lines) {
|
||||
const trimmed = line.trim()
|
||||
if (!trimmed || trimmed.startsWith('#')) continue // Skip empty lines and comments
|
||||
|
||||
// Calculate indentation
|
||||
const indent = line.length - line.trimStart().length
|
||||
|
||||
// Check for array item
|
||||
if (trimmed.startsWith('- ')) {
|
||||
const value = trimmed.substring(2).trim()
|
||||
if (indent === 0) {
|
||||
// Top-level array item
|
||||
if (value.includes(':')) {
|
||||
// Object in array
|
||||
currentObject = {}
|
||||
result.push(currentObject)
|
||||
const [key, val] = value.split(':').map(s => s.trim())
|
||||
currentObject[key] = this.parseYAMLValue(val)
|
||||
} else {
|
||||
result.push(this.parseYAMLValue(value))
|
||||
}
|
||||
} else if (currentObject) {
|
||||
// Nested array
|
||||
const lastKey = Object.keys(currentObject).pop()
|
||||
if (lastKey) {
|
||||
if (!Array.isArray(currentObject[lastKey])) {
|
||||
currentObject[lastKey] = []
|
||||
}
|
||||
currentObject[lastKey].push(this.parseYAMLValue(value))
|
||||
}
|
||||
}
|
||||
} else if (trimmed.includes(':')) {
|
||||
// Key-value pair
|
||||
const colonIndex = trimmed.indexOf(':')
|
||||
const key = trimmed.substring(0, colonIndex).trim()
|
||||
const value = trimmed.substring(colonIndex + 1).trim()
|
||||
|
||||
if (indent === 0) {
|
||||
// Top-level object
|
||||
if (!currentObject) {
|
||||
currentObject = {}
|
||||
result.push(currentObject)
|
||||
}
|
||||
currentObject[key] = this.parseYAMLValue(value)
|
||||
currentIndent = 0
|
||||
} else if (currentObject) {
|
||||
// Nested object
|
||||
if (indent > currentIndent && !value) {
|
||||
// Start of nested object
|
||||
const lastKey = Object.keys(currentObject).pop()
|
||||
if (lastKey) {
|
||||
currentObject[lastKey] = { [key]: '' }
|
||||
}
|
||||
} else {
|
||||
currentObject[key] = this.parseYAMLValue(value)
|
||||
}
|
||||
currentIndent = indent
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// If we built a single object and not an array, wrap it
|
||||
if (result.length === 0 && currentObject) {
|
||||
result.push(currentObject)
|
||||
}
|
||||
|
||||
return result.length > 0 ? result : [{ text: content }]
|
||||
} catch (error) {
|
||||
prodLog.warn('YAML parsing failed, treating as text:', error)
|
||||
return [{ text: content }]
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse a YAML value (handle strings, numbers, booleans, null)
|
||||
*/
|
||||
private parseYAMLValue(value: string): any {
|
||||
if (!value || value === '~' || value === 'null') return null
|
||||
if (value === 'true') return true
|
||||
if (value === 'false') return false
|
||||
|
||||
// Remove quotes if present
|
||||
if ((value.startsWith('"') && value.endsWith('"')) ||
|
||||
(value.startsWith("'") && value.endsWith("'"))) {
|
||||
return value.slice(1, -1)
|
||||
}
|
||||
|
||||
// Try to parse as number
|
||||
const num = Number(value)
|
||||
if (!isNaN(num) && value !== '') return num
|
||||
|
||||
return value
|
||||
}
|
||||
|
||||
/**
|
||||
* Perform neural analysis on parsed data
|
||||
*/
|
||||
private async performNeuralAnalysis(data: any[], config?: any): Promise<NeuralAnalysisResult> {
|
||||
const detectedEntities: DetectedEntity[] = []
|
||||
const detectedRelationships: DetectedRelationship[] = []
|
||||
const insights: NeuralInsight[] = []
|
||||
|
||||
// Simple entity detection (in real implementation, would use ML)
|
||||
for (const item of data) {
|
||||
if (typeof item === 'object') {
|
||||
// Detect entities from object properties
|
||||
const entityId = item.id || item.name || item.title || `entity_${Date.now()}_${Math.random()}`
|
||||
|
||||
detectedEntities.push({
|
||||
originalData: item,
|
||||
nounType: await this.inferNounType(item),
|
||||
confidence: 0.85,
|
||||
suggestedId: String(entityId),
|
||||
reasoning: 'Detected from structured data',
|
||||
alternativeTypes: []
|
||||
})
|
||||
|
||||
// Detect relationships from references
|
||||
await this.detectRelationships(item, entityId, detectedRelationships)
|
||||
}
|
||||
}
|
||||
|
||||
// Generate insights
|
||||
if (detectedEntities.length > 10) {
|
||||
insights.push({
|
||||
type: 'pattern',
|
||||
description: `Large dataset with ${detectedEntities.length} entities detected`,
|
||||
confidence: 0.9,
|
||||
affectedEntities: detectedEntities.slice(0, 5).map(e => e.suggestedId),
|
||||
recommendation: 'Consider batch processing for optimal performance'
|
||||
})
|
||||
}
|
||||
|
||||
// Look for clusters
|
||||
const typeGroups = this.groupByType(detectedEntities)
|
||||
if (Object.keys(typeGroups).length > 1) {
|
||||
insights.push({
|
||||
type: 'cluster',
|
||||
description: `Multiple entity types detected: ${Object.keys(typeGroups).join(', ')}`,
|
||||
confidence: 0.8,
|
||||
affectedEntities: [],
|
||||
recommendation: 'Data contains diverse entity types suitable for graph analysis'
|
||||
})
|
||||
}
|
||||
|
||||
return {
|
||||
detectedEntities,
|
||||
detectedRelationships,
|
||||
confidence: detectedEntities.length > 0 ? 0.85 : 0.5,
|
||||
insights
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Infer noun type from object structure using field heuristics
|
||||
*/
|
||||
private async inferNounType(obj: any): Promise<string> {
|
||||
if (typeof obj !== 'object' || obj === null) return NounType.Thing
|
||||
|
||||
// Check for explicit type field
|
||||
if (obj.type && typeof obj.type === 'string') {
|
||||
const normalized = obj.type.charAt(0).toUpperCase() + obj.type.slice(1)
|
||||
if (Object.values(NounType).includes(normalized as NounType)) {
|
||||
return normalized as NounType
|
||||
}
|
||||
}
|
||||
|
||||
if (obj.email || obj.firstName || obj.lastName || obj.username) return NounType.Person
|
||||
if (obj.companyName || obj.organizationId || obj.employees) return NounType.Organization
|
||||
if (obj.latitude || obj.longitude || obj.address || obj.city) return NounType.Location
|
||||
if ((obj.content && (obj.title || obj.author)) || obj.pages) return NounType.Document
|
||||
if (obj.startTime || obj.endTime || obj.date || obj.attendees) return NounType.Event
|
||||
if (obj.price || obj.sku || obj.productId) return NounType.Product
|
||||
if ((obj.status && obj.assignee) || obj.priority) return NounType.Task
|
||||
if (Array.isArray(obj.data) || obj.rows || obj.columns) return NounType.Dataset
|
||||
|
||||
return NounType.Thing
|
||||
}
|
||||
|
||||
/**
|
||||
* Detect relationships from object references
|
||||
*/
|
||||
private async detectRelationships(obj: any, sourceId: string, relationships: DetectedRelationship[]): Promise<void> {
|
||||
// Look for reference patterns
|
||||
for (const [key, value] of Object.entries(obj)) {
|
||||
if (key.endsWith('Id') || key.endsWith('_id') || key === 'parentId' || key === 'userId') {
|
||||
relationships.push({
|
||||
sourceId,
|
||||
targetId: String(value),
|
||||
verbType: await this.inferVerbType(key, obj, { id: value }),
|
||||
confidence: 0.75,
|
||||
weight: 1,
|
||||
reasoning: `Reference detected in field: ${key}`,
|
||||
context: key
|
||||
})
|
||||
}
|
||||
|
||||
// Array of IDs
|
||||
if (Array.isArray(value) && value.length > 0 && typeof value[0] === 'string') {
|
||||
if (key.endsWith('Ids') || key.endsWith('_ids')) {
|
||||
for (const targetId of value) {
|
||||
relationships.push({
|
||||
sourceId,
|
||||
targetId: String(targetId),
|
||||
verbType: await this.inferVerbType(key, obj, { id: targetId }),
|
||||
confidence: 0.7,
|
||||
weight: 1,
|
||||
reasoning: `Array reference in field: ${key}`,
|
||||
context: key
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Infer verb type from field name using common patterns
|
||||
*/
|
||||
private async inferVerbType(fieldName: string, _sourceObj?: any, _targetObj?: any): Promise<string> {
|
||||
const field = fieldName.toLowerCase()
|
||||
|
||||
if (field.includes('parent') || field.includes('child') || field.includes('contain')) {
|
||||
return VerbType.Contains
|
||||
}
|
||||
if (field.includes('owner') || field.includes('created') || field.includes('author')) {
|
||||
return VerbType.Creates
|
||||
}
|
||||
if (field.includes('member') || field.includes('belong')) {
|
||||
return VerbType.MemberOf
|
||||
}
|
||||
if (field.includes('depend') || field.includes('require')) {
|
||||
return VerbType.DependsOn
|
||||
}
|
||||
if (field.includes('ref') || field.includes('link') || field.includes('source')) {
|
||||
return VerbType.References
|
||||
}
|
||||
|
||||
return VerbType.RelatedTo
|
||||
}
|
||||
|
||||
/**
|
||||
* Group entities by type
|
||||
*/
|
||||
private groupByType(entities: DetectedEntity[]): Record<string, DetectedEntity[]> {
|
||||
const groups: Record<string, DetectedEntity[]> = {}
|
||||
|
||||
for (const entity of entities) {
|
||||
if (!groups[entity.nounType]) {
|
||||
groups[entity.nounType] = []
|
||||
}
|
||||
groups[entity.nounType].push(entity)
|
||||
}
|
||||
|
||||
return groups
|
||||
}
|
||||
|
||||
/**
|
||||
* Store neural analysis results
|
||||
*/
|
||||
private async storeNeuralAnalysis(analysis: NeuralAnalysisResult): Promise<void> {
|
||||
// Cache the analysis for potential later use
|
||||
const key = `analysis_${Date.now()}`
|
||||
this.analysisCache.set(key, analysis)
|
||||
|
||||
// Limit cache size
|
||||
if (this.analysisCache.size > 100) {
|
||||
const firstKey = this.analysisCache.keys().next().value
|
||||
if (firstKey) {
|
||||
this.analysisCache.delete(firstKey)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Helper to get data type from file path
|
||||
*/
|
||||
private getDataTypeFromPath(filePath: string): string {
|
||||
const ext = path.extname(filePath).toLowerCase()
|
||||
switch (ext) {
|
||||
case '.json': return 'json'
|
||||
case '.csv': return 'csv'
|
||||
case '.txt': return 'text'
|
||||
case '.yaml':
|
||||
case '.yml': return 'yaml'
|
||||
default: return 'text'
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* PUBLIC API: Process raw data (for external use, like Synapses)
|
||||
* This maintains compatibility with code that wants to use Neural Import directly
|
||||
*/
|
||||
async processRawData(
|
||||
rawData: Buffer | string,
|
||||
dataType: string,
|
||||
options?: Record<string, unknown>
|
||||
): Promise<{
|
||||
success: boolean
|
||||
data: {
|
||||
nouns: string[]
|
||||
verbs: string[]
|
||||
confidence?: number
|
||||
insights?: Array<{
|
||||
type: string
|
||||
description: string
|
||||
confidence: number
|
||||
}>
|
||||
metadata?: Record<string, unknown>
|
||||
}
|
||||
error?: string
|
||||
}> {
|
||||
try {
|
||||
const analysis = await this.getNeuralAnalysis(rawData, dataType)
|
||||
|
||||
// Convert to legacy format for compatibility
|
||||
const nouns = analysis.detectedEntities.map(e => e.suggestedId)
|
||||
const verbs = analysis.detectedRelationships.map(r =>
|
||||
`${r.sourceId}->${r.verbType}->${r.targetId}`
|
||||
)
|
||||
|
||||
return {
|
||||
success: true,
|
||||
data: {
|
||||
nouns,
|
||||
verbs,
|
||||
confidence: analysis.confidence,
|
||||
insights: analysis.insights.map(i => ({
|
||||
type: i.type,
|
||||
description: i.description,
|
||||
confidence: i.confidence
|
||||
})),
|
||||
metadata: {
|
||||
detectedEntities: analysis.detectedEntities.length,
|
||||
detectedRelationships: analysis.detectedRelationships.length,
|
||||
timestamp: new Date().toISOString()
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
return {
|
||||
success: false,
|
||||
data: { nouns: [], verbs: [] },
|
||||
error: error instanceof Error ? error.message : 'Neural analysis failed'
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -1,479 +0,0 @@
|
|||
/**
|
||||
* Smart Import Presets - Zero-Configuration Auto-Detection
|
||||
*
|
||||
* Automatically selects optimal import strategy based on:
|
||||
* - File type (Excel, CSV, PDF, Markdown, JSON)
|
||||
* - File size and row count
|
||||
* - Column structure (explicit relationships vs narrative)
|
||||
* - Available memory and performance requirements
|
||||
*
|
||||
* Production-ready: Handles billions of entities with optimal performance
|
||||
*/
|
||||
|
||||
import { NounType, VerbType } from '../types/graphTypes.js'
|
||||
|
||||
/**
|
||||
* Signal types used for entity classification
|
||||
*/
|
||||
export type SignalType = 'embedding' | 'exact' | 'pattern' | 'context'
|
||||
|
||||
/**
|
||||
* Strategy types used for relationship extraction
|
||||
*/
|
||||
export type StrategyType = 'explicit' | 'pattern' | 'embedding'
|
||||
|
||||
/**
|
||||
* Import context for preset auto-detection
|
||||
*/
|
||||
export interface ImportContext {
|
||||
fileType?: 'excel' | 'csv' | 'json' | 'pdf' | 'markdown' | 'unknown'
|
||||
fileSize?: number // bytes
|
||||
rowCount?: number
|
||||
hasExplicitColumns?: boolean // Has "Related Terms" or similar columns
|
||||
hasNarrativeContent?: boolean // Has long-form text/descriptions
|
||||
avgDefinitionLength?: number // Average length of definitions
|
||||
memoryAvailable?: number // bytes
|
||||
}
|
||||
|
||||
/**
|
||||
* Signal configuration with weights
|
||||
*/
|
||||
export interface SignalConfig {
|
||||
enabled: SignalType[]
|
||||
weights: Record<SignalType, number>
|
||||
timeout: number // milliseconds
|
||||
}
|
||||
|
||||
/**
|
||||
* Strategy configuration with priorities
|
||||
*/
|
||||
export interface StrategyConfig {
|
||||
enabled: StrategyType[]
|
||||
timeout: number // milliseconds
|
||||
earlyTermination: boolean
|
||||
minConfidence: number
|
||||
}
|
||||
|
||||
/**
|
||||
* Complete preset configuration
|
||||
*/
|
||||
export interface PresetConfig {
|
||||
name: string
|
||||
description: string
|
||||
signals: SignalConfig
|
||||
strategies: StrategyConfig
|
||||
streaming: boolean
|
||||
batchSize: number
|
||||
}
|
||||
|
||||
/**
|
||||
* Fast Preset - For large imports (>10K rows)
|
||||
*
|
||||
* Optimized for speed over accuracy:
|
||||
* - Only exact match and pattern signals
|
||||
* - Only explicit strategy (O(1) lookups)
|
||||
* - Streaming enabled for memory efficiency
|
||||
* - Early termination on first high-confidence match
|
||||
*
|
||||
* Use case: Bulk imports, data migrations
|
||||
* Performance: ~10ms per row
|
||||
* Accuracy: ~85%
|
||||
*/
|
||||
export const FAST_PRESET: PresetConfig = {
|
||||
name: 'fast',
|
||||
description: 'Fast bulk import for large datasets',
|
||||
signals: {
|
||||
enabled: ['exact', 'pattern'],
|
||||
weights: {
|
||||
exact: 0.70,
|
||||
pattern: 0.30,
|
||||
embedding: 0,
|
||||
context: 0
|
||||
},
|
||||
timeout: 50
|
||||
},
|
||||
strategies: {
|
||||
enabled: ['explicit'],
|
||||
timeout: 100,
|
||||
earlyTermination: true,
|
||||
minConfidence: 0.70
|
||||
},
|
||||
streaming: true,
|
||||
batchSize: 1000
|
||||
}
|
||||
|
||||
/**
|
||||
* Balanced Preset - Default for most imports
|
||||
*
|
||||
* Good balance of speed and accuracy:
|
||||
* - All signals except context (embedding, exact, pattern)
|
||||
* - All strategies with smart ordering
|
||||
* - Moderate timeouts
|
||||
* - Early termination after high-confidence matches
|
||||
*
|
||||
* Use case: Standard imports, general glossaries
|
||||
* Performance: ~30ms per row
|
||||
* Accuracy: ~92%
|
||||
*/
|
||||
export const BALANCED_PRESET: PresetConfig = {
|
||||
name: 'balanced',
|
||||
description: 'Balanced speed and accuracy for most imports',
|
||||
signals: {
|
||||
enabled: ['exact', 'embedding', 'pattern'],
|
||||
weights: {
|
||||
exact: 0.40,
|
||||
embedding: 0.35,
|
||||
pattern: 0.25,
|
||||
context: 0
|
||||
},
|
||||
timeout: 100
|
||||
},
|
||||
strategies: {
|
||||
enabled: ['explicit', 'pattern', 'embedding'],
|
||||
timeout: 200,
|
||||
earlyTermination: true,
|
||||
minConfidence: 0.65
|
||||
},
|
||||
streaming: false,
|
||||
batchSize: 500
|
||||
}
|
||||
|
||||
/**
|
||||
* Accurate Preset - For small, critical imports
|
||||
*
|
||||
* Optimized for accuracy over speed:
|
||||
* - All signals including context
|
||||
* - All strategies, no early termination
|
||||
* - Longer timeouts for thorough analysis
|
||||
* - Lower confidence threshold (accept more matches)
|
||||
*
|
||||
* Use case: Knowledge bases, critical taxonomies
|
||||
* Performance: ~100ms per row
|
||||
* Accuracy: ~97%
|
||||
*/
|
||||
export const ACCURATE_PRESET: PresetConfig = {
|
||||
name: 'accurate',
|
||||
description: 'Maximum accuracy for critical imports',
|
||||
signals: {
|
||||
enabled: ['exact', 'embedding', 'pattern', 'context'],
|
||||
weights: {
|
||||
exact: 0.40,
|
||||
embedding: 0.35,
|
||||
pattern: 0.20,
|
||||
context: 0.05
|
||||
},
|
||||
timeout: 500
|
||||
},
|
||||
strategies: {
|
||||
enabled: ['explicit', 'pattern', 'embedding'],
|
||||
timeout: 1000,
|
||||
earlyTermination: false,
|
||||
minConfidence: 0.50
|
||||
},
|
||||
streaming: false,
|
||||
batchSize: 100
|
||||
}
|
||||
|
||||
/**
|
||||
* Explicit Preset - For glossaries with relationship columns
|
||||
*
|
||||
* Optimized for structured data with explicit relationships:
|
||||
* - Only exact match signals (no AI needed)
|
||||
* - Only explicit and pattern strategies
|
||||
* - Fast, deterministic results
|
||||
* - Perfect for Excel/CSV with "Related Terms" columns
|
||||
*
|
||||
* Use case: structured taxonomies and glossaries from spreadsheet sources
|
||||
* Performance: ~5ms per row
|
||||
* Accuracy: ~99% (high confidence)
|
||||
*/
|
||||
export const EXPLICIT_PRESET: PresetConfig = {
|
||||
name: 'explicit',
|
||||
description: 'For glossaries with explicit relationship columns',
|
||||
signals: {
|
||||
enabled: ['exact', 'pattern'],
|
||||
weights: {
|
||||
exact: 0.70,
|
||||
pattern: 0.30,
|
||||
embedding: 0,
|
||||
context: 0
|
||||
},
|
||||
timeout: 50
|
||||
},
|
||||
strategies: {
|
||||
enabled: ['explicit', 'pattern'],
|
||||
timeout: 100,
|
||||
earlyTermination: true,
|
||||
minConfidence: 0.80
|
||||
},
|
||||
streaming: false,
|
||||
batchSize: 500
|
||||
}
|
||||
|
||||
/**
|
||||
* Pattern Preset - For documents with narrative content
|
||||
*
|
||||
* Optimized for unstructured text with rich patterns:
|
||||
* - Embedding and pattern signals (semantic understanding)
|
||||
* - Pattern and embedding strategies
|
||||
* - Good for PDFs, articles, documentation
|
||||
*
|
||||
* Use case: PDF imports, markdown docs, articles
|
||||
* Performance: ~50ms per row
|
||||
* Accuracy: ~90%
|
||||
*/
|
||||
export const PATTERN_PRESET: PresetConfig = {
|
||||
name: 'pattern',
|
||||
description: 'For documents with narrative content',
|
||||
signals: {
|
||||
enabled: ['embedding', 'pattern', 'context'],
|
||||
weights: {
|
||||
embedding: 0.50,
|
||||
pattern: 0.40,
|
||||
context: 0.10,
|
||||
exact: 0
|
||||
},
|
||||
timeout: 200
|
||||
},
|
||||
strategies: {
|
||||
enabled: ['pattern', 'embedding'],
|
||||
timeout: 300,
|
||||
earlyTermination: false,
|
||||
minConfidence: 0.60
|
||||
},
|
||||
streaming: false,
|
||||
batchSize: 200
|
||||
}
|
||||
|
||||
/**
|
||||
* All available presets
|
||||
*/
|
||||
export const PRESETS: Record<string, PresetConfig> = {
|
||||
fast: FAST_PRESET,
|
||||
balanced: BALANCED_PRESET,
|
||||
accurate: ACCURATE_PRESET,
|
||||
explicit: EXPLICIT_PRESET,
|
||||
pattern: PATTERN_PRESET
|
||||
}
|
||||
|
||||
/**
|
||||
* Auto-detect optimal preset based on import context
|
||||
*
|
||||
* Decision tree:
|
||||
* 1. Large dataset (>10K rows or >10MB) → fast
|
||||
* 2. Small dataset (<100 rows) → accurate
|
||||
* 3. Excel/CSV with explicit columns → explicit
|
||||
* 4. PDF/Markdown with long content → pattern
|
||||
* 5. Default → balanced
|
||||
*
|
||||
* @param context Import context (file type, size, structure)
|
||||
* @returns Optimal preset configuration
|
||||
*/
|
||||
export function autoDetectPreset(context: ImportContext = {}): PresetConfig {
|
||||
const {
|
||||
fileType = 'unknown',
|
||||
fileSize = 0,
|
||||
rowCount = 0,
|
||||
hasExplicitColumns = false,
|
||||
hasNarrativeContent = false,
|
||||
avgDefinitionLength = 0
|
||||
} = context
|
||||
|
||||
// Rule 1: Large imports → fast preset (prioritize speed)
|
||||
if (rowCount > 10000 || fileSize > 10_000_000) {
|
||||
return FAST_PRESET
|
||||
}
|
||||
|
||||
// Rule 2: Small critical imports → accurate preset (prioritize accuracy)
|
||||
if (rowCount > 0 && rowCount < 100) {
|
||||
return ACCURATE_PRESET
|
||||
}
|
||||
|
||||
// Rule 3: Structured data with explicit relationships → explicit preset
|
||||
// (Handles spreadsheet imports where relationships are encoded in columns.)
|
||||
if (hasExplicitColumns && (fileType === 'excel' || fileType === 'csv')) {
|
||||
return EXPLICIT_PRESET
|
||||
}
|
||||
|
||||
// Rule 4: Narrative content → pattern preset
|
||||
// Good for PDFs, articles, documentation
|
||||
if (
|
||||
hasNarrativeContent ||
|
||||
fileType === 'pdf' ||
|
||||
fileType === 'markdown' ||
|
||||
avgDefinitionLength > 500
|
||||
) {
|
||||
return PATTERN_PRESET
|
||||
}
|
||||
|
||||
// Rule 5: JSON data → balanced preset
|
||||
if (fileType === 'json') {
|
||||
return BALANCED_PRESET
|
||||
}
|
||||
|
||||
// Default: balanced preset
|
||||
return BALANCED_PRESET
|
||||
}
|
||||
|
||||
/**
|
||||
* Get preset by name
|
||||
*
|
||||
* @param name Preset name (fast, balanced, accurate, explicit, pattern)
|
||||
* @returns Preset configuration
|
||||
* @throws Error if preset not found
|
||||
*/
|
||||
export function getPreset(name: string): PresetConfig {
|
||||
const preset = PRESETS[name.toLowerCase()]
|
||||
if (!preset) {
|
||||
throw new Error(`Unknown preset: ${name}. Available: ${Object.keys(PRESETS).join(', ')}`)
|
||||
}
|
||||
return preset
|
||||
}
|
||||
|
||||
/**
|
||||
* Get all available preset names
|
||||
*
|
||||
* @returns Array of preset names
|
||||
*/
|
||||
export function getPresetNames(): string[] {
|
||||
return Object.keys(PRESETS)
|
||||
}
|
||||
|
||||
/**
|
||||
* Explain why a preset was selected
|
||||
*
|
||||
* @param context Import context
|
||||
* @returns Human-readable explanation
|
||||
*/
|
||||
export function explainPresetChoice(context: ImportContext = {}): string {
|
||||
const {
|
||||
fileType = 'unknown',
|
||||
fileSize = 0,
|
||||
rowCount = 0,
|
||||
hasExplicitColumns = false,
|
||||
hasNarrativeContent = false,
|
||||
avgDefinitionLength = 0
|
||||
} = context
|
||||
|
||||
if (rowCount > 10000 || fileSize > 10_000_000) {
|
||||
return `Large dataset (${rowCount} rows, ${(fileSize / 1_000_000).toFixed(1)}MB) → fast preset for optimal performance`
|
||||
}
|
||||
|
||||
if (rowCount > 0 && rowCount < 100) {
|
||||
return `Small critical dataset (${rowCount} rows) → accurate preset for maximum accuracy`
|
||||
}
|
||||
|
||||
if (hasExplicitColumns && (fileType === 'excel' || fileType === 'csv')) {
|
||||
return `${fileType.toUpperCase()} with explicit relationship columns → explicit preset for deterministic results`
|
||||
}
|
||||
|
||||
if (hasNarrativeContent || fileType === 'pdf' || fileType === 'markdown') {
|
||||
return `Narrative content (${fileType}) → pattern preset for semantic understanding`
|
||||
}
|
||||
|
||||
if (fileType === 'json') {
|
||||
return `JSON data → balanced preset for structured imports`
|
||||
}
|
||||
|
||||
return `Standard import → balanced preset (default)`
|
||||
}
|
||||
|
||||
/**
|
||||
* Create custom preset by merging with base preset
|
||||
*
|
||||
* @param baseName Base preset name
|
||||
* @param overrides Custom overrides
|
||||
* @returns Custom preset configuration
|
||||
*/
|
||||
export function createCustomPreset(
|
||||
baseName: string,
|
||||
overrides: Partial<PresetConfig>
|
||||
): PresetConfig {
|
||||
const base = getPreset(baseName)
|
||||
|
||||
return {
|
||||
...base,
|
||||
...overrides,
|
||||
signals: {
|
||||
...base.signals,
|
||||
...(overrides.signals || {})
|
||||
},
|
||||
strategies: {
|
||||
...base.strategies,
|
||||
...(overrides.strategies || {})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Validate preset configuration
|
||||
*
|
||||
* @param preset Preset to validate
|
||||
* @returns True if valid, throws error otherwise
|
||||
*/
|
||||
export function validatePreset(preset: PresetConfig): boolean {
|
||||
// Validate signals
|
||||
if (preset.signals.enabled.length === 0) {
|
||||
throw new Error('Preset must have at least one enabled signal')
|
||||
}
|
||||
|
||||
// Validate strategies
|
||||
if (preset.strategies.enabled.length === 0) {
|
||||
throw new Error('Preset must have at least one enabled strategy')
|
||||
}
|
||||
|
||||
// Validate weights sum to ~1.0
|
||||
const enabledSignals = preset.signals.enabled
|
||||
const totalWeight = enabledSignals.reduce(
|
||||
(sum, signal) => sum + preset.signals.weights[signal],
|
||||
0
|
||||
)
|
||||
|
||||
if (Math.abs(totalWeight - 1.0) > 0.01) {
|
||||
throw new Error(
|
||||
`Signal weights must sum to 1.0, got ${totalWeight.toFixed(2)}`
|
||||
)
|
||||
}
|
||||
|
||||
// Validate timeouts
|
||||
if (preset.signals.timeout <= 0 || preset.strategies.timeout <= 0) {
|
||||
throw new Error('Timeouts must be positive')
|
||||
}
|
||||
|
||||
// Validate batch size
|
||||
if (preset.batchSize <= 0) {
|
||||
throw new Error('Batch size must be positive')
|
||||
}
|
||||
|
||||
return true
|
||||
}
|
||||
|
||||
/**
|
||||
* Format preset for display
|
||||
*
|
||||
* @param preset Preset configuration
|
||||
* @returns Human-readable preset summary
|
||||
*/
|
||||
export function formatPreset(preset: PresetConfig): string {
|
||||
const lines = [
|
||||
`Preset: ${preset.name}`,
|
||||
`Description: ${preset.description}`,
|
||||
'',
|
||||
'Signals:',
|
||||
...preset.signals.enabled.map(
|
||||
(s) => ` - ${s}: ${(preset.signals.weights[s] * 100).toFixed(0)}%`
|
||||
),
|
||||
` Timeout: ${preset.signals.timeout}ms`,
|
||||
'',
|
||||
'Strategies:',
|
||||
...preset.strategies.enabled.map((s) => ` - ${s}`),
|
||||
` Timeout: ${preset.strategies.timeout}ms`,
|
||||
` Early termination: ${preset.strategies.earlyTermination}`,
|
||||
` Min confidence: ${preset.strategies.minConfidence}`,
|
||||
'',
|
||||
`Streaming: ${preset.streaming}`,
|
||||
`Batch size: ${preset.batchSize}`
|
||||
]
|
||||
|
||||
return lines.join('\n')
|
||||
}
|
||||
|
|
@ -1,309 +0,0 @@
|
|||
/**
|
||||
* Relationship Confidence Scoring
|
||||
*
|
||||
* Scores the confidence of detected relationships based on multiple factors:
|
||||
* - Entity proximity in text
|
||||
* - Entity confidence scores
|
||||
* - Pattern matches
|
||||
* - Structural analysis
|
||||
*/
|
||||
|
||||
import { ExtractedEntity } from './entityExtractor.js'
|
||||
import { VerbType } from '../types/graphTypes.js'
|
||||
import { RelationEvidence } from '../types/brainy.types.js'
|
||||
|
||||
/**
|
||||
* Detected relationship with confidence
|
||||
*/
|
||||
export interface DetectedRelationship {
|
||||
sourceEntity: ExtractedEntity
|
||||
targetEntity: ExtractedEntity
|
||||
verbType: VerbType
|
||||
confidence: number
|
||||
evidence: RelationEvidence
|
||||
}
|
||||
|
||||
/**
|
||||
* Configuration for relationship detection
|
||||
*/
|
||||
export interface RelationshipDetectionConfig {
|
||||
minConfidence?: number // Minimum confidence to return (default: 0.5)
|
||||
maxDistance?: number // Maximum token distance between entities (default: 50)
|
||||
useProximityBoost?: boolean // Boost score based on proximity (default: true)
|
||||
usePatternMatching?: boolean // Use verb pattern matching (default: true)
|
||||
useStructuralAnalysis?: boolean // Analyze sentence structure (default: true)
|
||||
}
|
||||
|
||||
/**
|
||||
* Relationship confidence scorer
|
||||
*/
|
||||
export class RelationshipConfidenceScorer {
|
||||
private config: Required<RelationshipDetectionConfig>
|
||||
|
||||
constructor(config: RelationshipDetectionConfig = {}) {
|
||||
this.config = {
|
||||
minConfidence: config.minConfidence || 0.5,
|
||||
maxDistance: config.maxDistance || 50,
|
||||
useProximityBoost: config.useProximityBoost !== false,
|
||||
usePatternMatching: config.usePatternMatching !== false,
|
||||
useStructuralAnalysis: config.useStructuralAnalysis !== false
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Score a potential relationship between two entities
|
||||
*/
|
||||
scoreRelationship(
|
||||
source: ExtractedEntity,
|
||||
target: ExtractedEntity,
|
||||
verbType: VerbType,
|
||||
context: string
|
||||
): { confidence: number, evidence: RelationEvidence } {
|
||||
let confidence = 0.5 // Base confidence
|
||||
|
||||
// Evidence tracking
|
||||
const reasoningParts: string[] = []
|
||||
|
||||
// Factor 1: Proximity boost (closer entities = higher confidence)
|
||||
if (this.config.useProximityBoost) {
|
||||
const proximityBoost = this.calculateProximityBoost(source, target)
|
||||
confidence += proximityBoost
|
||||
if (proximityBoost > 0) {
|
||||
reasoningParts.push(
|
||||
`Entities are close together (boost: +${proximityBoost.toFixed(2)})`
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
// Factor 2: Entity confidence boost
|
||||
const entityConfidence = (source.confidence + target.confidence) / 2
|
||||
const entityBoost = (entityConfidence - 0.5) * 0.2 // Scale to 0-0.2
|
||||
confidence *= (1 + entityBoost)
|
||||
if (entityBoost > 0) {
|
||||
reasoningParts.push(
|
||||
`High entity confidence (boost: ${entityBoost.toFixed(2)})`
|
||||
)
|
||||
}
|
||||
|
||||
// Factor 3: Pattern match boost
|
||||
if (this.config.usePatternMatching) {
|
||||
const patternBoost = this.checkVerbPattern(source, target, verbType, context)
|
||||
confidence += patternBoost
|
||||
if (patternBoost > 0) {
|
||||
reasoningParts.push(
|
||||
`Matches relationship pattern (boost: +${patternBoost.toFixed(2)})`
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
// Factor 4: Structural boost (same sentence, clause, etc.)
|
||||
if (this.config.useStructuralAnalysis) {
|
||||
const structuralBoost = this.analyzeStructure(source, target, context)
|
||||
confidence += structuralBoost
|
||||
if (structuralBoost > 0) {
|
||||
reasoningParts.push(
|
||||
`Structural relationship (boost: +${structuralBoost.toFixed(2)})`
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
// Cap confidence at 1.0
|
||||
confidence = Math.min(confidence, 1.0)
|
||||
|
||||
// Extract source text evidence
|
||||
const start = Math.min(source.position.start, target.position.start)
|
||||
const end = Math.max(source.position.end, target.position.end)
|
||||
|
||||
const evidence: RelationEvidence = {
|
||||
sourceText: context.substring(start, end),
|
||||
position: { start, end },
|
||||
method: 'neural',
|
||||
reasoning: reasoningParts.join('; ')
|
||||
}
|
||||
|
||||
return { confidence, evidence }
|
||||
}
|
||||
|
||||
/**
|
||||
* Calculate proximity boost based on distance between entities
|
||||
*/
|
||||
private calculateProximityBoost(
|
||||
source: ExtractedEntity,
|
||||
target: ExtractedEntity
|
||||
): number {
|
||||
const distance = Math.abs(source.position.start - target.position.start)
|
||||
|
||||
if (distance === 0) return 0 // Same position, not meaningful
|
||||
|
||||
// Very close (< 20 chars): +0.2
|
||||
if (distance < 20) return 0.2
|
||||
|
||||
// Close (< 50 chars): +0.1
|
||||
if (distance < 50) return 0.1
|
||||
|
||||
// Medium (< 100 chars): +0.05
|
||||
if (distance < 100) return 0.05
|
||||
|
||||
// Far (> 100 chars): no boost
|
||||
return 0
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if entities match a verb pattern
|
||||
*/
|
||||
private checkVerbPattern(
|
||||
source: ExtractedEntity,
|
||||
target: ExtractedEntity,
|
||||
verbType: VerbType,
|
||||
context: string
|
||||
): number {
|
||||
const contextBetween = this.getContextBetween(source, target, context)
|
||||
const contextLower = contextBetween.toLowerCase()
|
||||
|
||||
// Verb-specific patterns
|
||||
const patterns: Record<string, string[]> = {
|
||||
[VerbType.Creates]: ['creates', 'made', 'built', 'developed', 'produces'],
|
||||
[VerbType.Owns]: ['owns', 'belongs to', 'possessed by', 'has'],
|
||||
[VerbType.Contains]: ['contains', 'includes', 'has', 'holds'],
|
||||
[VerbType.Requires]: ['requires', 'needs', 'depends on', 'relies on'],
|
||||
[VerbType.Uses]: ['uses', 'utilizes', 'employs', 'applies'],
|
||||
[VerbType.ReportsTo]: ['manages', 'oversees', 'supervises', 'controls'],
|
||||
[VerbType.Causes]: ['influences', 'affects', 'impacts', 'shapes', 'causes'],
|
||||
[VerbType.DependsOn]: ['depends on', 'relies on', 'based on'],
|
||||
[VerbType.Modifies]: ['modifies', 'changes', 'alters', 'updates'],
|
||||
[VerbType.References]: ['references', 'cites', 'mentions', 'refers to']
|
||||
}
|
||||
|
||||
const verbPatterns = patterns[verbType] || []
|
||||
|
||||
for (const pattern of verbPatterns) {
|
||||
if (contextLower.includes(pattern)) {
|
||||
return 0.2 // Strong pattern match
|
||||
}
|
||||
}
|
||||
|
||||
return 0 // No pattern match
|
||||
}
|
||||
|
||||
/**
|
||||
* Analyze structural relationship
|
||||
*/
|
||||
private analyzeStructure(
|
||||
source: ExtractedEntity,
|
||||
target: ExtractedEntity,
|
||||
context: string
|
||||
): number {
|
||||
const contextBetween = this.getContextBetween(source, target, context)
|
||||
|
||||
// Same sentence (no sentence-ending punctuation between them)
|
||||
if (!contextBetween.match(/[.!?]/)) {
|
||||
return 0.1
|
||||
}
|
||||
|
||||
// Same paragraph (single newline between them)
|
||||
if (!contextBetween.match(/\n\n/)) {
|
||||
return 0.05
|
||||
}
|
||||
|
||||
return 0
|
||||
}
|
||||
|
||||
/**
|
||||
* Get context text between two entities
|
||||
*/
|
||||
private getContextBetween(
|
||||
source: ExtractedEntity,
|
||||
target: ExtractedEntity,
|
||||
context: string
|
||||
): string {
|
||||
const start = Math.min(source.position.end, target.position.end)
|
||||
const end = Math.max(source.position.start, target.position.start)
|
||||
|
||||
if (start >= end) return ''
|
||||
|
||||
return context.substring(start, end)
|
||||
}
|
||||
|
||||
/**
|
||||
* Detect relationships between a list of entities
|
||||
*/
|
||||
detectRelationships(
|
||||
entities: ExtractedEntity[],
|
||||
context: string,
|
||||
verbHints?: VerbType[]
|
||||
): DetectedRelationship[] {
|
||||
const relationships: DetectedRelationship[] = []
|
||||
const verbs = verbHints || [
|
||||
VerbType.Creates,
|
||||
VerbType.Uses,
|
||||
VerbType.Contains,
|
||||
VerbType.Requires,
|
||||
VerbType.RelatedTo
|
||||
]
|
||||
|
||||
// Check all entity pairs
|
||||
for (let i = 0; i < entities.length; i++) {
|
||||
for (let j = i + 1; j < entities.length; j++) {
|
||||
const source = entities[i]
|
||||
const target = entities[j]
|
||||
|
||||
// Check distance
|
||||
const distance = Math.abs(source.position.start - target.position.start)
|
||||
if (distance > this.config.maxDistance) {
|
||||
continue // Too far apart
|
||||
}
|
||||
|
||||
// Try each verb type
|
||||
for (const verbType of verbs) {
|
||||
const { confidence, evidence } = this.scoreRelationship(
|
||||
source,
|
||||
target,
|
||||
verbType,
|
||||
context
|
||||
)
|
||||
|
||||
if (confidence >= this.config.minConfidence) {
|
||||
relationships.push({
|
||||
sourceEntity: source,
|
||||
targetEntity: target,
|
||||
verbType,
|
||||
confidence,
|
||||
evidence
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Sort by confidence (highest first)
|
||||
relationships.sort((a, b) => b.confidence - a.confidence)
|
||||
|
||||
return relationships
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Convenience function to score a single relationship
|
||||
*/
|
||||
export function scoreRelationshipConfidence(
|
||||
source: ExtractedEntity,
|
||||
target: ExtractedEntity,
|
||||
verbType: VerbType,
|
||||
context: string,
|
||||
config?: RelationshipDetectionConfig
|
||||
): { confidence: number, evidence: RelationEvidence } {
|
||||
const scorer = new RelationshipConfidenceScorer(config)
|
||||
return scorer.scoreRelationship(source, target, verbType, context)
|
||||
}
|
||||
|
||||
/**
|
||||
* Convenience function to detect all relationships in text
|
||||
*/
|
||||
export function detectRelationshipsWithConfidence(
|
||||
entities: ExtractedEntity[],
|
||||
context: string,
|
||||
config?: RelationshipDetectionConfig
|
||||
): DetectedRelationship[] {
|
||||
const scorer = new RelationshipConfidenceScorer(config)
|
||||
return scorer.detectRelationships(entities, context)
|
||||
}
|
||||
|
|
@ -1,188 +0,0 @@
|
|||
/**
|
||||
* Static Pattern Matcher - NO runtime initialization, NO Brainy needed
|
||||
*
|
||||
* All patterns and embeddings are pre-computed at build time
|
||||
* This is pure pattern matching with zero dependencies
|
||||
*/
|
||||
|
||||
import { EMBEDDED_PATTERNS, getPatternEmbeddings } from './embeddedPatterns.js'
|
||||
import type { Vector } from '../coreTypes.js'
|
||||
import type { TripleQuery } from '../triple/TripleIntelligence.js'
|
||||
|
||||
// Pre-load patterns and embeddings at module load time (happens once)
|
||||
const patterns = new Map(EMBEDDED_PATTERNS.map(p => [p.id, p]))
|
||||
const patternEmbeddings = getPatternEmbeddings()
|
||||
|
||||
/**
|
||||
* Cosine similarity between two vectors.
|
||||
* Accepts any number-indexed sequence so pre-computed Float32Array pattern
|
||||
* embeddings can be compared against number[] query vectors without copying.
|
||||
*/
|
||||
function cosineSimilarity(a: ArrayLike<number>, b: ArrayLike<number>): number {
|
||||
if (!a || !b || a.length !== b.length) return 0
|
||||
|
||||
let dotProduct = 0
|
||||
let normA = 0
|
||||
let normB = 0
|
||||
|
||||
for (let i = 0; i < a.length; i++) {
|
||||
dotProduct += a[i] * b[i]
|
||||
normA += a[i] * a[i]
|
||||
normB += b[i] * b[i]
|
||||
}
|
||||
|
||||
const denominator = Math.sqrt(normA) * Math.sqrt(normB)
|
||||
return denominator === 0 ? 0 : dotProduct / denominator
|
||||
}
|
||||
|
||||
/**
|
||||
* Extract slots from matched pattern
|
||||
*/
|
||||
function extractSlots(query: string, pattern: string): Record<string, string> | null {
|
||||
try {
|
||||
const regex = new RegExp(pattern, 'i')
|
||||
const match = query.match(regex)
|
||||
|
||||
if (!match) return null
|
||||
|
||||
const slots: Record<string, string> = {}
|
||||
for (let i = 1; i < match.length; i++) {
|
||||
if (match[i]) {
|
||||
slots[`$${i}`] = match[i]
|
||||
}
|
||||
}
|
||||
|
||||
return Object.keys(slots).length > 0 ? slots : null
|
||||
} catch {
|
||||
return null
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Apply template with extracted slots
|
||||
*/
|
||||
function applyTemplate(template: any, slots: Record<string, string>): any {
|
||||
if (!template || !slots) return template
|
||||
|
||||
const result = JSON.parse(JSON.stringify(template))
|
||||
const applySlots = (obj: any): any => {
|
||||
if (typeof obj === 'string') {
|
||||
return obj.replace(/\$\{(\d+)\}/g, (_, num) => slots[`$${num}`] || '')
|
||||
}
|
||||
if (Array.isArray(obj)) {
|
||||
return obj.map(applySlots)
|
||||
}
|
||||
if (typeof obj === 'object' && obj !== null) {
|
||||
const newObj: any = {}
|
||||
for (const [key, value] of Object.entries(obj)) {
|
||||
newObj[key] = applySlots(value)
|
||||
}
|
||||
return newObj
|
||||
}
|
||||
return obj
|
||||
}
|
||||
|
||||
return applySlots(result)
|
||||
}
|
||||
|
||||
/**
|
||||
* Match query against all patterns using embeddings
|
||||
*/
|
||||
export function findBestPatterns(
|
||||
queryEmbedding: Vector,
|
||||
k: number = 3
|
||||
): Array<{ pattern: typeof EMBEDDED_PATTERNS[0]; similarity: number }> {
|
||||
|
||||
const matches: Array<{ pattern: typeof EMBEDDED_PATTERNS[0]; similarity: number }> = []
|
||||
|
||||
for (const pattern of EMBEDDED_PATTERNS) {
|
||||
|
||||
const patternEmbedding = patternEmbeddings.get(pattern.id)
|
||||
if (!patternEmbedding) continue
|
||||
|
||||
// Pass Float32Array directly, no need for Array.from()!
|
||||
const similarity = cosineSimilarity(queryEmbedding, patternEmbedding)
|
||||
if (similarity > 0.5) { // Threshold for relevance
|
||||
matches.push({ pattern, similarity })
|
||||
}
|
||||
}
|
||||
|
||||
// Sort by similarity and return top k
|
||||
return matches
|
||||
.sort((a, b) => b.similarity - a.similarity)
|
||||
.slice(0, k)
|
||||
}
|
||||
|
||||
/**
|
||||
* Match query against patterns using regex
|
||||
*/
|
||||
export function matchPatternByRegex(query: string): {
|
||||
pattern: typeof EMBEDDED_PATTERNS[0]
|
||||
slots: Record<string, string>
|
||||
query: TripleQuery
|
||||
} | null {
|
||||
// Try direct regex matching first (fastest)
|
||||
for (const pattern of EMBEDDED_PATTERNS) {
|
||||
const slots = extractSlots(query, pattern.pattern)
|
||||
if (slots) {
|
||||
const templatedQuery = applyTemplate(pattern.template, slots)
|
||||
return {
|
||||
pattern,
|
||||
slots,
|
||||
query: templatedQuery
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return null
|
||||
}
|
||||
|
||||
/**
|
||||
* Convert natural language to structured query using STATIC patterns
|
||||
* NO initialization needed, NO Brainy required
|
||||
*/
|
||||
export function patternMatchQuery(
|
||||
query: string,
|
||||
queryEmbedding?: Vector
|
||||
): TripleQuery {
|
||||
|
||||
// ALWAYS use vector similarity when we have embeddings (which we always do!)
|
||||
if (queryEmbedding && queryEmbedding.length === 384) {
|
||||
const bestPatterns = findBestPatterns(queryEmbedding, 5) // Get top 5 matches
|
||||
|
||||
// Try to extract slots from best matching patterns
|
||||
for (const { pattern, similarity } of bestPatterns) {
|
||||
// Only try patterns with good similarity
|
||||
if (similarity < 0.7) break
|
||||
|
||||
const slots = extractSlots(query, pattern.pattern)
|
||||
if (slots) {
|
||||
// Found a good match with extractable slots!
|
||||
const result = applyTemplate(pattern.template, slots)
|
||||
console.log('[NLP] Applied template with slots:', JSON.stringify(result))
|
||||
return result
|
||||
}
|
||||
}
|
||||
|
||||
// If no slots extracted but we have a good match, use the template as-is
|
||||
if (bestPatterns.length > 0 && bestPatterns[0].similarity > 0.75) {
|
||||
console.log('[NLP] Returning template as-is:', JSON.stringify(bestPatterns[0].pattern.template))
|
||||
return bestPatterns[0].pattern.template
|
||||
}
|
||||
}
|
||||
|
||||
// Fallback: simple vector search (should rarely happen)
|
||||
console.log('[NLP] Fallback - returning simple query')
|
||||
return {
|
||||
like: query,
|
||||
limit: 10
|
||||
}
|
||||
}
|
||||
|
||||
// Export pattern statistics for monitoring
|
||||
export const PATTERN_STATS = {
|
||||
totalPatterns: EMBEDDED_PATTERNS.length,
|
||||
categories: [...new Set(EMBEDDED_PATTERNS.map(p => p.category))],
|
||||
domains: [...new Set(EMBEDDED_PATTERNS.filter(p => p.domain).map(p => p.domain!))],
|
||||
hasEmbeddings: patternEmbeddings.size > 0
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue