refactor: remove src/cortex/ directory and fix README claims
- Move neuralImport.ts and neuralImportAugmentation.ts to src/neural/ - Delete 3 dead files (healthCheck, backupRestore, performanceMonitor) - Remove cortex entries from package.json browser field - Fix unsubstantiated performance claims in README (add PROJECTED labels) - Delete stale dist/cortex/ artifacts
This commit is contained in:
parent
0292cf2ddf
commit
36db644eca
9 changed files with 16 additions and 1702 deletions
837
src/neural/neuralImport.ts
Normal file
837
src/neural/neuralImport.ts
Normal file
|
|
@ -0,0 +1,837 @@
|
|||
/**
|
||||
* Neural Import - Atomic Age AI-Powered Data Understanding System
|
||||
*
|
||||
* 🧠 Leveraging the brain-in-jar to understand and automatically structure data
|
||||
* ⚛️ Complete with confidence scoring and relationship weight calculation
|
||||
*/
|
||||
|
||||
import { Brainy } from '../brainy.js'
|
||||
import { NounType, VerbType } from '../types/graphTypes.js'
|
||||
import * as fs from '../universal/fs.js'
|
||||
import * as path from '../universal/path.js'
|
||||
// @ts-ignore
|
||||
import chalk from 'chalk'
|
||||
// @ts-ignore
|
||||
import ora from 'ora'
|
||||
// @ts-ignore
|
||||
import boxen from 'boxen'
|
||||
// @ts-ignore
|
||||
import Table from 'cli-table3'
|
||||
// @ts-ignore
|
||||
import prompts from 'prompts'
|
||||
|
||||
// Neural Import Types
|
||||
export interface NeuralAnalysisResult {
|
||||
detectedEntities: DetectedEntity[]
|
||||
detectedRelationships: DetectedRelationship[]
|
||||
confidence: number
|
||||
insights: NeuralInsight[]
|
||||
preview: ProcessedData[]
|
||||
}
|
||||
|
||||
export interface DetectedEntity {
|
||||
originalData: any
|
||||
nounType: string
|
||||
confidence: number
|
||||
suggestedId: string
|
||||
reasoning: string
|
||||
alternativeTypes: Array<{ type: string, confidence: number }>
|
||||
}
|
||||
|
||||
export interface DetectedRelationship {
|
||||
sourceId: string
|
||||
targetId: string
|
||||
verbType: string
|
||||
confidence: number
|
||||
weight: number
|
||||
reasoning: string
|
||||
context: string
|
||||
metadata?: Record<string, any>
|
||||
}
|
||||
|
||||
export interface NeuralInsight {
|
||||
type: 'hierarchy' | 'cluster' | 'pattern' | 'anomaly' | 'opportunity'
|
||||
description: string
|
||||
confidence: number
|
||||
affectedEntities: string[]
|
||||
recommendation?: string
|
||||
}
|
||||
|
||||
export interface ProcessedData {
|
||||
id: string
|
||||
nounType: string
|
||||
data: any
|
||||
relationships: Array<{
|
||||
target: string
|
||||
verbType: string
|
||||
weight: number
|
||||
confidence: number
|
||||
}>
|
||||
}
|
||||
|
||||
export interface NeuralImportOptions {
|
||||
confidenceThreshold: number
|
||||
autoApply: boolean
|
||||
enableWeights: boolean
|
||||
previewOnly: boolean
|
||||
validateOnly: boolean
|
||||
categoryFilter?: string[]
|
||||
skipDuplicates: boolean
|
||||
}
|
||||
|
||||
/**
|
||||
* Neural Import Engine - The Brain Behind the Analysis
|
||||
*/
|
||||
export class NeuralImport {
|
||||
private brainy: Brainy
|
||||
private colors = {
|
||||
primary: chalk.hex('#3A5F4A'),
|
||||
success: chalk.hex('#2D4A3A'),
|
||||
warning: chalk.hex('#D67441'),
|
||||
error: chalk.hex('#B85C35'),
|
||||
info: chalk.hex('#4A6B5A'),
|
||||
dim: chalk.hex('#8A9B8A'),
|
||||
highlight: chalk.hex('#E88B5A'),
|
||||
accent: chalk.hex('#F5E6D3'),
|
||||
brain: chalk.hex('#E88B5A')
|
||||
}
|
||||
|
||||
private emojis = {
|
||||
brain: '🧠',
|
||||
atom: '⚛️',
|
||||
lab: '🔬',
|
||||
data: '🎛️',
|
||||
magic: '⚡',
|
||||
check: '✅',
|
||||
warning: '⚠️',
|
||||
sparkle: '✨',
|
||||
rocket: '🚀',
|
||||
gear: '⚙️'
|
||||
}
|
||||
|
||||
constructor(brainy: Brainy) {
|
||||
this.brainy = brainy
|
||||
}
|
||||
|
||||
/**
|
||||
* Main Neural Import Function - The Master Controller
|
||||
*/
|
||||
async neuralImport(filePath: string, options: Partial<NeuralImportOptions> = {}): Promise<NeuralAnalysisResult> {
|
||||
const opts: NeuralImportOptions = {
|
||||
confidenceThreshold: 0.7,
|
||||
autoApply: false,
|
||||
enableWeights: true,
|
||||
previewOnly: false,
|
||||
validateOnly: false,
|
||||
skipDuplicates: true,
|
||||
...options
|
||||
}
|
||||
|
||||
console.log(boxen(
|
||||
`${this.emojis.brain} ${this.colors.brain('NEURAL IMPORT INITIATED')} ${this.emojis.atom}\n\n` +
|
||||
`${this.colors.accent('◆')} ${this.colors.dim('Activating atomic age AI analysis')}\n` +
|
||||
`${this.colors.accent('◆')} ${this.colors.dim('File:')} ${this.colors.highlight(filePath)}\n` +
|
||||
`${this.colors.accent('◆')} ${this.colors.dim('Confidence Threshold:')} ${this.colors.highlight(opts.confidenceThreshold.toString())}`,
|
||||
{ padding: 1, borderStyle: 'round', borderColor: '#E88B5A' }
|
||||
))
|
||||
|
||||
const spinner = ora(`${this.emojis.brain} Initializing neural analysis...`).start()
|
||||
|
||||
try {
|
||||
// Phase 1: Data Parsing
|
||||
spinner.text = `${this.emojis.lab} Parsing data structure...`
|
||||
const rawData = await this.parseFile(filePath)
|
||||
|
||||
// Phase 2: Neural Entity Detection
|
||||
spinner.text = `${this.emojis.atom} Analyzing ${Object.keys(NounType).length} entity types...`
|
||||
const detectedEntities = await this.detectEntitiesWithNeuralAnalysis(rawData, opts)
|
||||
|
||||
// Phase 3: Neural Relationship Detection
|
||||
spinner.text = `${this.emojis.data} Testing ${Object.keys(VerbType).length} relationship patterns...`
|
||||
const detectedRelationships = await this.detectRelationshipsWithNeuralAnalysis(detectedEntities, rawData, opts)
|
||||
|
||||
// Phase 4: Neural Insights Generation
|
||||
spinner.text = `${this.emojis.magic} Computing neural insights...`
|
||||
const insights = await this.generateNeuralInsights(detectedEntities, detectedRelationships)
|
||||
|
||||
// Phase 5: Confidence Scoring
|
||||
const overallConfidence = this.calculateOverallConfidence(detectedEntities, detectedRelationships)
|
||||
|
||||
spinner.stop()
|
||||
|
||||
const result: NeuralAnalysisResult = {
|
||||
detectedEntities,
|
||||
detectedRelationships,
|
||||
confidence: overallConfidence,
|
||||
insights,
|
||||
preview: await this.generatePreview(detectedEntities, detectedRelationships)
|
||||
}
|
||||
|
||||
// Display results
|
||||
await this.displayNeuralAnalysisResults(result, opts)
|
||||
|
||||
// Handle execution based on options
|
||||
if (opts.previewOnly || opts.validateOnly) {
|
||||
return result
|
||||
}
|
||||
|
||||
if (!opts.autoApply) {
|
||||
const shouldExecute = await this.confirmNeuralImport(result)
|
||||
if (!shouldExecute) {
|
||||
console.log(this.colors.dim('Neural import cancelled'))
|
||||
return result
|
||||
}
|
||||
}
|
||||
|
||||
// Execute the import
|
||||
await this.executeNeuralImport(result, opts)
|
||||
|
||||
return result
|
||||
|
||||
} catch (error) {
|
||||
spinner.fail('Neural analysis failed')
|
||||
throw error
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse file based on extension
|
||||
*/
|
||||
private async parseFile(filePath: string): Promise<any[]> {
|
||||
const ext = path.extname(filePath).toLowerCase()
|
||||
const content = await fs.readFile(filePath, 'utf8')
|
||||
|
||||
switch (ext) {
|
||||
case '.json':
|
||||
const jsonData = JSON.parse(content)
|
||||
return Array.isArray(jsonData) ? jsonData : [jsonData]
|
||||
|
||||
case '.csv':
|
||||
return this.parseCSV(content)
|
||||
|
||||
case '.yaml':
|
||||
case '.yml':
|
||||
// For now, basic YAML support - in full implementation would use yaml parser
|
||||
return JSON.parse(content) // Placeholder
|
||||
|
||||
default:
|
||||
throw new Error(`Unsupported file format: ${ext}`)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Basic CSV parser
|
||||
*/
|
||||
private parseCSV(content: string): any[] {
|
||||
const lines = content.split('\n').filter(line => line.trim())
|
||||
if (lines.length < 2) return []
|
||||
|
||||
const headers = lines[0].split(',').map(h => h.trim().replace(/"/g, ''))
|
||||
const data: any[] = []
|
||||
|
||||
for (let i = 1; i < lines.length; i++) {
|
||||
const values = lines[i].split(',').map(v => v.trim().replace(/"/g, ''))
|
||||
const row: any = {}
|
||||
|
||||
headers.forEach((header, index) => {
|
||||
row[header] = values[index] || ''
|
||||
})
|
||||
|
||||
data.push(row)
|
||||
}
|
||||
|
||||
return data
|
||||
}
|
||||
|
||||
/**
|
||||
* Neural Entity Detection - The Core AI Engine
|
||||
*/
|
||||
private async detectEntitiesWithNeuralAnalysis(rawData: any[], options: NeuralImportOptions): Promise<DetectedEntity[]> {
|
||||
const entities: DetectedEntity[] = []
|
||||
const nounTypes = Object.values(NounType)
|
||||
|
||||
for (const [index, dataItem] of rawData.entries()) {
|
||||
const mainText = this.extractMainText(dataItem)
|
||||
const detections: Array<{ type: string, confidence: number, reasoning: string }> = []
|
||||
|
||||
// Test against all noun types using semantic similarity
|
||||
for (const nounType of nounTypes) {
|
||||
const confidence = await this.calculateEntityTypeConfidence(mainText, dataItem, nounType)
|
||||
if (confidence >= options.confidenceThreshold - 0.2) { // Allow slightly lower for alternatives
|
||||
const reasoning = await this.generateEntityReasoning(mainText, dataItem, nounType)
|
||||
detections.push({ type: nounType, confidence, reasoning })
|
||||
}
|
||||
}
|
||||
|
||||
if (detections.length > 0) {
|
||||
// Sort by confidence
|
||||
detections.sort((a, b) => b.confidence - a.confidence)
|
||||
const primaryType = detections[0]
|
||||
const alternatives = detections.slice(1, 3) // Top 2 alternatives
|
||||
|
||||
entities.push({
|
||||
originalData: dataItem,
|
||||
nounType: primaryType.type,
|
||||
confidence: primaryType.confidence,
|
||||
suggestedId: this.generateSmartId(dataItem, primaryType.type, index),
|
||||
reasoning: primaryType.reasoning,
|
||||
alternativeTypes: alternatives
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
return entities
|
||||
}
|
||||
|
||||
/**
|
||||
* Calculate entity type confidence using AI
|
||||
*/
|
||||
private async calculateEntityTypeConfidence(text: string, data: any, nounType: string): Promise<number> {
|
||||
// Base semantic similarity using search instead of similarity method
|
||||
const searchResults = await this.brainy.find(text + ' ' + nounType)
|
||||
const textSimilarity = searchResults.length > 0 ? searchResults[0].score : 0.5
|
||||
|
||||
// Field-based confidence boost
|
||||
const fieldBoost = this.calculateFieldBasedConfidence(data, nounType)
|
||||
|
||||
// Pattern-based confidence boost
|
||||
const patternBoost = this.calculatePatternBasedConfidence(text, data, nounType)
|
||||
|
||||
// Combine confidences with weights
|
||||
const combined = (textSimilarity * 0.5) + (fieldBoost * 0.3) + (patternBoost * 0.2)
|
||||
|
||||
return Math.min(combined, 1.0)
|
||||
}
|
||||
|
||||
/**
|
||||
* Field-based confidence calculation
|
||||
*/
|
||||
private calculateFieldBasedConfidence(data: any, nounType: string): number {
|
||||
const fields = Object.keys(data)
|
||||
let boost = 0
|
||||
|
||||
// Field patterns that boost confidence for specific noun types
|
||||
const fieldPatterns: Record<string, string[]> = {
|
||||
[NounType.Person]: ['name', 'email', 'phone', 'age', 'firstname', 'lastname', 'employee'],
|
||||
[NounType.Organization]: ['company', 'organization', 'corp', 'inc', 'ltd', 'department', 'team'],
|
||||
[NounType.Project]: ['project', 'task', 'deadline', 'status', 'milestone', 'deliverable'],
|
||||
[NounType.Location]: ['address', 'city', 'country', 'state', 'zip', 'location', 'coordinates'],
|
||||
[NounType.Product]: ['product', 'price', 'sku', 'inventory', 'category', 'brand'],
|
||||
[NounType.Event]: ['date', 'time', 'venue', 'event', 'meeting', 'conference', 'schedule']
|
||||
}
|
||||
|
||||
const relevantPatterns = fieldPatterns[nounType] || []
|
||||
for (const field of fields) {
|
||||
for (const pattern of relevantPatterns) {
|
||||
if (field.toLowerCase().includes(pattern)) {
|
||||
boost += 0.1
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return Math.min(boost, 0.5)
|
||||
}
|
||||
|
||||
/**
|
||||
* Pattern-based confidence calculation
|
||||
*/
|
||||
private calculatePatternBasedConfidence(text: string, data: any, nounType: string): number {
|
||||
let boost = 0
|
||||
|
||||
// Content patterns that indicate entity types
|
||||
const patterns: Record<string, RegExp[]> = {
|
||||
[NounType.Person]: [
|
||||
/@.*\.com/i, // Email pattern
|
||||
/\b[A-Z][a-z]+ [A-Z][a-z]+\b/, // Name pattern
|
||||
/Mr\.|Mrs\.|Dr\.|Prof\./i // Title pattern
|
||||
],
|
||||
[NounType.Organization]: [
|
||||
/\bInc\.|Corp\.|LLC\.|Ltd\./i, // Corporate suffixes
|
||||
/Company|Corporation|Enterprise/i
|
||||
],
|
||||
[NounType.Location]: [
|
||||
/\b\d{5}(-\d{4})?\b/, // ZIP code
|
||||
/Street|Ave|Road|Blvd/i
|
||||
]
|
||||
}
|
||||
|
||||
const relevantPatterns = patterns[nounType] || []
|
||||
for (const pattern of relevantPatterns) {
|
||||
if (pattern.test(text)) {
|
||||
boost += 0.15
|
||||
}
|
||||
}
|
||||
|
||||
return Math.min(boost, 0.3)
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate reasoning for entity type selection
|
||||
*/
|
||||
private async generateEntityReasoning(text: string, data: any, nounType: string): Promise<string> {
|
||||
const reasons: string[] = []
|
||||
|
||||
// Semantic similarity reason using search
|
||||
const searchResults = await this.brainy.find(text + ' ' + nounType)
|
||||
const similarity = searchResults.length > 0 ? searchResults[0].score : 0.5
|
||||
if (similarity > 0.7) {
|
||||
reasons.push(`High semantic similarity (${(similarity * 100).toFixed(1)}%)`)
|
||||
}
|
||||
|
||||
// Field-based reasons
|
||||
const relevantFields = this.getRelevantFields(data, nounType)
|
||||
if (relevantFields.length > 0) {
|
||||
reasons.push(`Contains ${nounType}-specific fields: ${relevantFields.join(', ')}`)
|
||||
}
|
||||
|
||||
// Pattern-based reasons
|
||||
const matchedPatterns = this.getMatchedPatterns(text, data, nounType)
|
||||
if (matchedPatterns.length > 0) {
|
||||
reasons.push(`Matches ${nounType} patterns: ${matchedPatterns.join(', ')}`)
|
||||
}
|
||||
|
||||
return reasons.length > 0 ? reasons.join('; ') : 'General semantic match'
|
||||
}
|
||||
|
||||
/**
|
||||
* Neural Relationship Detection
|
||||
*/
|
||||
private async detectRelationshipsWithNeuralAnalysis(
|
||||
entities: DetectedEntity[],
|
||||
rawData: any[],
|
||||
options: NeuralImportOptions
|
||||
): Promise<DetectedRelationship[]> {
|
||||
const relationships: DetectedRelationship[] = []
|
||||
const verbTypes = Object.values(VerbType)
|
||||
|
||||
// For each pair of entities, test relationship possibilities
|
||||
for (let i = 0; i < entities.length; i++) {
|
||||
for (let j = i + 1; j < entities.length; j++) {
|
||||
const sourceEntity = entities[i]
|
||||
const targetEntity = entities[j]
|
||||
|
||||
// Extract context for relationship detection
|
||||
const context = this.extractRelationshipContext(sourceEntity.originalData, targetEntity.originalData, rawData)
|
||||
|
||||
// Test all verb types
|
||||
for (const verbType of verbTypes) {
|
||||
const confidence = await this.calculateRelationshipConfidence(
|
||||
sourceEntity, targetEntity, verbType, context
|
||||
)
|
||||
|
||||
if (confidence >= options.confidenceThreshold - 0.1) { // Slightly lower threshold for relationships
|
||||
const weight = options.enableWeights ?
|
||||
this.calculateRelationshipWeight(sourceEntity, targetEntity, verbType, context) :
|
||||
0.5
|
||||
|
||||
const reasoning = await this.generateRelationshipReasoning(sourceEntity, targetEntity, verbType, context)
|
||||
|
||||
relationships.push({
|
||||
sourceId: sourceEntity.suggestedId,
|
||||
targetId: targetEntity.suggestedId,
|
||||
verbType,
|
||||
confidence,
|
||||
weight,
|
||||
reasoning,
|
||||
context,
|
||||
metadata: this.extractRelationshipMetadata(sourceEntity.originalData, targetEntity.originalData, verbType)
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Sort by confidence and remove duplicates/conflicts
|
||||
return this.pruneRelationships(relationships)
|
||||
}
|
||||
|
||||
/**
|
||||
* Calculate relationship confidence
|
||||
*/
|
||||
private async calculateRelationshipConfidence(
|
||||
source: DetectedEntity,
|
||||
target: DetectedEntity,
|
||||
verbType: string,
|
||||
context: string
|
||||
): Promise<number> {
|
||||
// Semantic similarity between entities and verb type using search
|
||||
const relationshipText = `${this.extractMainText(source.originalData)} ${verbType} ${this.extractMainText(target.originalData)}`
|
||||
const directResults = await this.brainy.find(relationshipText)
|
||||
const directScore = directResults.length > 0 ? directResults[0].score : 0.4
|
||||
|
||||
const contextResults = await this.brainy.find(context + ' ' + verbType)
|
||||
const contextSimilarity = contextResults.length > 0 ? contextResults[0].score : 0.5
|
||||
|
||||
// Entity type compatibility
|
||||
const typeCompatibility = this.calculateTypeCompatibility(source.nounType, target.nounType, verbType)
|
||||
|
||||
// Combine with weights
|
||||
return (directScore * 0.4) + (contextSimilarity * 0.4) + (typeCompatibility * 0.2)
|
||||
}
|
||||
|
||||
/**
|
||||
* Calculate relationship weight/strength
|
||||
*/
|
||||
private calculateRelationshipWeight(
|
||||
source: DetectedEntity,
|
||||
target: DetectedEntity,
|
||||
verbType: string,
|
||||
context: string
|
||||
): number {
|
||||
let weight = 0.5 // Base weight
|
||||
|
||||
// Context richness (more descriptive = stronger)
|
||||
const contextWords = context.split(' ').length
|
||||
weight += Math.min(contextWords / 20, 0.2)
|
||||
|
||||
// Entity importance (higher confidence entities = stronger relationships)
|
||||
const avgEntityConfidence = (source.confidence + target.confidence) / 2
|
||||
weight += avgEntityConfidence * 0.2
|
||||
|
||||
// Verb type specificity (more specific verbs = stronger)
|
||||
const verbSpecificity = this.getVerbSpecificity(verbType)
|
||||
weight += verbSpecificity * 0.1
|
||||
|
||||
return Math.min(weight, 1.0)
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate Neural Insights - The Intelligence Layer
|
||||
*/
|
||||
private async generateNeuralInsights(entities: DetectedEntity[], relationships: DetectedRelationship[]): Promise<NeuralInsight[]> {
|
||||
const insights: NeuralInsight[] = []
|
||||
|
||||
// Detect hierarchies
|
||||
const hierarchies = this.detectHierarchies(relationships)
|
||||
hierarchies.forEach(hierarchy => {
|
||||
insights.push({
|
||||
type: 'hierarchy',
|
||||
description: `Detected ${hierarchy.type} hierarchy with ${hierarchy.levels} levels`,
|
||||
confidence: hierarchy.confidence,
|
||||
affectedEntities: hierarchy.entities,
|
||||
recommendation: `Consider visualizing the ${hierarchy.type} structure`
|
||||
})
|
||||
})
|
||||
|
||||
// Detect clusters
|
||||
const clusters = this.detectClusters(entities, relationships)
|
||||
clusters.forEach(cluster => {
|
||||
insights.push({
|
||||
type: 'cluster',
|
||||
description: `Found cluster of ${cluster.size} ${cluster.primaryType} entities`,
|
||||
confidence: cluster.confidence,
|
||||
affectedEntities: cluster.entities,
|
||||
recommendation: `These ${cluster.primaryType}s might form a natural grouping`
|
||||
})
|
||||
})
|
||||
|
||||
// Detect patterns
|
||||
const patterns = this.detectPatterns(relationships)
|
||||
patterns.forEach(pattern => {
|
||||
insights.push({
|
||||
type: 'pattern',
|
||||
description: `Common relationship pattern: ${pattern.description}`,
|
||||
confidence: pattern.confidence,
|
||||
affectedEntities: pattern.entities,
|
||||
recommendation: pattern.recommendation
|
||||
})
|
||||
})
|
||||
|
||||
return insights
|
||||
}
|
||||
|
||||
/**
|
||||
* Display Neural Analysis Results
|
||||
*/
|
||||
private async displayNeuralAnalysisResults(result: NeuralAnalysisResult, options: NeuralImportOptions): Promise<void> {
|
||||
// Entity summary
|
||||
const entityTable = new Table({
|
||||
head: [this.colors.brain('Entity Type'), this.colors.brain('Count'), this.colors.brain('Avg Confidence')],
|
||||
colWidths: [20, 10, 15]
|
||||
})
|
||||
|
||||
const entitySummary = this.summarizeEntities(result.detectedEntities)
|
||||
Object.entries(entitySummary).forEach(([type, stats]) => {
|
||||
entityTable.push([
|
||||
this.colors.highlight(type),
|
||||
this.colors.primary(stats.count.toString()),
|
||||
this.colors.success(`${(stats.avgConfidence * 100).toFixed(1)}%`)
|
||||
])
|
||||
})
|
||||
|
||||
// Relationship summary
|
||||
const relationshipTable = new Table({
|
||||
head: [this.colors.brain('Relationship Type'), this.colors.brain('Count'), this.colors.brain('Avg Weight'), this.colors.brain('Avg Confidence')],
|
||||
colWidths: [20, 10, 12, 15]
|
||||
})
|
||||
|
||||
const relationshipSummary = this.summarizeRelationships(result.detectedRelationships)
|
||||
Object.entries(relationshipSummary).forEach(([type, stats]) => {
|
||||
relationshipTable.push([
|
||||
this.colors.highlight(type),
|
||||
this.colors.primary(stats.count.toString()),
|
||||
this.colors.warning(`${stats.avgWeight.toFixed(2)}`),
|
||||
this.colors.success(`${(stats.avgConfidence * 100).toFixed(1)}%`)
|
||||
])
|
||||
})
|
||||
|
||||
console.log(boxen(
|
||||
`${this.emojis.atom} ${this.colors.brain('NEURAL CLASSIFICATION RESULTS')}\n\n` +
|
||||
entityTable.toString(),
|
||||
{ padding: 1, borderStyle: 'round', borderColor: '#D67441' }
|
||||
))
|
||||
|
||||
console.log(boxen(
|
||||
`${this.emojis.data} ${this.colors.brain('NEURAL RELATIONSHIP MAPPING')}\n\n` +
|
||||
relationshipTable.toString(),
|
||||
{ padding: 1, borderStyle: 'round', borderColor: '#D67441' }
|
||||
))
|
||||
|
||||
// Display insights
|
||||
if (result.insights.length > 0) {
|
||||
const insightsText = result.insights.map(insight =>
|
||||
`${this.colors.accent('◆')} ${insight.description} (${(insight.confidence * 100).toFixed(1)}% confidence)`
|
||||
).join('\n')
|
||||
|
||||
console.log(boxen(
|
||||
`${this.emojis.magic} ${this.colors.brain('NEURAL INSIGHTS')}\n\n` +
|
||||
insightsText,
|
||||
{ padding: 1, borderStyle: 'round', borderColor: '#E88B5A' }
|
||||
))
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Helper methods for the neural system
|
||||
*/
|
||||
|
||||
private extractMainText(data: any): string {
|
||||
// Extract the most relevant text from a data object
|
||||
const textFields = ['name', 'title', 'description', 'content', 'text', 'label']
|
||||
|
||||
for (const field of textFields) {
|
||||
if (data[field] && typeof data[field] === 'string') {
|
||||
return data[field]
|
||||
}
|
||||
}
|
||||
|
||||
// Fallback: concatenate all string values
|
||||
return Object.values(data)
|
||||
.filter(v => typeof v === 'string')
|
||||
.join(' ')
|
||||
.substring(0, 200) // Limit length
|
||||
}
|
||||
|
||||
private generateSmartId(data: any, nounType: string, index: number): string {
|
||||
const mainText = this.extractMainText(data)
|
||||
const cleanText = mainText.toLowerCase().replace(/[^a-z0-9]/g, '_').substring(0, 20)
|
||||
return `${nounType}_${cleanText}_${index}`
|
||||
}
|
||||
|
||||
private extractRelationshipContext(source: any, target: any, allData: any[]): string {
|
||||
// Extract context for relationship detection
|
||||
return [
|
||||
this.extractMainText(source),
|
||||
this.extractMainText(target),
|
||||
// Add more contextual information
|
||||
].join(' ')
|
||||
}
|
||||
|
||||
private calculateTypeCompatibility(sourceType: string, targetType: string, verbType: string): number {
|
||||
// Define type compatibility matrix for relationships
|
||||
const compatibilityMatrix: Record<string, Record<string, string[]>> = {
|
||||
[NounType.Person]: {
|
||||
[NounType.Organization]: [VerbType.MemberOf, VerbType.WorksWith],
|
||||
[NounType.Project]: [VerbType.WorksWith, VerbType.Creates],
|
||||
[NounType.Person]: [VerbType.WorksWith, VerbType.Mentors, VerbType.ReportsTo]
|
||||
}
|
||||
// Add more compatibility rules
|
||||
}
|
||||
|
||||
const sourceCompatibility = compatibilityMatrix[sourceType]
|
||||
if (sourceCompatibility && sourceCompatibility[targetType]) {
|
||||
return sourceCompatibility[targetType].includes(verbType) ? 1.0 : 0.3
|
||||
}
|
||||
|
||||
return 0.5 // Default compatibility
|
||||
}
|
||||
|
||||
private getVerbSpecificity(verbType: string): number {
|
||||
// More specific verbs get higher scores
|
||||
const specificityScores: Record<string, number> = {
|
||||
[VerbType.RelatedTo]: 0.1, // Very generic
|
||||
[VerbType.WorksWith]: 0.7, // Specific
|
||||
[VerbType.Mentors]: 0.9, // Very specific
|
||||
[VerbType.ReportsTo]: 0.9, // Very specific
|
||||
}
|
||||
|
||||
return specificityScores[verbType] || 0.5
|
||||
}
|
||||
|
||||
private getRelevantFields(data: any, nounType: string): string[] {
|
||||
// Implementation for finding relevant fields
|
||||
return []
|
||||
}
|
||||
|
||||
private getMatchedPatterns(text: string, data: any, nounType: string): string[] {
|
||||
// Implementation for finding matched patterns
|
||||
return []
|
||||
}
|
||||
|
||||
private pruneRelationships(relationships: DetectedRelationship[]): DetectedRelationship[] {
|
||||
// Remove duplicates and low-confidence relationships
|
||||
return relationships
|
||||
.sort((a, b) => b.confidence - a.confidence)
|
||||
.slice(0, 1000) // Limit to top 1000 relationships
|
||||
}
|
||||
|
||||
private detectHierarchies(relationships: DetectedRelationship[]): any[] {
|
||||
// Detect hierarchical structures
|
||||
return []
|
||||
}
|
||||
|
||||
private detectClusters(entities: DetectedEntity[], relationships: DetectedRelationship[]): any[] {
|
||||
// Detect entity clusters
|
||||
return []
|
||||
}
|
||||
|
||||
private detectPatterns(relationships: DetectedRelationship[]): any[] {
|
||||
// Detect relationship patterns
|
||||
return []
|
||||
}
|
||||
|
||||
private summarizeEntities(entities: DetectedEntity[]): Record<string, any> {
|
||||
const summary: Record<string, any> = {}
|
||||
|
||||
entities.forEach(entity => {
|
||||
if (!summary[entity.nounType]) {
|
||||
summary[entity.nounType] = { count: 0, totalConfidence: 0 }
|
||||
}
|
||||
summary[entity.nounType].count++
|
||||
summary[entity.nounType].totalConfidence += entity.confidence
|
||||
})
|
||||
|
||||
Object.keys(summary).forEach(type => {
|
||||
summary[type].avgConfidence = summary[type].totalConfidence / summary[type].count
|
||||
})
|
||||
|
||||
return summary
|
||||
}
|
||||
|
||||
private summarizeRelationships(relationships: DetectedRelationship[]): Record<string, any> {
|
||||
const summary: Record<string, any> = {}
|
||||
|
||||
relationships.forEach(rel => {
|
||||
if (!summary[rel.verbType]) {
|
||||
summary[rel.verbType] = { count: 0, totalWeight: 0, totalConfidence: 0 }
|
||||
}
|
||||
summary[rel.verbType].count++
|
||||
summary[rel.verbType].totalWeight += rel.weight
|
||||
summary[rel.verbType].totalConfidence += rel.confidence
|
||||
})
|
||||
|
||||
Object.keys(summary).forEach(type => {
|
||||
const stats = summary[type]
|
||||
stats.avgWeight = stats.totalWeight / stats.count
|
||||
stats.avgConfidence = stats.totalConfidence / stats.count
|
||||
})
|
||||
|
||||
return summary
|
||||
}
|
||||
|
||||
private calculateOverallConfidence(entities: DetectedEntity[], relationships: DetectedRelationship[]): number {
|
||||
const entityConfidence = entities.reduce((sum, e) => sum + e.confidence, 0) / entities.length
|
||||
const relationshipConfidence = relationships.reduce((sum, r) => sum + r.confidence, 0) / relationships.length
|
||||
return (entityConfidence + relationshipConfidence) / 2
|
||||
}
|
||||
|
||||
private async generatePreview(entities: DetectedEntity[], relationships: DetectedRelationship[]): Promise<ProcessedData[]> {
|
||||
return entities.slice(0, 5).map(entity => ({
|
||||
id: entity.suggestedId,
|
||||
nounType: entity.nounType,
|
||||
data: entity.originalData,
|
||||
relationships: relationships
|
||||
.filter(r => r.sourceId === entity.suggestedId)
|
||||
.slice(0, 3)
|
||||
.map(r => ({
|
||||
target: r.targetId,
|
||||
verbType: r.verbType,
|
||||
weight: r.weight,
|
||||
confidence: r.confidence
|
||||
}))
|
||||
}))
|
||||
}
|
||||
|
||||
private async confirmNeuralImport(result: NeuralAnalysisResult): Promise<boolean> {
|
||||
const { confirm } = await prompts({
|
||||
type: 'confirm',
|
||||
name: 'confirm',
|
||||
message: `${this.emojis.rocket} Execute neural import?`,
|
||||
initial: true
|
||||
})
|
||||
return confirm
|
||||
}
|
||||
|
||||
private async executeNeuralImport(result: NeuralAnalysisResult, options: NeuralImportOptions): Promise<void> {
|
||||
const spinner = ora(`${this.emojis.gear} Executing neural import...`).start()
|
||||
|
||||
try {
|
||||
// Add entities to Brainy
|
||||
for (const entity of result.detectedEntities) {
|
||||
await this.brainy.add({
|
||||
data: this.extractMainText(entity.originalData),
|
||||
type: entity.nounType as NounType,
|
||||
metadata: {
|
||||
...entity.originalData,
|
||||
confidence: entity.confidence,
|
||||
id: entity.suggestedId
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// Add relationships to Brainy
|
||||
for (const relationship of result.detectedRelationships) {
|
||||
await this.brainy.relate({
|
||||
from: relationship.sourceId,
|
||||
to: relationship.targetId,
|
||||
type: relationship.verbType as VerbType,
|
||||
weight: relationship.weight,
|
||||
metadata: {
|
||||
confidence: relationship.confidence,
|
||||
context: relationship.context,
|
||||
...relationship.metadata
|
||||
}
|
||||
}
|
||||
)
|
||||
}
|
||||
|
||||
spinner.succeed(this.colors.success(
|
||||
`${this.emojis.check} Neural import complete! ` +
|
||||
`${result.detectedEntities.length} entities and ` +
|
||||
`${result.detectedRelationships.length} relationships imported.`
|
||||
))
|
||||
|
||||
} catch (error) {
|
||||
spinner.fail('Neural import failed')
|
||||
throw error
|
||||
}
|
||||
}
|
||||
|
||||
private async generateRelationshipReasoning(
|
||||
source: DetectedEntity,
|
||||
target: DetectedEntity,
|
||||
verbType: string,
|
||||
context: string
|
||||
): Promise<string> {
|
||||
return `Neural analysis detected ${verbType} relationship based on semantic context`
|
||||
}
|
||||
|
||||
private extractRelationshipMetadata(sourceData: any, targetData: any, verbType: string): Record<string, any> {
|
||||
return {
|
||||
sourceType: typeof sourceData,
|
||||
targetType: typeof targetData,
|
||||
detectedBy: 'neural-import',
|
||||
timestamp: new Date().toISOString()
|
||||
}
|
||||
}
|
||||
}
|
||||
633
src/neural/neuralImportAugmentation.ts
Normal file
633
src/neural/neuralImportAugmentation.ts
Normal file
|
|
@ -0,0 +1,633 @@
|
|||
/**
|
||||
* Neural Import - AI-Powered Data Understanding
|
||||
*
|
||||
* Standalone implementation for intelligent data processing.
|
||||
*/
|
||||
|
||||
import { NounType, VerbType } from '../types/graphTypes.js'
|
||||
import * as fs from '../universal/fs.js'
|
||||
import * as path from '../universal/path.js'
|
||||
import { prodLog } from '../utils/logger.js'
|
||||
|
||||
// Neural Import Analysis Types
|
||||
export interface NeuralAnalysisResult {
|
||||
detectedEntities: DetectedEntity[]
|
||||
detectedRelationships: DetectedRelationship[]
|
||||
confidence: number
|
||||
insights: NeuralInsight[]
|
||||
}
|
||||
|
||||
export interface DetectedEntity {
|
||||
originalData: any
|
||||
nounType: string
|
||||
confidence: number
|
||||
suggestedId: string
|
||||
reasoning: string
|
||||
alternativeTypes: Array<{ type: string, confidence: number }>
|
||||
}
|
||||
|
||||
export interface DetectedRelationship {
|
||||
sourceId: string
|
||||
targetId: string
|
||||
verbType: string
|
||||
confidence: number
|
||||
weight: number
|
||||
reasoning: string
|
||||
context: string
|
||||
metadata?: Record<string, any>
|
||||
}
|
||||
|
||||
export interface NeuralInsight {
|
||||
type: 'hierarchy' | 'cluster' | 'pattern' | 'anomaly' | 'opportunity'
|
||||
description: string
|
||||
confidence: number
|
||||
affectedEntities: string[]
|
||||
recommendation?: string
|
||||
}
|
||||
|
||||
export interface NeuralImportConfig {
|
||||
confidenceThreshold: number
|
||||
enableWeights: boolean
|
||||
skipDuplicates: boolean
|
||||
categoryFilter?: string[]
|
||||
dataType?: string
|
||||
}
|
||||
|
||||
/**
|
||||
* Neural Import Augmentation - Unified Implementation
|
||||
* Processes data with AI before storage operations
|
||||
*/
|
||||
export class NeuralImportAugmentation {
|
||||
readonly name = 'neural-import'
|
||||
private operations = ['add', 'addNoun', 'addVerb', 'all']
|
||||
|
||||
protected config: NeuralImportConfig
|
||||
private analysisCache = new Map<string, NeuralAnalysisResult>()
|
||||
private context?: { brain: any }
|
||||
|
||||
constructor(config: Partial<NeuralImportConfig> = {}) {
|
||||
this.config = {
|
||||
confidenceThreshold: 0.7,
|
||||
enableWeights: true,
|
||||
skipDuplicates: true,
|
||||
dataType: 'json',
|
||||
...config
|
||||
}
|
||||
}
|
||||
|
||||
async init(): Promise<void> {
|
||||
// No external dependencies to initialize
|
||||
}
|
||||
|
||||
private log(message: string, _level?: string): void {
|
||||
// Silent by default
|
||||
}
|
||||
|
||||
/**
|
||||
* Execute augmentation - process data with AI before storage
|
||||
*/
|
||||
async execute<T = any>(
|
||||
operation: string,
|
||||
params: any,
|
||||
next: () => Promise<T>
|
||||
): Promise<T> {
|
||||
// Only process on add operations
|
||||
if (!this.operations.includes(operation as any)) {
|
||||
return next()
|
||||
}
|
||||
|
||||
try {
|
||||
// Extract data from params based on operation
|
||||
const rawData = this.extractRawData(operation, params)
|
||||
if (!rawData) {
|
||||
return next()
|
||||
}
|
||||
|
||||
// Perform neural analysis
|
||||
const analysis = await this.performNeuralAnalysis(rawData, this.config)
|
||||
|
||||
// Enhance params with neural insights
|
||||
if (params.metadata) {
|
||||
params.metadata._neuralProcessed = true
|
||||
params.metadata._neuralConfidence = analysis.confidence
|
||||
params.metadata._detectedEntities = analysis.detectedEntities.length
|
||||
params.metadata._detectedRelationships = analysis.detectedRelationships.length
|
||||
params.metadata._neuralInsights = analysis.insights
|
||||
} else if (typeof params === 'object') {
|
||||
params.metadata = {
|
||||
_neuralProcessed: true,
|
||||
_neuralConfidence: analysis.confidence,
|
||||
_detectedEntities: analysis.detectedEntities.length,
|
||||
_detectedRelationships: analysis.detectedRelationships.length,
|
||||
_neuralInsights: analysis.insights
|
||||
}
|
||||
}
|
||||
|
||||
// Store neural analysis for later retrieval
|
||||
await this.storeNeuralAnalysis(analysis)
|
||||
|
||||
// If we detected entities/relationships, potentially add them
|
||||
if (this.context?.brain && analysis.detectedEntities.length > 0) {
|
||||
// This could automatically create entities/relationships
|
||||
// But for now, just enhance the metadata
|
||||
this.log(`Detected ${analysis.detectedEntities.length} entities and ${analysis.detectedRelationships.length} relationships`)
|
||||
}
|
||||
|
||||
// Continue with enhanced data
|
||||
return next()
|
||||
} catch (error) {
|
||||
this.log(`Neural analysis failed: ${error}`, 'warn')
|
||||
// Continue without neural processing
|
||||
return next()
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Extract raw data from operation params
|
||||
*/
|
||||
private extractRawData(operation: string, params: any): any {
|
||||
switch (operation) {
|
||||
case 'add':
|
||||
return params.content || params.data || params
|
||||
case 'addNoun':
|
||||
return params.noun || params.data || params
|
||||
case 'addVerb':
|
||||
return params.verb || params
|
||||
case 'addBatch':
|
||||
return params.items || params.batch || params
|
||||
default:
|
||||
return null
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the full neural analysis result (for external use)
|
||||
*/
|
||||
async getNeuralAnalysis(rawData: Buffer | string, dataType?: string): Promise<NeuralAnalysisResult> {
|
||||
const parsedData = await this.parseRawData(rawData, dataType || this.config.dataType || 'json')
|
||||
return await this.performNeuralAnalysis(parsedData, this.config)
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse raw data based on type
|
||||
*/
|
||||
private async parseRawData(rawData: Buffer | string, dataType: string): Promise<any[]> {
|
||||
const content = typeof rawData === 'string' ? rawData : rawData.toString('utf8')
|
||||
|
||||
switch (dataType.toLowerCase()) {
|
||||
case 'json':
|
||||
try {
|
||||
const jsonData = JSON.parse(content)
|
||||
return Array.isArray(jsonData) ? jsonData : [jsonData]
|
||||
} catch {
|
||||
// If JSON parse fails, treat as text
|
||||
return [{ text: content }]
|
||||
}
|
||||
|
||||
case 'csv':
|
||||
return this.parseCSV(content)
|
||||
|
||||
case 'yaml':
|
||||
case 'yml':
|
||||
return this.parseYAML(content)
|
||||
|
||||
case 'txt':
|
||||
case 'text':
|
||||
// Split text into sentences/paragraphs for analysis
|
||||
return content.split(/\n+/).filter(line => line.trim()).map(line => ({ text: line }))
|
||||
|
||||
default:
|
||||
// Unknown type, treat as text
|
||||
return [{ text: content }]
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse CSV data - handles quoted values, escaped quotes, and edge cases
|
||||
*/
|
||||
private parseCSV(content: string): any[] {
|
||||
const lines = content.split('\n')
|
||||
if (lines.length === 0) return []
|
||||
|
||||
// Parse a CSV line handling quotes
|
||||
const parseLine = (line: string): string[] => {
|
||||
const result: string[] = []
|
||||
let current = ''
|
||||
let inQuotes = false
|
||||
let i = 0
|
||||
|
||||
while (i < line.length) {
|
||||
const char = line[i]
|
||||
const nextChar = line[i + 1]
|
||||
|
||||
if (char === '"') {
|
||||
if (inQuotes && nextChar === '"') {
|
||||
// Escaped quote
|
||||
current += '"'
|
||||
i += 2
|
||||
} else {
|
||||
// Toggle quote mode
|
||||
inQuotes = !inQuotes
|
||||
i++
|
||||
}
|
||||
} else if (char === ',' && !inQuotes) {
|
||||
// Field separator
|
||||
result.push(current.trim())
|
||||
current = ''
|
||||
i++
|
||||
} else {
|
||||
current += char
|
||||
i++
|
||||
}
|
||||
}
|
||||
|
||||
// Add last field
|
||||
result.push(current.trim())
|
||||
return result
|
||||
}
|
||||
|
||||
// Parse headers
|
||||
const headers = parseLine(lines[0])
|
||||
const data = []
|
||||
|
||||
// Parse data rows
|
||||
for (let i = 1; i < lines.length; i++) {
|
||||
const line = lines[i].trim()
|
||||
if (!line) continue // Skip empty lines
|
||||
|
||||
const values = parseLine(line)
|
||||
const row: any = {}
|
||||
|
||||
headers.forEach((header, index) => {
|
||||
const value = values[index] || ''
|
||||
// Try to parse numbers
|
||||
const num = Number(value)
|
||||
row[header] = !isNaN(num) && value !== '' ? num : value
|
||||
})
|
||||
|
||||
data.push(row)
|
||||
}
|
||||
|
||||
return data
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse YAML data
|
||||
*/
|
||||
private parseYAML(content: string): any[] {
|
||||
try {
|
||||
// Simple YAML parser for basic structures
|
||||
// For full YAML support, we'd use js-yaml library
|
||||
const lines = content.split('\n')
|
||||
const result: any[] = []
|
||||
let currentObject: any = null
|
||||
let currentIndent = 0
|
||||
|
||||
for (const line of lines) {
|
||||
const trimmed = line.trim()
|
||||
if (!trimmed || trimmed.startsWith('#')) continue // Skip empty lines and comments
|
||||
|
||||
// Calculate indentation
|
||||
const indent = line.length - line.trimStart().length
|
||||
|
||||
// Check for array item
|
||||
if (trimmed.startsWith('- ')) {
|
||||
const value = trimmed.substring(2).trim()
|
||||
if (indent === 0) {
|
||||
// Top-level array item
|
||||
if (value.includes(':')) {
|
||||
// Object in array
|
||||
currentObject = {}
|
||||
result.push(currentObject)
|
||||
const [key, val] = value.split(':').map(s => s.trim())
|
||||
currentObject[key] = this.parseYAMLValue(val)
|
||||
} else {
|
||||
result.push(this.parseYAMLValue(value))
|
||||
}
|
||||
} else if (currentObject) {
|
||||
// Nested array
|
||||
const lastKey = Object.keys(currentObject).pop()
|
||||
if (lastKey) {
|
||||
if (!Array.isArray(currentObject[lastKey])) {
|
||||
currentObject[lastKey] = []
|
||||
}
|
||||
currentObject[lastKey].push(this.parseYAMLValue(value))
|
||||
}
|
||||
}
|
||||
} else if (trimmed.includes(':')) {
|
||||
// Key-value pair
|
||||
const colonIndex = trimmed.indexOf(':')
|
||||
const key = trimmed.substring(0, colonIndex).trim()
|
||||
const value = trimmed.substring(colonIndex + 1).trim()
|
||||
|
||||
if (indent === 0) {
|
||||
// Top-level object
|
||||
if (!currentObject) {
|
||||
currentObject = {}
|
||||
result.push(currentObject)
|
||||
}
|
||||
currentObject[key] = this.parseYAMLValue(value)
|
||||
currentIndent = 0
|
||||
} else if (currentObject) {
|
||||
// Nested object
|
||||
if (indent > currentIndent && !value) {
|
||||
// Start of nested object
|
||||
const lastKey = Object.keys(currentObject).pop()
|
||||
if (lastKey) {
|
||||
currentObject[lastKey] = { [key]: '' }
|
||||
}
|
||||
} else {
|
||||
currentObject[key] = this.parseYAMLValue(value)
|
||||
}
|
||||
currentIndent = indent
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// If we built a single object and not an array, wrap it
|
||||
if (result.length === 0 && currentObject) {
|
||||
result.push(currentObject)
|
||||
}
|
||||
|
||||
return result.length > 0 ? result : [{ text: content }]
|
||||
} catch (error) {
|
||||
prodLog.warn('YAML parsing failed, treating as text:', error)
|
||||
return [{ text: content }]
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse a YAML value (handle strings, numbers, booleans, null)
|
||||
*/
|
||||
private parseYAMLValue(value: string): any {
|
||||
if (!value || value === '~' || value === 'null') return null
|
||||
if (value === 'true') return true
|
||||
if (value === 'false') return false
|
||||
|
||||
// Remove quotes if present
|
||||
if ((value.startsWith('"') && value.endsWith('"')) ||
|
||||
(value.startsWith("'") && value.endsWith("'"))) {
|
||||
return value.slice(1, -1)
|
||||
}
|
||||
|
||||
// Try to parse as number
|
||||
const num = Number(value)
|
||||
if (!isNaN(num) && value !== '') return num
|
||||
|
||||
return value
|
||||
}
|
||||
|
||||
/**
|
||||
* Perform neural analysis on parsed data
|
||||
*/
|
||||
private async performNeuralAnalysis(data: any[], config?: any): Promise<NeuralAnalysisResult> {
|
||||
const detectedEntities: DetectedEntity[] = []
|
||||
const detectedRelationships: DetectedRelationship[] = []
|
||||
const insights: NeuralInsight[] = []
|
||||
|
||||
// Simple entity detection (in real implementation, would use ML)
|
||||
for (const item of data) {
|
||||
if (typeof item === 'object') {
|
||||
// Detect entities from object properties
|
||||
const entityId = item.id || item.name || item.title || `entity_${Date.now()}_${Math.random()}`
|
||||
|
||||
detectedEntities.push({
|
||||
originalData: item,
|
||||
nounType: await this.inferNounType(item),
|
||||
confidence: 0.85,
|
||||
suggestedId: String(entityId),
|
||||
reasoning: 'Detected from structured data',
|
||||
alternativeTypes: []
|
||||
})
|
||||
|
||||
// Detect relationships from references
|
||||
await this.detectRelationships(item, entityId, detectedRelationships)
|
||||
}
|
||||
}
|
||||
|
||||
// Generate insights
|
||||
if (detectedEntities.length > 10) {
|
||||
insights.push({
|
||||
type: 'pattern',
|
||||
description: `Large dataset with ${detectedEntities.length} entities detected`,
|
||||
confidence: 0.9,
|
||||
affectedEntities: detectedEntities.slice(0, 5).map(e => e.suggestedId),
|
||||
recommendation: 'Consider batch processing for optimal performance'
|
||||
})
|
||||
}
|
||||
|
||||
// Look for clusters
|
||||
const typeGroups = this.groupByType(detectedEntities)
|
||||
if (Object.keys(typeGroups).length > 1) {
|
||||
insights.push({
|
||||
type: 'cluster',
|
||||
description: `Multiple entity types detected: ${Object.keys(typeGroups).join(', ')}`,
|
||||
confidence: 0.8,
|
||||
affectedEntities: [],
|
||||
recommendation: 'Data contains diverse entity types suitable for graph analysis'
|
||||
})
|
||||
}
|
||||
|
||||
return {
|
||||
detectedEntities,
|
||||
detectedRelationships,
|
||||
confidence: detectedEntities.length > 0 ? 0.85 : 0.5,
|
||||
insights
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Infer noun type from object structure using field heuristics
|
||||
*/
|
||||
private async inferNounType(obj: any): Promise<string> {
|
||||
if (typeof obj !== 'object' || obj === null) return NounType.Thing
|
||||
|
||||
// Check for explicit type field
|
||||
if (obj.type && typeof obj.type === 'string') {
|
||||
const normalized = obj.type.charAt(0).toUpperCase() + obj.type.slice(1)
|
||||
if (Object.values(NounType).includes(normalized as NounType)) {
|
||||
return normalized as NounType
|
||||
}
|
||||
}
|
||||
|
||||
if (obj.email || obj.firstName || obj.lastName || obj.username) return NounType.Person
|
||||
if (obj.companyName || obj.organizationId || obj.employees) return NounType.Organization
|
||||
if (obj.latitude || obj.longitude || obj.address || obj.city) return NounType.Location
|
||||
if ((obj.content && (obj.title || obj.author)) || obj.pages) return NounType.Document
|
||||
if (obj.startTime || obj.endTime || obj.date || obj.attendees) return NounType.Event
|
||||
if (obj.price || obj.sku || obj.productId) return NounType.Product
|
||||
if ((obj.status && obj.assignee) || obj.priority) return NounType.Task
|
||||
if (Array.isArray(obj.data) || obj.rows || obj.columns) return NounType.Dataset
|
||||
|
||||
return NounType.Thing
|
||||
}
|
||||
|
||||
/**
|
||||
* Detect relationships from object references
|
||||
*/
|
||||
private async detectRelationships(obj: any, sourceId: string, relationships: DetectedRelationship[]): Promise<void> {
|
||||
// Look for reference patterns
|
||||
for (const [key, value] of Object.entries(obj)) {
|
||||
if (key.endsWith('Id') || key.endsWith('_id') || key === 'parentId' || key === 'userId') {
|
||||
relationships.push({
|
||||
sourceId,
|
||||
targetId: String(value),
|
||||
verbType: await this.inferVerbType(key, obj, { id: value }),
|
||||
confidence: 0.75,
|
||||
weight: 1,
|
||||
reasoning: `Reference detected in field: ${key}`,
|
||||
context: key
|
||||
})
|
||||
}
|
||||
|
||||
// Array of IDs
|
||||
if (Array.isArray(value) && value.length > 0 && typeof value[0] === 'string') {
|
||||
if (key.endsWith('Ids') || key.endsWith('_ids')) {
|
||||
for (const targetId of value) {
|
||||
relationships.push({
|
||||
sourceId,
|
||||
targetId: String(targetId),
|
||||
verbType: await this.inferVerbType(key, obj, { id: targetId }),
|
||||
confidence: 0.7,
|
||||
weight: 1,
|
||||
reasoning: `Array reference in field: ${key}`,
|
||||
context: key
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Infer verb type from field name using common patterns
|
||||
*/
|
||||
private async inferVerbType(fieldName: string, _sourceObj?: any, _targetObj?: any): Promise<string> {
|
||||
const field = fieldName.toLowerCase()
|
||||
|
||||
if (field.includes('parent') || field.includes('child') || field.includes('contain')) {
|
||||
return VerbType.Contains
|
||||
}
|
||||
if (field.includes('owner') || field.includes('created') || field.includes('author')) {
|
||||
return VerbType.Creates
|
||||
}
|
||||
if (field.includes('member') || field.includes('belong')) {
|
||||
return VerbType.MemberOf
|
||||
}
|
||||
if (field.includes('depend') || field.includes('require')) {
|
||||
return VerbType.DependsOn
|
||||
}
|
||||
if (field.includes('ref') || field.includes('link') || field.includes('source')) {
|
||||
return VerbType.References
|
||||
}
|
||||
|
||||
return VerbType.RelatedTo
|
||||
}
|
||||
|
||||
/**
|
||||
* Group entities by type
|
||||
*/
|
||||
private groupByType(entities: DetectedEntity[]): Record<string, DetectedEntity[]> {
|
||||
const groups: Record<string, DetectedEntity[]> = {}
|
||||
|
||||
for (const entity of entities) {
|
||||
if (!groups[entity.nounType]) {
|
||||
groups[entity.nounType] = []
|
||||
}
|
||||
groups[entity.nounType].push(entity)
|
||||
}
|
||||
|
||||
return groups
|
||||
}
|
||||
|
||||
/**
|
||||
* Store neural analysis results
|
||||
*/
|
||||
private async storeNeuralAnalysis(analysis: NeuralAnalysisResult): Promise<void> {
|
||||
// Cache the analysis for potential later use
|
||||
const key = `analysis_${Date.now()}`
|
||||
this.analysisCache.set(key, analysis)
|
||||
|
||||
// Limit cache size
|
||||
if (this.analysisCache.size > 100) {
|
||||
const firstKey = this.analysisCache.keys().next().value
|
||||
if (firstKey) {
|
||||
this.analysisCache.delete(firstKey)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Helper to get data type from file path
|
||||
*/
|
||||
private getDataTypeFromPath(filePath: string): string {
|
||||
const ext = path.extname(filePath).toLowerCase()
|
||||
switch (ext) {
|
||||
case '.json': return 'json'
|
||||
case '.csv': return 'csv'
|
||||
case '.txt': return 'text'
|
||||
case '.yaml':
|
||||
case '.yml': return 'yaml'
|
||||
default: return 'text'
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* PUBLIC API: Process raw data (for external use, like Synapses)
|
||||
* This maintains compatibility with code that wants to use Neural Import directly
|
||||
*/
|
||||
async processRawData(
|
||||
rawData: Buffer | string,
|
||||
dataType: string,
|
||||
options?: Record<string, unknown>
|
||||
): Promise<{
|
||||
success: boolean
|
||||
data: {
|
||||
nouns: string[]
|
||||
verbs: string[]
|
||||
confidence?: number
|
||||
insights?: Array<{
|
||||
type: string
|
||||
description: string
|
||||
confidence: number
|
||||
}>
|
||||
metadata?: Record<string, unknown>
|
||||
}
|
||||
error?: string
|
||||
}> {
|
||||
try {
|
||||
const analysis = await this.getNeuralAnalysis(rawData, dataType)
|
||||
|
||||
// Convert to legacy format for compatibility
|
||||
const nouns = analysis.detectedEntities.map(e => e.suggestedId)
|
||||
const verbs = analysis.detectedRelationships.map(r =>
|
||||
`${r.sourceId}->${r.verbType}->${r.targetId}`
|
||||
)
|
||||
|
||||
return {
|
||||
success: true,
|
||||
data: {
|
||||
nouns,
|
||||
verbs,
|
||||
confidence: analysis.confidence,
|
||||
insights: analysis.insights.map(i => ({
|
||||
type: i.type,
|
||||
description: i.description,
|
||||
confidence: i.confidence
|
||||
})),
|
||||
metadata: {
|
||||
detectedEntities: analysis.detectedEntities.length,
|
||||
detectedRelationships: analysis.detectedRelationships.length,
|
||||
timestamp: new Date().toISOString()
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
return {
|
||||
success: false,
|
||||
data: { nouns: [], verbs: [] },
|
||||
error: error instanceof Error ? error.message : 'Neural analysis failed'
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue