750 lines
33 KiB
JavaScript
750 lines
33 KiB
JavaScript
|
/**
|
|||
|
|
* Neural Import Augmentation - AI-Powered Data Understanding
|
|||
|
|
*
|
|||
|
|
* 🧠 Built-in AI augmentation for intelligent data processing
|
|||
|
|
* ⚛️ Always free, always included, always enabled
|
|||
|
|
*
|
|||
|
|
* This is the default AI-powered augmentation that comes with every Brainy installation.
|
|||
|
|
* It provides intelligent data understanding, entity detection, and relationship analysis.
|
|||
|
|
*/
|
|||
|
|
import { NounType, VerbType } from '../types/graphTypes.js';
|
|||
|
|
import * as fs from '../universal/fs.js';
|
|||
|
|
import * as path from '../universal/path.js';
|
|||
|
|
/**
|
|||
|
|
* Neural Import SENSE Augmentation - The Brain's Perceptual System
|
|||
|
|
*/
|
|||
|
|
export class NeuralImportAugmentation {
|
|||
|
|
constructor(brainy, config = {}) {
|
|||
|
|
this.name = 'neural-import';
|
|||
|
|
this.description = 'Built-in AI-powered data understanding and entity detection';
|
|||
|
|
this.enabled = true;
|
|||
|
|
this.brainy = brainy;
|
|||
|
|
this.config = {
|
|||
|
|
confidenceThreshold: 0.7,
|
|||
|
|
enableWeights: true,
|
|||
|
|
skipDuplicates: true,
|
|||
|
|
...config
|
|||
|
|
};
|
|||
|
|
}
|
|||
|
|
async initialize() {
|
|||
|
|
// Initialize the cortex analysis system
|
|||
|
|
console.log('🧠 Neural Import augmentation initialized');
|
|||
|
|
}
|
|||
|
|
async shutDown() {
|
|||
|
|
console.log('🧠 Neural Import SENSE augmentation shut down');
|
|||
|
|
}
|
|||
|
|
async getStatus() {
|
|||
|
|
return this.enabled ? 'active' : 'inactive';
|
|||
|
|
}
|
|||
|
|
/**
|
|||
|
|
* Process raw data into structured nouns and verbs using neural analysis
|
|||
|
|
*/
|
|||
|
|
async processRawData(rawData, dataType, options) {
|
|||
|
|
try {
|
|||
|
|
// Merge options with config
|
|||
|
|
const mergedConfig = { ...this.config, ...options };
|
|||
|
|
// Parse the raw data based on type
|
|||
|
|
const parsedData = await this.parseRawData(rawData, dataType);
|
|||
|
|
// Perform neural analysis
|
|||
|
|
const analysis = await this.performNeuralAnalysis(parsedData, mergedConfig);
|
|||
|
|
// Extract nouns and verbs for the ISenseAugmentation interface
|
|||
|
|
const nouns = analysis.detectedEntities.map(entity => entity.suggestedId);
|
|||
|
|
const verbs = analysis.detectedRelationships.map(rel => `${rel.sourceId}->${rel.verbType}->${rel.targetId}`);
|
|||
|
|
// Store the full analysis for later retrieval
|
|||
|
|
await this.storeNeuralAnalysis(analysis);
|
|||
|
|
return {
|
|||
|
|
success: true,
|
|||
|
|
data: {
|
|||
|
|
nouns,
|
|||
|
|
verbs,
|
|||
|
|
confidence: analysis.confidence,
|
|||
|
|
insights: analysis.insights.map((insight) => ({
|
|||
|
|
type: insight.type,
|
|||
|
|
description: insight.description,
|
|||
|
|
confidence: insight.confidence
|
|||
|
|
})),
|
|||
|
|
metadata: {
|
|||
|
|
detectedEntities: analysis.detectedEntities.length,
|
|||
|
|
detectedRelationships: analysis.detectedRelationships.length,
|
|||
|
|
timestamp: new Date().toISOString(),
|
|||
|
|
augmentation: 'neural-import-sense'
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
};
|
|||
|
|
}
|
|||
|
|
catch (error) {
|
|||
|
|
return {
|
|||
|
|
success: false,
|
|||
|
|
data: { nouns: [], verbs: [] },
|
|||
|
|
error: error instanceof Error ? error.message : 'Neural analysis failed'
|
|||
|
|
};
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
/**
|
|||
|
|
* Listen to real-time data feeds and process them
|
|||
|
|
*/
|
|||
|
|
async listenToFeed(feedUrl, callback) {
|
|||
|
|
// For file-based feeds, watch for changes
|
|||
|
|
if (feedUrl.startsWith('file://')) {
|
|||
|
|
const filePath = feedUrl.replace('file://', '');
|
|||
|
|
// Watch file for changes using Node.js fs.watch
|
|||
|
|
const fsWatch = require('fs');
|
|||
|
|
const watcher = fsWatch.watch(filePath, async (eventType) => {
|
|||
|
|
if (eventType === 'change') {
|
|||
|
|
try {
|
|||
|
|
const fileContent = await fs.readFile(filePath);
|
|||
|
|
const result = await this.processRawData(fileContent, this.getDataTypeFromPath(filePath));
|
|||
|
|
if (result.success) {
|
|||
|
|
callback({
|
|||
|
|
nouns: result.data.nouns,
|
|||
|
|
verbs: result.data.verbs,
|
|||
|
|
confidence: result.data.confidence
|
|||
|
|
});
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
catch (error) {
|
|||
|
|
console.error('Neural Import feed error:', error);
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
});
|
|||
|
|
return;
|
|||
|
|
}
|
|||
|
|
// For other feed types, implement appropriate listeners
|
|||
|
|
console.log(`🧠 Neural Import listening to feed: ${feedUrl}`);
|
|||
|
|
}
|
|||
|
|
/**
|
|||
|
|
* Analyze data structure without processing (preview mode)
|
|||
|
|
*/
|
|||
|
|
async analyzeStructure(rawData, dataType, options) {
|
|||
|
|
try {
|
|||
|
|
// Parse the raw data
|
|||
|
|
const parsedData = await this.parseRawData(rawData, dataType);
|
|||
|
|
// Perform lightweight analysis for structure detection
|
|||
|
|
const analysis = await this.performNeuralAnalysis(parsedData, { ...this.config, ...options });
|
|||
|
|
// Summarize entity types
|
|||
|
|
const entityTypeCounts = new Map();
|
|||
|
|
analysis.detectedEntities.forEach(entity => {
|
|||
|
|
const existing = entityTypeCounts.get(entity.nounType) || { count: 0, totalConfidence: 0 };
|
|||
|
|
entityTypeCounts.set(entity.nounType, {
|
|||
|
|
count: existing.count + 1,
|
|||
|
|
totalConfidence: existing.totalConfidence + entity.confidence
|
|||
|
|
});
|
|||
|
|
});
|
|||
|
|
const entityTypes = Array.from(entityTypeCounts.entries()).map(([type, stats]) => ({
|
|||
|
|
type,
|
|||
|
|
count: stats.count,
|
|||
|
|
confidence: stats.totalConfidence / stats.count
|
|||
|
|
}));
|
|||
|
|
// Summarize relationship types
|
|||
|
|
const relationshipTypeCounts = new Map();
|
|||
|
|
analysis.detectedRelationships.forEach(rel => {
|
|||
|
|
const existing = relationshipTypeCounts.get(rel.verbType) || { count: 0, totalConfidence: 0 };
|
|||
|
|
relationshipTypeCounts.set(rel.verbType, {
|
|||
|
|
count: existing.count + 1,
|
|||
|
|
totalConfidence: existing.totalConfidence + rel.confidence
|
|||
|
|
});
|
|||
|
|
});
|
|||
|
|
const relationshipTypes = Array.from(relationshipTypeCounts.entries()).map(([type, stats]) => ({
|
|||
|
|
type,
|
|||
|
|
count: stats.count,
|
|||
|
|
confidence: stats.totalConfidence / stats.count
|
|||
|
|
}));
|
|||
|
|
// Assess data quality
|
|||
|
|
const dataQuality = this.assessDataQuality(parsedData, analysis);
|
|||
|
|
// Generate recommendations
|
|||
|
|
const recommendations = this.generateRecommendations(parsedData, analysis, entityTypes, relationshipTypes);
|
|||
|
|
return {
|
|||
|
|
success: true,
|
|||
|
|
data: {
|
|||
|
|
entityTypes,
|
|||
|
|
relationshipTypes,
|
|||
|
|
dataQuality,
|
|||
|
|
recommendations
|
|||
|
|
}
|
|||
|
|
};
|
|||
|
|
}
|
|||
|
|
catch (error) {
|
|||
|
|
return {
|
|||
|
|
success: false,
|
|||
|
|
data: {
|
|||
|
|
entityTypes: [],
|
|||
|
|
relationshipTypes: [],
|
|||
|
|
dataQuality: { completeness: 0, consistency: 0, accuracy: 0 },
|
|||
|
|
recommendations: []
|
|||
|
|
},
|
|||
|
|
error: error instanceof Error ? error.message : 'Structure analysis failed'
|
|||
|
|
};
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
/**
|
|||
|
|
* Validate data compatibility with current knowledge base
|
|||
|
|
*/
|
|||
|
|
async validateCompatibility(rawData, dataType) {
|
|||
|
|
try {
|
|||
|
|
// Parse the raw data
|
|||
|
|
const parsedData = await this.parseRawData(rawData, dataType);
|
|||
|
|
// Perform neural analysis
|
|||
|
|
const analysis = await this.performNeuralAnalysis(parsedData);
|
|||
|
|
const issues = [];
|
|||
|
|
const suggestions = [];
|
|||
|
|
// Check for low confidence entities
|
|||
|
|
const lowConfidenceEntities = analysis.detectedEntities.filter((e) => e.confidence < 0.5);
|
|||
|
|
if (lowConfidenceEntities.length > 0) {
|
|||
|
|
issues.push({
|
|||
|
|
type: 'confidence',
|
|||
|
|
description: `${lowConfidenceEntities.length} entities have low confidence scores`,
|
|||
|
|
severity: 'medium'
|
|||
|
|
});
|
|||
|
|
suggestions.push('Consider reviewing field names and data structure for better entity detection');
|
|||
|
|
}
|
|||
|
|
// Check for missing relationships
|
|||
|
|
if (analysis.detectedRelationships.length === 0 && analysis.detectedEntities.length > 1) {
|
|||
|
|
issues.push({
|
|||
|
|
type: 'relationships',
|
|||
|
|
description: 'No relationships detected between entities',
|
|||
|
|
severity: 'low'
|
|||
|
|
});
|
|||
|
|
suggestions.push('Consider adding contextual fields that describe entity relationships');
|
|||
|
|
}
|
|||
|
|
// Check for data type compatibility
|
|||
|
|
const supportedTypes = ['json', 'csv', 'yaml', 'text'];
|
|||
|
|
if (!supportedTypes.includes(dataType.toLowerCase())) {
|
|||
|
|
issues.push({
|
|||
|
|
type: 'format',
|
|||
|
|
description: `Data type '${dataType}' may not be fully supported`,
|
|||
|
|
severity: 'high'
|
|||
|
|
});
|
|||
|
|
suggestions.push(`Convert data to one of: ${supportedTypes.join(', ')}`);
|
|||
|
|
}
|
|||
|
|
// Check for data completeness
|
|||
|
|
const incompleteEntities = analysis.detectedEntities.filter((e) => !e.originalData || Object.keys(e.originalData).length < 2);
|
|||
|
|
if (incompleteEntities.length > 0) {
|
|||
|
|
issues.push({
|
|||
|
|
type: 'completeness',
|
|||
|
|
description: `${incompleteEntities.length} entities have insufficient data`,
|
|||
|
|
severity: 'medium'
|
|||
|
|
});
|
|||
|
|
suggestions.push('Ensure each entity has multiple descriptive fields');
|
|||
|
|
}
|
|||
|
|
const compatible = issues.filter(i => i.severity === 'high').length === 0;
|
|||
|
|
return {
|
|||
|
|
success: true,
|
|||
|
|
data: {
|
|||
|
|
compatible,
|
|||
|
|
issues,
|
|||
|
|
suggestions
|
|||
|
|
}
|
|||
|
|
};
|
|||
|
|
}
|
|||
|
|
catch (error) {
|
|||
|
|
return {
|
|||
|
|
success: false,
|
|||
|
|
data: {
|
|||
|
|
compatible: false,
|
|||
|
|
issues: [{
|
|||
|
|
type: 'error',
|
|||
|
|
description: error instanceof Error ? error.message : 'Validation failed',
|
|||
|
|
severity: 'high'
|
|||
|
|
}],
|
|||
|
|
suggestions: []
|
|||
|
|
},
|
|||
|
|
error: error instanceof Error ? error.message : 'Compatibility validation failed'
|
|||
|
|
};
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
/**
|
|||
|
|
* Get the full neural analysis result (custom method for Cortex integration)
|
|||
|
|
*/
|
|||
|
|
async getNeuralAnalysis(rawData, dataType) {
|
|||
|
|
const parsedData = await this.parseRawData(rawData, dataType);
|
|||
|
|
return await this.performNeuralAnalysis(parsedData);
|
|||
|
|
}
|
|||
|
|
/**
|
|||
|
|
* Parse raw data based on type
|
|||
|
|
*/
|
|||
|
|
async parseRawData(rawData, dataType) {
|
|||
|
|
const content = typeof rawData === 'string' ? rawData : rawData.toString('utf8');
|
|||
|
|
switch (dataType.toLowerCase()) {
|
|||
|
|
case 'json':
|
|||
|
|
const jsonData = JSON.parse(content);
|
|||
|
|
return Array.isArray(jsonData) ? jsonData : [jsonData];
|
|||
|
|
case 'csv':
|
|||
|
|
return this.parseCSV(content);
|
|||
|
|
case 'yaml':
|
|||
|
|
case 'yml':
|
|||
|
|
// For now, basic YAML support - in full implementation would use yaml parser
|
|||
|
|
return JSON.parse(content); // Placeholder
|
|||
|
|
case 'txt':
|
|||
|
|
case 'text':
|
|||
|
|
// Split text into sentences/paragraphs for analysis
|
|||
|
|
return content.split(/\n+/).filter(line => line.trim()).map(line => ({ text: line }));
|
|||
|
|
default:
|
|||
|
|
throw new Error(`Unsupported data type: ${dataType}`);
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
/**
|
|||
|
|
* Basic CSV parser
|
|||
|
|
*/
|
|||
|
|
parseCSV(content) {
|
|||
|
|
const lines = content.split('\n').filter(line => line.trim());
|
|||
|
|
if (lines.length < 2)
|
|||
|
|
return [];
|
|||
|
|
const headers = lines[0].split(',').map(h => h.trim().replace(/"/g, ''));
|
|||
|
|
const data = [];
|
|||
|
|
for (let i = 1; i < lines.length; i++) {
|
|||
|
|
const values = lines[i].split(',').map(v => v.trim().replace(/"/g, ''));
|
|||
|
|
const row = {};
|
|||
|
|
headers.forEach((header, index) => {
|
|||
|
|
row[header] = values[index] || '';
|
|||
|
|
});
|
|||
|
|
data.push(row);
|
|||
|
|
}
|
|||
|
|
return data;
|
|||
|
|
}
|
|||
|
|
/**
|
|||
|
|
* Perform neural analysis on parsed data
|
|||
|
|
*/
|
|||
|
|
async performNeuralAnalysis(parsedData, config = this.config) {
|
|||
|
|
// Phase 1: Neural Entity Detection
|
|||
|
|
const detectedEntities = await this.detectEntitiesWithNeuralAnalysis(parsedData, config);
|
|||
|
|
// Phase 2: Neural Relationship Detection
|
|||
|
|
const detectedRelationships = await this.detectRelationshipsWithNeuralAnalysis(detectedEntities, parsedData, config);
|
|||
|
|
// Phase 3: Neural Insights Generation
|
|||
|
|
const insights = await this.generateNeuralInsights(detectedEntities, detectedRelationships);
|
|||
|
|
// Phase 4: Confidence Scoring
|
|||
|
|
const overallConfidence = this.calculateOverallConfidence(detectedEntities, detectedRelationships);
|
|||
|
|
return {
|
|||
|
|
detectedEntities,
|
|||
|
|
detectedRelationships,
|
|||
|
|
confidence: overallConfidence,
|
|||
|
|
insights
|
|||
|
|
};
|
|||
|
|
}
|
|||
|
|
/**
|
|||
|
|
* Neural Entity Detection - The Core AI Engine
|
|||
|
|
*/
|
|||
|
|
async detectEntitiesWithNeuralAnalysis(rawData, config = this.config) {
|
|||
|
|
const entities = [];
|
|||
|
|
const nounTypes = Object.values(NounType);
|
|||
|
|
for (const [index, dataItem] of rawData.entries()) {
|
|||
|
|
const mainText = this.extractMainText(dataItem);
|
|||
|
|
const detections = [];
|
|||
|
|
// Test against all noun types using semantic similarity
|
|||
|
|
for (const nounType of nounTypes) {
|
|||
|
|
const confidence = await this.calculateEntityTypeConfidence(mainText, dataItem, nounType);
|
|||
|
|
if (confidence >= config.confidenceThreshold - 0.2) { // Allow slightly lower for alternatives
|
|||
|
|
const reasoning = await this.generateEntityReasoning(mainText, dataItem, nounType);
|
|||
|
|
detections.push({ type: nounType, confidence, reasoning });
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
if (detections.length > 0) {
|
|||
|
|
// Sort by confidence
|
|||
|
|
detections.sort((a, b) => b.confidence - a.confidence);
|
|||
|
|
const primaryType = detections[0];
|
|||
|
|
const alternatives = detections.slice(1, 3); // Top 2 alternatives
|
|||
|
|
entities.push({
|
|||
|
|
originalData: dataItem,
|
|||
|
|
nounType: primaryType.type,
|
|||
|
|
confidence: primaryType.confidence,
|
|||
|
|
suggestedId: this.generateSmartId(dataItem, primaryType.type, index),
|
|||
|
|
reasoning: primaryType.reasoning,
|
|||
|
|
alternativeTypes: alternatives
|
|||
|
|
});
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
return entities;
|
|||
|
|
}
|
|||
|
|
/**
|
|||
|
|
* Calculate entity type confidence using AI
|
|||
|
|
*/
|
|||
|
|
async calculateEntityTypeConfidence(text, data, nounType) {
|
|||
|
|
// Base semantic similarity using search
|
|||
|
|
const searchResults = await this.brainy.search(text + ' ' + nounType, 1);
|
|||
|
|
const textSimilarity = searchResults.length > 0 ? searchResults[0].score : 0.5;
|
|||
|
|
// Field-based confidence boost
|
|||
|
|
const fieldBoost = this.calculateFieldBasedConfidence(data, nounType);
|
|||
|
|
// Pattern-based confidence boost
|
|||
|
|
const patternBoost = this.calculatePatternBasedConfidence(text, data, nounType);
|
|||
|
|
// Combine confidences with weights
|
|||
|
|
const combined = (textSimilarity * 0.5) + (fieldBoost * 0.3) + (patternBoost * 0.2);
|
|||
|
|
return Math.min(combined, 1.0);
|
|||
|
|
}
|
|||
|
|
/**
|
|||
|
|
* Field-based confidence calculation
|
|||
|
|
*/
|
|||
|
|
calculateFieldBasedConfidence(data, nounType) {
|
|||
|
|
const fields = Object.keys(data);
|
|||
|
|
let boost = 0;
|
|||
|
|
// Field patterns that boost confidence for specific noun types
|
|||
|
|
const fieldPatterns = {
|
|||
|
|
[NounType.Person]: ['name', 'email', 'phone', 'age', 'firstname', 'lastname', 'employee'],
|
|||
|
|
[NounType.Organization]: ['company', 'organization', 'corp', 'inc', 'ltd', 'department', 'team'],
|
|||
|
|
[NounType.Project]: ['project', 'task', 'deadline', 'status', 'milestone', 'deliverable'],
|
|||
|
|
[NounType.Location]: ['address', 'city', 'country', 'state', 'zip', 'location', 'coordinates'],
|
|||
|
|
[NounType.Product]: ['product', 'price', 'sku', 'inventory', 'category', 'brand'],
|
|||
|
|
[NounType.Event]: ['date', 'time', 'venue', 'event', 'meeting', 'conference', 'schedule']
|
|||
|
|
};
|
|||
|
|
const relevantPatterns = fieldPatterns[nounType] || [];
|
|||
|
|
for (const field of fields) {
|
|||
|
|
for (const pattern of relevantPatterns) {
|
|||
|
|
if (field.toLowerCase().includes(pattern)) {
|
|||
|
|
boost += 0.1;
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
return Math.min(boost, 0.5);
|
|||
|
|
}
|
|||
|
|
/**
|
|||
|
|
* Pattern-based confidence calculation
|
|||
|
|
*/
|
|||
|
|
calculatePatternBasedConfidence(text, data, nounType) {
|
|||
|
|
let boost = 0;
|
|||
|
|
// Content patterns that indicate entity types
|
|||
|
|
const patterns = {
|
|||
|
|
[NounType.Person]: [
|
|||
|
|
/@.*\.com/i, // Email pattern
|
|||
|
|
/\b[A-Z][a-z]+ [A-Z][a-z]+\b/, // Name pattern
|
|||
|
|
/Mr\.|Mrs\.|Dr\.|Prof\./i // Title pattern
|
|||
|
|
],
|
|||
|
|
[NounType.Organization]: [
|
|||
|
|
/\bInc\.|Corp\.|LLC\.|Ltd\./i, // Corporate suffixes
|
|||
|
|
/Company|Corporation|Enterprise/i
|
|||
|
|
],
|
|||
|
|
[NounType.Location]: [
|
|||
|
|
/\b\d{5}(-\d{4})?\b/, // ZIP code
|
|||
|
|
/Street|Ave|Road|Blvd/i
|
|||
|
|
]
|
|||
|
|
};
|
|||
|
|
const relevantPatterns = patterns[nounType] || [];
|
|||
|
|
for (const pattern of relevantPatterns) {
|
|||
|
|
if (pattern.test(text)) {
|
|||
|
|
boost += 0.15;
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
return Math.min(boost, 0.3);
|
|||
|
|
}
|
|||
|
|
/**
|
|||
|
|
* Generate reasoning for entity type selection
|
|||
|
|
*/
|
|||
|
|
async generateEntityReasoning(text, data, nounType) {
|
|||
|
|
const reasons = [];
|
|||
|
|
// Semantic similarity reason
|
|||
|
|
const searchResults = await this.brainy.search(text + ' ' + nounType, 1);
|
|||
|
|
const similarity = searchResults.length > 0 ? searchResults[0].score : 0.5;
|
|||
|
|
if (similarity > 0.7) {
|
|||
|
|
reasons.push(`High semantic similarity (${(similarity * 100).toFixed(1)}%)`);
|
|||
|
|
}
|
|||
|
|
// Field-based reasons
|
|||
|
|
const relevantFields = this.getRelevantFields(data, nounType);
|
|||
|
|
if (relevantFields.length > 0) {
|
|||
|
|
reasons.push(`Contains ${nounType}-specific fields: ${relevantFields.join(', ')}`);
|
|||
|
|
}
|
|||
|
|
// Pattern-based reasons
|
|||
|
|
const matchedPatterns = this.getMatchedPatterns(text, data, nounType);
|
|||
|
|
if (matchedPatterns.length > 0) {
|
|||
|
|
reasons.push(`Matches ${nounType} patterns: ${matchedPatterns.join(', ')}`);
|
|||
|
|
}
|
|||
|
|
return reasons.length > 0 ? reasons.join('; ') : 'General semantic match';
|
|||
|
|
}
|
|||
|
|
/**
|
|||
|
|
* Neural Relationship Detection
|
|||
|
|
*/
|
|||
|
|
async detectRelationshipsWithNeuralAnalysis(entities, rawData, config = this.config) {
|
|||
|
|
const relationships = [];
|
|||
|
|
const verbTypes = Object.values(VerbType);
|
|||
|
|
// For each pair of entities, test relationship possibilities
|
|||
|
|
for (let i = 0; i < entities.length; i++) {
|
|||
|
|
for (let j = i + 1; j < entities.length; j++) {
|
|||
|
|
const sourceEntity = entities[i];
|
|||
|
|
const targetEntity = entities[j];
|
|||
|
|
// Extract context for relationship detection
|
|||
|
|
const context = this.extractRelationshipContext(sourceEntity.originalData, targetEntity.originalData, rawData);
|
|||
|
|
// Test all verb types
|
|||
|
|
for (const verbType of verbTypes) {
|
|||
|
|
const confidence = await this.calculateRelationshipConfidence(sourceEntity, targetEntity, verbType, context);
|
|||
|
|
if (confidence >= config.confidenceThreshold - 0.1) { // Slightly lower threshold for relationships
|
|||
|
|
const weight = config.enableWeights ?
|
|||
|
|
this.calculateRelationshipWeight(sourceEntity, targetEntity, verbType, context) :
|
|||
|
|
0.5;
|
|||
|
|
const reasoning = await this.generateRelationshipReasoning(sourceEntity, targetEntity, verbType, context);
|
|||
|
|
relationships.push({
|
|||
|
|
sourceId: sourceEntity.suggestedId,
|
|||
|
|
targetId: targetEntity.suggestedId,
|
|||
|
|
verbType,
|
|||
|
|
confidence,
|
|||
|
|
weight,
|
|||
|
|
reasoning,
|
|||
|
|
context,
|
|||
|
|
metadata: this.extractRelationshipMetadata(sourceEntity.originalData, targetEntity.originalData, verbType)
|
|||
|
|
});
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
// Sort by confidence and remove duplicates/conflicts
|
|||
|
|
return this.pruneRelationships(relationships);
|
|||
|
|
}
|
|||
|
|
/**
|
|||
|
|
* Calculate relationship confidence
|
|||
|
|
*/
|
|||
|
|
async calculateRelationshipConfidence(source, target, verbType, context) {
|
|||
|
|
// Semantic similarity between entities and verb type
|
|||
|
|
const relationshipText = `${this.extractMainText(source.originalData)} ${verbType} ${this.extractMainText(target.originalData)}`;
|
|||
|
|
const directResults = await this.brainy.search(relationshipText, 1);
|
|||
|
|
const directSimilarity = directResults.length > 0 ? directResults[0].score : 0.5;
|
|||
|
|
// Context-based similarity
|
|||
|
|
const contextResults = await this.brainy.search(context + ' ' + verbType, 1);
|
|||
|
|
const contextSimilarity = contextResults.length > 0 ? contextResults[0].score : 0.5;
|
|||
|
|
// Entity type compatibility
|
|||
|
|
const typeCompatibility = this.calculateTypeCompatibility(source.nounType, target.nounType, verbType);
|
|||
|
|
// Combine with weights
|
|||
|
|
return (directSimilarity * 0.4) + (contextSimilarity * 0.4) + (typeCompatibility * 0.2);
|
|||
|
|
}
|
|||
|
|
/**
|
|||
|
|
* Calculate relationship weight/strength
|
|||
|
|
*/
|
|||
|
|
calculateRelationshipWeight(source, target, verbType, context) {
|
|||
|
|
let weight = 0.5; // Base weight
|
|||
|
|
// Context richness (more descriptive = stronger)
|
|||
|
|
const contextWords = context.split(' ').length;
|
|||
|
|
weight += Math.min(contextWords / 20, 0.2);
|
|||
|
|
// Entity importance (higher confidence entities = stronger relationships)
|
|||
|
|
const avgEntityConfidence = (source.confidence + target.confidence) / 2;
|
|||
|
|
weight += avgEntityConfidence * 0.2;
|
|||
|
|
// Verb type specificity (more specific verbs = stronger)
|
|||
|
|
const verbSpecificity = this.getVerbSpecificity(verbType);
|
|||
|
|
weight += verbSpecificity * 0.1;
|
|||
|
|
return Math.min(weight, 1.0);
|
|||
|
|
}
|
|||
|
|
/**
|
|||
|
|
* Generate Neural Insights - The Intelligence Layer
|
|||
|
|
*/
|
|||
|
|
async generateNeuralInsights(entities, relationships) {
|
|||
|
|
const insights = [];
|
|||
|
|
// Detect hierarchies
|
|||
|
|
const hierarchies = this.detectHierarchies(relationships);
|
|||
|
|
hierarchies.forEach(hierarchy => {
|
|||
|
|
insights.push({
|
|||
|
|
type: 'hierarchy',
|
|||
|
|
description: `Detected ${hierarchy.type} hierarchy with ${hierarchy.levels} levels`,
|
|||
|
|
confidence: hierarchy.confidence,
|
|||
|
|
affectedEntities: hierarchy.entities,
|
|||
|
|
recommendation: `Consider visualizing the ${hierarchy.type} structure`
|
|||
|
|
});
|
|||
|
|
});
|
|||
|
|
// Detect clusters
|
|||
|
|
const clusters = this.detectClusters(entities, relationships);
|
|||
|
|
clusters.forEach(cluster => {
|
|||
|
|
insights.push({
|
|||
|
|
type: 'cluster',
|
|||
|
|
description: `Found cluster of ${cluster.size} ${cluster.primaryType} entities`,
|
|||
|
|
confidence: cluster.confidence,
|
|||
|
|
affectedEntities: cluster.entities,
|
|||
|
|
recommendation: `These ${cluster.primaryType}s might form a natural grouping`
|
|||
|
|
});
|
|||
|
|
});
|
|||
|
|
// Detect patterns
|
|||
|
|
const patterns = this.detectPatterns(relationships);
|
|||
|
|
patterns.forEach(pattern => {
|
|||
|
|
insights.push({
|
|||
|
|
type: 'pattern',
|
|||
|
|
description: `Common relationship pattern: ${pattern.description}`,
|
|||
|
|
confidence: pattern.confidence,
|
|||
|
|
affectedEntities: pattern.entities,
|
|||
|
|
recommendation: pattern.recommendation
|
|||
|
|
});
|
|||
|
|
});
|
|||
|
|
return insights;
|
|||
|
|
}
|
|||
|
|
/**
|
|||
|
|
* Helper methods for the neural system
|
|||
|
|
*/
|
|||
|
|
extractMainText(data) {
|
|||
|
|
// Extract the most relevant text from a data object
|
|||
|
|
const textFields = ['name', 'title', 'description', 'content', 'text', 'label'];
|
|||
|
|
for (const field of textFields) {
|
|||
|
|
if (data[field] && typeof data[field] === 'string') {
|
|||
|
|
return data[field];
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
// Fallback: concatenate all string values
|
|||
|
|
return Object.values(data)
|
|||
|
|
.filter(v => typeof v === 'string')
|
|||
|
|
.join(' ')
|
|||
|
|
.substring(0, 200); // Limit length
|
|||
|
|
}
|
|||
|
|
generateSmartId(data, nounType, index) {
|
|||
|
|
const mainText = this.extractMainText(data);
|
|||
|
|
const cleanText = mainText.toLowerCase().replace(/[^a-z0-9]/g, '_').substring(0, 20);
|
|||
|
|
return `${nounType}_${cleanText}_${index}`;
|
|||
|
|
}
|
|||
|
|
extractRelationshipContext(source, target, allData) {
|
|||
|
|
// Extract context for relationship detection
|
|||
|
|
return [
|
|||
|
|
this.extractMainText(source),
|
|||
|
|
this.extractMainText(target),
|
|||
|
|
// Add more contextual information
|
|||
|
|
].join(' ');
|
|||
|
|
}
|
|||
|
|
calculateTypeCompatibility(sourceType, targetType, verbType) {
|
|||
|
|
// Define type compatibility matrix for relationships
|
|||
|
|
const compatibilityMatrix = {
|
|||
|
|
[NounType.Person]: {
|
|||
|
|
[NounType.Organization]: [VerbType.MemberOf, VerbType.WorksWith],
|
|||
|
|
[NounType.Project]: [VerbType.WorksWith, VerbType.Creates],
|
|||
|
|
[NounType.Person]: [VerbType.WorksWith, VerbType.Mentors, VerbType.ReportsTo]
|
|||
|
|
}
|
|||
|
|
// Add more compatibility rules
|
|||
|
|
};
|
|||
|
|
const sourceCompatibility = compatibilityMatrix[sourceType];
|
|||
|
|
if (sourceCompatibility && sourceCompatibility[targetType]) {
|
|||
|
|
return sourceCompatibility[targetType].includes(verbType) ? 1.0 : 0.3;
|
|||
|
|
}
|
|||
|
|
return 0.5; // Default compatibility
|
|||
|
|
}
|
|||
|
|
getVerbSpecificity(verbType) {
|
|||
|
|
// More specific verbs get higher scores
|
|||
|
|
const specificityScores = {
|
|||
|
|
[VerbType.RelatedTo]: 0.1, // Very generic
|
|||
|
|
[VerbType.WorksWith]: 0.7, // Specific
|
|||
|
|
[VerbType.Mentors]: 0.9, // Very specific
|
|||
|
|
[VerbType.ReportsTo]: 0.9, // Very specific
|
|||
|
|
[VerbType.Supervises]: 0.9 // Very specific
|
|||
|
|
};
|
|||
|
|
return specificityScores[verbType] || 0.5;
|
|||
|
|
}
|
|||
|
|
getRelevantFields(data, nounType) {
|
|||
|
|
// Implementation for finding relevant fields
|
|||
|
|
return [];
|
|||
|
|
}
|
|||
|
|
getMatchedPatterns(text, data, nounType) {
|
|||
|
|
// Implementation for finding matched patterns
|
|||
|
|
return [];
|
|||
|
|
}
|
|||
|
|
pruneRelationships(relationships) {
|
|||
|
|
// Remove duplicates and low-confidence relationships
|
|||
|
|
return relationships
|
|||
|
|
.sort((a, b) => b.confidence - a.confidence)
|
|||
|
|
.slice(0, 1000); // Limit to top 1000 relationships
|
|||
|
|
}
|
|||
|
|
detectHierarchies(relationships) {
|
|||
|
|
// Detect hierarchical structures
|
|||
|
|
return [];
|
|||
|
|
}
|
|||
|
|
detectClusters(entities, relationships) {
|
|||
|
|
// Detect entity clusters
|
|||
|
|
return [];
|
|||
|
|
}
|
|||
|
|
detectPatterns(relationships) {
|
|||
|
|
// Detect relationship patterns
|
|||
|
|
return [];
|
|||
|
|
}
|
|||
|
|
calculateOverallConfidence(entities, relationships) {
|
|||
|
|
if (entities.length === 0)
|
|||
|
|
return 0;
|
|||
|
|
const entityConfidence = entities.reduce((sum, e) => sum + e.confidence, 0) / entities.length;
|
|||
|
|
if (relationships.length === 0)
|
|||
|
|
return entityConfidence;
|
|||
|
|
const relationshipConfidence = relationships.reduce((sum, r) => sum + r.confidence, 0) / relationships.length;
|
|||
|
|
return (entityConfidence + relationshipConfidence) / 2;
|
|||
|
|
}
|
|||
|
|
async storeNeuralAnalysis(analysis) {
|
|||
|
|
// Store the full analysis result for later retrieval by Neural Import or other systems
|
|||
|
|
// This could be stored in the brainy instance metadata or a separate analysis store
|
|||
|
|
}
|
|||
|
|
getDataTypeFromPath(filePath) {
|
|||
|
|
const ext = path.extname(filePath).toLowerCase();
|
|||
|
|
switch (ext) {
|
|||
|
|
case '.json': return 'json';
|
|||
|
|
case '.csv': return 'csv';
|
|||
|
|
case '.yaml':
|
|||
|
|
case '.yml': return 'yaml';
|
|||
|
|
case '.txt': return 'text';
|
|||
|
|
default: return 'text';
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
async generateRelationshipReasoning(source, target, verbType, context) {
|
|||
|
|
return `Neural analysis detected ${verbType} relationship based on semantic context`;
|
|||
|
|
}
|
|||
|
|
extractRelationshipMetadata(sourceData, targetData, verbType) {
|
|||
|
|
return {
|
|||
|
|
sourceType: typeof sourceData,
|
|||
|
|
targetType: typeof targetData,
|
|||
|
|
detectedBy: 'neural-import-sense',
|
|||
|
|
timestamp: new Date().toISOString()
|
|||
|
|
};
|
|||
|
|
}
|
|||
|
|
/**
|
|||
|
|
* Assess data quality metrics
|
|||
|
|
*/
|
|||
|
|
assessDataQuality(parsedData, analysis) {
|
|||
|
|
// Completeness: ratio of fields with data
|
|||
|
|
let totalFields = 0;
|
|||
|
|
let filledFields = 0;
|
|||
|
|
parsedData.forEach(item => {
|
|||
|
|
const fields = Object.keys(item);
|
|||
|
|
totalFields += fields.length;
|
|||
|
|
filledFields += fields.filter(field => item[field] !== null &&
|
|||
|
|
item[field] !== undefined &&
|
|||
|
|
item[field] !== '').length;
|
|||
|
|
});
|
|||
|
|
const completeness = totalFields > 0 ? filledFields / totalFields : 0;
|
|||
|
|
// Consistency: variance in field structure
|
|||
|
|
const fieldSets = parsedData.map(item => new Set(Object.keys(item)));
|
|||
|
|
const allFields = new Set(fieldSets.flatMap(set => Array.from(set)));
|
|||
|
|
let consistencyScore = 0;
|
|||
|
|
if (fieldSets.length > 0) {
|
|||
|
|
consistencyScore = Array.from(allFields).reduce((score, field) => {
|
|||
|
|
const hasField = fieldSets.filter(set => set.has(field)).length;
|
|||
|
|
return score + (hasField / fieldSets.length);
|
|||
|
|
}, 0) / allFields.size;
|
|||
|
|
}
|
|||
|
|
// Accuracy: average confidence of detected entities
|
|||
|
|
const accuracy = analysis.detectedEntities.length > 0 ?
|
|||
|
|
analysis.detectedEntities.reduce((sum, e) => sum + e.confidence, 0) / analysis.detectedEntities.length :
|
|||
|
|
0;
|
|||
|
|
return {
|
|||
|
|
completeness,
|
|||
|
|
consistency: consistencyScore,
|
|||
|
|
accuracy
|
|||
|
|
};
|
|||
|
|
}
|
|||
|
|
/**
|
|||
|
|
* Generate recommendations based on analysis
|
|||
|
|
*/
|
|||
|
|
generateRecommendations(parsedData, analysis, entityTypes, relationshipTypes) {
|
|||
|
|
const recommendations = [];
|
|||
|
|
// Low entity confidence recommendations
|
|||
|
|
const lowConfidenceEntities = entityTypes.filter(et => et.confidence < 0.7);
|
|||
|
|
if (lowConfidenceEntities.length > 0) {
|
|||
|
|
recommendations.push(`Consider improving field names for ${lowConfidenceEntities.map(e => e.type).join(', ')} entities`);
|
|||
|
|
}
|
|||
|
|
// Missing relationships recommendations
|
|||
|
|
if (relationshipTypes.length === 0 && entityTypes.length > 1) {
|
|||
|
|
recommendations.push('Add fields that describe how entities relate to each other');
|
|||
|
|
}
|
|||
|
|
// Data structure recommendations
|
|||
|
|
if (parsedData.length > 0) {
|
|||
|
|
const firstItem = parsedData[0];
|
|||
|
|
const fieldCount = Object.keys(firstItem).length;
|
|||
|
|
if (fieldCount < 3) {
|
|||
|
|
recommendations.push('Consider adding more descriptive fields to each entity');
|
|||
|
|
}
|
|||
|
|
if (fieldCount > 20) {
|
|||
|
|
recommendations.push('Consider grouping related fields or splitting complex entities');
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
// Entity distribution recommendations
|
|||
|
|
const dominantEntityType = entityTypes.reduce((max, current) => current.count > max.count ? current : max, entityTypes[0] || { count: 0 });
|
|||
|
|
if (dominantEntityType && dominantEntityType.count > parsedData.length * 0.8) {
|
|||
|
|
recommendations.push(`Consider diversifying entity types - ${dominantEntityType.type} dominates the dataset`);
|
|||
|
|
}
|
|||
|
|
// Relationship quality recommendations
|
|||
|
|
const lowWeightRelationships = relationshipTypes.filter(rt => rt.confidence < 0.6);
|
|||
|
|
if (lowWeightRelationships.length > 0) {
|
|||
|
|
recommendations.push('Consider adding more contextual information to strengthen relationship detection');
|
|||
|
|
}
|
|||
|
|
return recommendations;
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
//# sourceMappingURL=neuralImport.js.map
|