Initial commit: Brainy - Multi-Dimensional AI Database
Open source vector database with HNSW indexing, graph relationships, and metadata facets. Features CLI with professional augmentation registry integration for discovering extensions and capabilities.
This commit is contained in:
commit
f8c45f2d8d
448 changed files with 103294 additions and 0 deletions
750
dist/augmentations/neuralImport.js
vendored
Normal file
750
dist/augmentations/neuralImport.js
vendored
Normal file
|
|
@ -0,0 +1,750 @@
|
|||
/**
|
||||
* Neural Import Augmentation - AI-Powered Data Understanding
|
||||
*
|
||||
* 🧠 Built-in AI augmentation for intelligent data processing
|
||||
* ⚛️ Always free, always included, always enabled
|
||||
*
|
||||
* This is the default AI-powered augmentation that comes with every Brainy installation.
|
||||
* It provides intelligent data understanding, entity detection, and relationship analysis.
|
||||
*/
|
||||
import { NounType, VerbType } from '../types/graphTypes.js';
|
||||
import * as fs from '../universal/fs.js';
|
||||
import * as path from '../universal/path.js';
|
||||
/**
|
||||
* Neural Import SENSE Augmentation - The Brain's Perceptual System
|
||||
*/
|
||||
export class NeuralImportAugmentation {
|
||||
constructor(brainy, config = {}) {
|
||||
this.name = 'neural-import';
|
||||
this.description = 'Built-in AI-powered data understanding and entity detection';
|
||||
this.enabled = true;
|
||||
this.brainy = brainy;
|
||||
this.config = {
|
||||
confidenceThreshold: 0.7,
|
||||
enableWeights: true,
|
||||
skipDuplicates: true,
|
||||
...config
|
||||
};
|
||||
}
|
||||
async initialize() {
|
||||
// Initialize the cortex analysis system
|
||||
console.log('🧠 Neural Import augmentation initialized');
|
||||
}
|
||||
async shutDown() {
|
||||
console.log('🧠 Neural Import SENSE augmentation shut down');
|
||||
}
|
||||
async getStatus() {
|
||||
return this.enabled ? 'active' : 'inactive';
|
||||
}
|
||||
/**
|
||||
* Process raw data into structured nouns and verbs using neural analysis
|
||||
*/
|
||||
async processRawData(rawData, dataType, options) {
|
||||
try {
|
||||
// Merge options with config
|
||||
const mergedConfig = { ...this.config, ...options };
|
||||
// Parse the raw data based on type
|
||||
const parsedData = await this.parseRawData(rawData, dataType);
|
||||
// Perform neural analysis
|
||||
const analysis = await this.performNeuralAnalysis(parsedData, mergedConfig);
|
||||
// Extract nouns and verbs for the ISenseAugmentation interface
|
||||
const nouns = analysis.detectedEntities.map(entity => entity.suggestedId);
|
||||
const verbs = analysis.detectedRelationships.map(rel => `${rel.sourceId}->${rel.verbType}->${rel.targetId}`);
|
||||
// Store the full analysis for later retrieval
|
||||
await this.storeNeuralAnalysis(analysis);
|
||||
return {
|
||||
success: true,
|
||||
data: {
|
||||
nouns,
|
||||
verbs,
|
||||
confidence: analysis.confidence,
|
||||
insights: analysis.insights.map((insight) => ({
|
||||
type: insight.type,
|
||||
description: insight.description,
|
||||
confidence: insight.confidence
|
||||
})),
|
||||
metadata: {
|
||||
detectedEntities: analysis.detectedEntities.length,
|
||||
detectedRelationships: analysis.detectedRelationships.length,
|
||||
timestamp: new Date().toISOString(),
|
||||
augmentation: 'neural-import-sense'
|
||||
}
|
||||
}
|
||||
};
|
||||
}
|
||||
catch (error) {
|
||||
return {
|
||||
success: false,
|
||||
data: { nouns: [], verbs: [] },
|
||||
error: error instanceof Error ? error.message : 'Neural analysis failed'
|
||||
};
|
||||
}
|
||||
}
|
||||
/**
|
||||
* Listen to real-time data feeds and process them
|
||||
*/
|
||||
async listenToFeed(feedUrl, callback) {
|
||||
// For file-based feeds, watch for changes
|
||||
if (feedUrl.startsWith('file://')) {
|
||||
const filePath = feedUrl.replace('file://', '');
|
||||
// Watch file for changes using Node.js fs.watch
|
||||
const fsWatch = require('fs');
|
||||
const watcher = fsWatch.watch(filePath, async (eventType) => {
|
||||
if (eventType === 'change') {
|
||||
try {
|
||||
const fileContent = await fs.readFile(filePath);
|
||||
const result = await this.processRawData(fileContent, this.getDataTypeFromPath(filePath));
|
||||
if (result.success) {
|
||||
callback({
|
||||
nouns: result.data.nouns,
|
||||
verbs: result.data.verbs,
|
||||
confidence: result.data.confidence
|
||||
});
|
||||
}
|
||||
}
|
||||
catch (error) {
|
||||
console.error('Neural Import feed error:', error);
|
||||
}
|
||||
}
|
||||
});
|
||||
return;
|
||||
}
|
||||
// For other feed types, implement appropriate listeners
|
||||
console.log(`🧠 Neural Import listening to feed: ${feedUrl}`);
|
||||
}
|
||||
/**
|
||||
* Analyze data structure without processing (preview mode)
|
||||
*/
|
||||
async analyzeStructure(rawData, dataType, options) {
|
||||
try {
|
||||
// Parse the raw data
|
||||
const parsedData = await this.parseRawData(rawData, dataType);
|
||||
// Perform lightweight analysis for structure detection
|
||||
const analysis = await this.performNeuralAnalysis(parsedData, { ...this.config, ...options });
|
||||
// Summarize entity types
|
||||
const entityTypeCounts = new Map();
|
||||
analysis.detectedEntities.forEach(entity => {
|
||||
const existing = entityTypeCounts.get(entity.nounType) || { count: 0, totalConfidence: 0 };
|
||||
entityTypeCounts.set(entity.nounType, {
|
||||
count: existing.count + 1,
|
||||
totalConfidence: existing.totalConfidence + entity.confidence
|
||||
});
|
||||
});
|
||||
const entityTypes = Array.from(entityTypeCounts.entries()).map(([type, stats]) => ({
|
||||
type,
|
||||
count: stats.count,
|
||||
confidence: stats.totalConfidence / stats.count
|
||||
}));
|
||||
// Summarize relationship types
|
||||
const relationshipTypeCounts = new Map();
|
||||
analysis.detectedRelationships.forEach(rel => {
|
||||
const existing = relationshipTypeCounts.get(rel.verbType) || { count: 0, totalConfidence: 0 };
|
||||
relationshipTypeCounts.set(rel.verbType, {
|
||||
count: existing.count + 1,
|
||||
totalConfidence: existing.totalConfidence + rel.confidence
|
||||
});
|
||||
});
|
||||
const relationshipTypes = Array.from(relationshipTypeCounts.entries()).map(([type, stats]) => ({
|
||||
type,
|
||||
count: stats.count,
|
||||
confidence: stats.totalConfidence / stats.count
|
||||
}));
|
||||
// Assess data quality
|
||||
const dataQuality = this.assessDataQuality(parsedData, analysis);
|
||||
// Generate recommendations
|
||||
const recommendations = this.generateRecommendations(parsedData, analysis, entityTypes, relationshipTypes);
|
||||
return {
|
||||
success: true,
|
||||
data: {
|
||||
entityTypes,
|
||||
relationshipTypes,
|
||||
dataQuality,
|
||||
recommendations
|
||||
}
|
||||
};
|
||||
}
|
||||
catch (error) {
|
||||
return {
|
||||
success: false,
|
||||
data: {
|
||||
entityTypes: [],
|
||||
relationshipTypes: [],
|
||||
dataQuality: { completeness: 0, consistency: 0, accuracy: 0 },
|
||||
recommendations: []
|
||||
},
|
||||
error: error instanceof Error ? error.message : 'Structure analysis failed'
|
||||
};
|
||||
}
|
||||
}
|
||||
/**
|
||||
* Validate data compatibility with current knowledge base
|
||||
*/
|
||||
async validateCompatibility(rawData, dataType) {
|
||||
try {
|
||||
// Parse the raw data
|
||||
const parsedData = await this.parseRawData(rawData, dataType);
|
||||
// Perform neural analysis
|
||||
const analysis = await this.performNeuralAnalysis(parsedData);
|
||||
const issues = [];
|
||||
const suggestions = [];
|
||||
// Check for low confidence entities
|
||||
const lowConfidenceEntities = analysis.detectedEntities.filter((e) => e.confidence < 0.5);
|
||||
if (lowConfidenceEntities.length > 0) {
|
||||
issues.push({
|
||||
type: 'confidence',
|
||||
description: `${lowConfidenceEntities.length} entities have low confidence scores`,
|
||||
severity: 'medium'
|
||||
});
|
||||
suggestions.push('Consider reviewing field names and data structure for better entity detection');
|
||||
}
|
||||
// Check for missing relationships
|
||||
if (analysis.detectedRelationships.length === 0 && analysis.detectedEntities.length > 1) {
|
||||
issues.push({
|
||||
type: 'relationships',
|
||||
description: 'No relationships detected between entities',
|
||||
severity: 'low'
|
||||
});
|
||||
suggestions.push('Consider adding contextual fields that describe entity relationships');
|
||||
}
|
||||
// Check for data type compatibility
|
||||
const supportedTypes = ['json', 'csv', 'yaml', 'text'];
|
||||
if (!supportedTypes.includes(dataType.toLowerCase())) {
|
||||
issues.push({
|
||||
type: 'format',
|
||||
description: `Data type '${dataType}' may not be fully supported`,
|
||||
severity: 'high'
|
||||
});
|
||||
suggestions.push(`Convert data to one of: ${supportedTypes.join(', ')}`);
|
||||
}
|
||||
// Check for data completeness
|
||||
const incompleteEntities = analysis.detectedEntities.filter((e) => !e.originalData || Object.keys(e.originalData).length < 2);
|
||||
if (incompleteEntities.length > 0) {
|
||||
issues.push({
|
||||
type: 'completeness',
|
||||
description: `${incompleteEntities.length} entities have insufficient data`,
|
||||
severity: 'medium'
|
||||
});
|
||||
suggestions.push('Ensure each entity has multiple descriptive fields');
|
||||
}
|
||||
const compatible = issues.filter(i => i.severity === 'high').length === 0;
|
||||
return {
|
||||
success: true,
|
||||
data: {
|
||||
compatible,
|
||||
issues,
|
||||
suggestions
|
||||
}
|
||||
};
|
||||
}
|
||||
catch (error) {
|
||||
return {
|
||||
success: false,
|
||||
data: {
|
||||
compatible: false,
|
||||
issues: [{
|
||||
type: 'error',
|
||||
description: error instanceof Error ? error.message : 'Validation failed',
|
||||
severity: 'high'
|
||||
}],
|
||||
suggestions: []
|
||||
},
|
||||
error: error instanceof Error ? error.message : 'Compatibility validation failed'
|
||||
};
|
||||
}
|
||||
}
|
||||
/**
|
||||
* Get the full neural analysis result (custom method for Cortex integration)
|
||||
*/
|
||||
async getNeuralAnalysis(rawData, dataType) {
|
||||
const parsedData = await this.parseRawData(rawData, dataType);
|
||||
return await this.performNeuralAnalysis(parsedData);
|
||||
}
|
||||
/**
|
||||
* Parse raw data based on type
|
||||
*/
|
||||
async parseRawData(rawData, dataType) {
|
||||
const content = typeof rawData === 'string' ? rawData : rawData.toString('utf8');
|
||||
switch (dataType.toLowerCase()) {
|
||||
case 'json':
|
||||
const jsonData = JSON.parse(content);
|
||||
return Array.isArray(jsonData) ? jsonData : [jsonData];
|
||||
case 'csv':
|
||||
return this.parseCSV(content);
|
||||
case 'yaml':
|
||||
case 'yml':
|
||||
// For now, basic YAML support - in full implementation would use yaml parser
|
||||
return JSON.parse(content); // Placeholder
|
||||
case 'txt':
|
||||
case 'text':
|
||||
// Split text into sentences/paragraphs for analysis
|
||||
return content.split(/\n+/).filter(line => line.trim()).map(line => ({ text: line }));
|
||||
default:
|
||||
throw new Error(`Unsupported data type: ${dataType}`);
|
||||
}
|
||||
}
|
||||
/**
|
||||
* Basic CSV parser
|
||||
*/
|
||||
parseCSV(content) {
|
||||
const lines = content.split('\n').filter(line => line.trim());
|
||||
if (lines.length < 2)
|
||||
return [];
|
||||
const headers = lines[0].split(',').map(h => h.trim().replace(/"/g, ''));
|
||||
const data = [];
|
||||
for (let i = 1; i < lines.length; i++) {
|
||||
const values = lines[i].split(',').map(v => v.trim().replace(/"/g, ''));
|
||||
const row = {};
|
||||
headers.forEach((header, index) => {
|
||||
row[header] = values[index] || '';
|
||||
});
|
||||
data.push(row);
|
||||
}
|
||||
return data;
|
||||
}
|
||||
/**
|
||||
* Perform neural analysis on parsed data
|
||||
*/
|
||||
async performNeuralAnalysis(parsedData, config = this.config) {
|
||||
// Phase 1: Neural Entity Detection
|
||||
const detectedEntities = await this.detectEntitiesWithNeuralAnalysis(parsedData, config);
|
||||
// Phase 2: Neural Relationship Detection
|
||||
const detectedRelationships = await this.detectRelationshipsWithNeuralAnalysis(detectedEntities, parsedData, config);
|
||||
// Phase 3: Neural Insights Generation
|
||||
const insights = await this.generateNeuralInsights(detectedEntities, detectedRelationships);
|
||||
// Phase 4: Confidence Scoring
|
||||
const overallConfidence = this.calculateOverallConfidence(detectedEntities, detectedRelationships);
|
||||
return {
|
||||
detectedEntities,
|
||||
detectedRelationships,
|
||||
confidence: overallConfidence,
|
||||
insights
|
||||
};
|
||||
}
|
||||
/**
|
||||
* Neural Entity Detection - The Core AI Engine
|
||||
*/
|
||||
async detectEntitiesWithNeuralAnalysis(rawData, config = this.config) {
|
||||
const entities = [];
|
||||
const nounTypes = Object.values(NounType);
|
||||
for (const [index, dataItem] of rawData.entries()) {
|
||||
const mainText = this.extractMainText(dataItem);
|
||||
const detections = [];
|
||||
// Test against all noun types using semantic similarity
|
||||
for (const nounType of nounTypes) {
|
||||
const confidence = await this.calculateEntityTypeConfidence(mainText, dataItem, nounType);
|
||||
if (confidence >= config.confidenceThreshold - 0.2) { // Allow slightly lower for alternatives
|
||||
const reasoning = await this.generateEntityReasoning(mainText, dataItem, nounType);
|
||||
detections.push({ type: nounType, confidence, reasoning });
|
||||
}
|
||||
}
|
||||
if (detections.length > 0) {
|
||||
// Sort by confidence
|
||||
detections.sort((a, b) => b.confidence - a.confidence);
|
||||
const primaryType = detections[0];
|
||||
const alternatives = detections.slice(1, 3); // Top 2 alternatives
|
||||
entities.push({
|
||||
originalData: dataItem,
|
||||
nounType: primaryType.type,
|
||||
confidence: primaryType.confidence,
|
||||
suggestedId: this.generateSmartId(dataItem, primaryType.type, index),
|
||||
reasoning: primaryType.reasoning,
|
||||
alternativeTypes: alternatives
|
||||
});
|
||||
}
|
||||
}
|
||||
return entities;
|
||||
}
|
||||
/**
|
||||
* Calculate entity type confidence using AI
|
||||
*/
|
||||
async calculateEntityTypeConfidence(text, data, nounType) {
|
||||
// Base semantic similarity using search
|
||||
const searchResults = await this.brainy.search(text + ' ' + nounType, 1);
|
||||
const textSimilarity = searchResults.length > 0 ? searchResults[0].score : 0.5;
|
||||
// Field-based confidence boost
|
||||
const fieldBoost = this.calculateFieldBasedConfidence(data, nounType);
|
||||
// Pattern-based confidence boost
|
||||
const patternBoost = this.calculatePatternBasedConfidence(text, data, nounType);
|
||||
// Combine confidences with weights
|
||||
const combined = (textSimilarity * 0.5) + (fieldBoost * 0.3) + (patternBoost * 0.2);
|
||||
return Math.min(combined, 1.0);
|
||||
}
|
||||
/**
|
||||
* Field-based confidence calculation
|
||||
*/
|
||||
calculateFieldBasedConfidence(data, nounType) {
|
||||
const fields = Object.keys(data);
|
||||
let boost = 0;
|
||||
// Field patterns that boost confidence for specific noun types
|
||||
const fieldPatterns = {
|
||||
[NounType.Person]: ['name', 'email', 'phone', 'age', 'firstname', 'lastname', 'employee'],
|
||||
[NounType.Organization]: ['company', 'organization', 'corp', 'inc', 'ltd', 'department', 'team'],
|
||||
[NounType.Project]: ['project', 'task', 'deadline', 'status', 'milestone', 'deliverable'],
|
||||
[NounType.Location]: ['address', 'city', 'country', 'state', 'zip', 'location', 'coordinates'],
|
||||
[NounType.Product]: ['product', 'price', 'sku', 'inventory', 'category', 'brand'],
|
||||
[NounType.Event]: ['date', 'time', 'venue', 'event', 'meeting', 'conference', 'schedule']
|
||||
};
|
||||
const relevantPatterns = fieldPatterns[nounType] || [];
|
||||
for (const field of fields) {
|
||||
for (const pattern of relevantPatterns) {
|
||||
if (field.toLowerCase().includes(pattern)) {
|
||||
boost += 0.1;
|
||||
}
|
||||
}
|
||||
}
|
||||
return Math.min(boost, 0.5);
|
||||
}
|
||||
/**
|
||||
* Pattern-based confidence calculation
|
||||
*/
|
||||
calculatePatternBasedConfidence(text, data, nounType) {
|
||||
let boost = 0;
|
||||
// Content patterns that indicate entity types
|
||||
const patterns = {
|
||||
[NounType.Person]: [
|
||||
/@.*\.com/i, // Email pattern
|
||||
/\b[A-Z][a-z]+ [A-Z][a-z]+\b/, // Name pattern
|
||||
/Mr\.|Mrs\.|Dr\.|Prof\./i // Title pattern
|
||||
],
|
||||
[NounType.Organization]: [
|
||||
/\bInc\.|Corp\.|LLC\.|Ltd\./i, // Corporate suffixes
|
||||
/Company|Corporation|Enterprise/i
|
||||
],
|
||||
[NounType.Location]: [
|
||||
/\b\d{5}(-\d{4})?\b/, // ZIP code
|
||||
/Street|Ave|Road|Blvd/i
|
||||
]
|
||||
};
|
||||
const relevantPatterns = patterns[nounType] || [];
|
||||
for (const pattern of relevantPatterns) {
|
||||
if (pattern.test(text)) {
|
||||
boost += 0.15;
|
||||
}
|
||||
}
|
||||
return Math.min(boost, 0.3);
|
||||
}
|
||||
/**
|
||||
* Generate reasoning for entity type selection
|
||||
*/
|
||||
async generateEntityReasoning(text, data, nounType) {
|
||||
const reasons = [];
|
||||
// Semantic similarity reason
|
||||
const searchResults = await this.brainy.search(text + ' ' + nounType, 1);
|
||||
const similarity = searchResults.length > 0 ? searchResults[0].score : 0.5;
|
||||
if (similarity > 0.7) {
|
||||
reasons.push(`High semantic similarity (${(similarity * 100).toFixed(1)}%)`);
|
||||
}
|
||||
// Field-based reasons
|
||||
const relevantFields = this.getRelevantFields(data, nounType);
|
||||
if (relevantFields.length > 0) {
|
||||
reasons.push(`Contains ${nounType}-specific fields: ${relevantFields.join(', ')}`);
|
||||
}
|
||||
// Pattern-based reasons
|
||||
const matchedPatterns = this.getMatchedPatterns(text, data, nounType);
|
||||
if (matchedPatterns.length > 0) {
|
||||
reasons.push(`Matches ${nounType} patterns: ${matchedPatterns.join(', ')}`);
|
||||
}
|
||||
return reasons.length > 0 ? reasons.join('; ') : 'General semantic match';
|
||||
}
|
||||
/**
|
||||
* Neural Relationship Detection
|
||||
*/
|
||||
async detectRelationshipsWithNeuralAnalysis(entities, rawData, config = this.config) {
|
||||
const relationships = [];
|
||||
const verbTypes = Object.values(VerbType);
|
||||
// For each pair of entities, test relationship possibilities
|
||||
for (let i = 0; i < entities.length; i++) {
|
||||
for (let j = i + 1; j < entities.length; j++) {
|
||||
const sourceEntity = entities[i];
|
||||
const targetEntity = entities[j];
|
||||
// Extract context for relationship detection
|
||||
const context = this.extractRelationshipContext(sourceEntity.originalData, targetEntity.originalData, rawData);
|
||||
// Test all verb types
|
||||
for (const verbType of verbTypes) {
|
||||
const confidence = await this.calculateRelationshipConfidence(sourceEntity, targetEntity, verbType, context);
|
||||
if (confidence >= config.confidenceThreshold - 0.1) { // Slightly lower threshold for relationships
|
||||
const weight = config.enableWeights ?
|
||||
this.calculateRelationshipWeight(sourceEntity, targetEntity, verbType, context) :
|
||||
0.5;
|
||||
const reasoning = await this.generateRelationshipReasoning(sourceEntity, targetEntity, verbType, context);
|
||||
relationships.push({
|
||||
sourceId: sourceEntity.suggestedId,
|
||||
targetId: targetEntity.suggestedId,
|
||||
verbType,
|
||||
confidence,
|
||||
weight,
|
||||
reasoning,
|
||||
context,
|
||||
metadata: this.extractRelationshipMetadata(sourceEntity.originalData, targetEntity.originalData, verbType)
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
// Sort by confidence and remove duplicates/conflicts
|
||||
return this.pruneRelationships(relationships);
|
||||
}
|
||||
/**
|
||||
* Calculate relationship confidence
|
||||
*/
|
||||
async calculateRelationshipConfidence(source, target, verbType, context) {
|
||||
// Semantic similarity between entities and verb type
|
||||
const relationshipText = `${this.extractMainText(source.originalData)} ${verbType} ${this.extractMainText(target.originalData)}`;
|
||||
const directResults = await this.brainy.search(relationshipText, 1);
|
||||
const directSimilarity = directResults.length > 0 ? directResults[0].score : 0.5;
|
||||
// Context-based similarity
|
||||
const contextResults = await this.brainy.search(context + ' ' + verbType, 1);
|
||||
const contextSimilarity = contextResults.length > 0 ? contextResults[0].score : 0.5;
|
||||
// Entity type compatibility
|
||||
const typeCompatibility = this.calculateTypeCompatibility(source.nounType, target.nounType, verbType);
|
||||
// Combine with weights
|
||||
return (directSimilarity * 0.4) + (contextSimilarity * 0.4) + (typeCompatibility * 0.2);
|
||||
}
|
||||
/**
|
||||
* Calculate relationship weight/strength
|
||||
*/
|
||||
calculateRelationshipWeight(source, target, verbType, context) {
|
||||
let weight = 0.5; // Base weight
|
||||
// Context richness (more descriptive = stronger)
|
||||
const contextWords = context.split(' ').length;
|
||||
weight += Math.min(contextWords / 20, 0.2);
|
||||
// Entity importance (higher confidence entities = stronger relationships)
|
||||
const avgEntityConfidence = (source.confidence + target.confidence) / 2;
|
||||
weight += avgEntityConfidence * 0.2;
|
||||
// Verb type specificity (more specific verbs = stronger)
|
||||
const verbSpecificity = this.getVerbSpecificity(verbType);
|
||||
weight += verbSpecificity * 0.1;
|
||||
return Math.min(weight, 1.0);
|
||||
}
|
||||
/**
|
||||
* Generate Neural Insights - The Intelligence Layer
|
||||
*/
|
||||
async generateNeuralInsights(entities, relationships) {
|
||||
const insights = [];
|
||||
// Detect hierarchies
|
||||
const hierarchies = this.detectHierarchies(relationships);
|
||||
hierarchies.forEach(hierarchy => {
|
||||
insights.push({
|
||||
type: 'hierarchy',
|
||||
description: `Detected ${hierarchy.type} hierarchy with ${hierarchy.levels} levels`,
|
||||
confidence: hierarchy.confidence,
|
||||
affectedEntities: hierarchy.entities,
|
||||
recommendation: `Consider visualizing the ${hierarchy.type} structure`
|
||||
});
|
||||
});
|
||||
// Detect clusters
|
||||
const clusters = this.detectClusters(entities, relationships);
|
||||
clusters.forEach(cluster => {
|
||||
insights.push({
|
||||
type: 'cluster',
|
||||
description: `Found cluster of ${cluster.size} ${cluster.primaryType} entities`,
|
||||
confidence: cluster.confidence,
|
||||
affectedEntities: cluster.entities,
|
||||
recommendation: `These ${cluster.primaryType}s might form a natural grouping`
|
||||
});
|
||||
});
|
||||
// Detect patterns
|
||||
const patterns = this.detectPatterns(relationships);
|
||||
patterns.forEach(pattern => {
|
||||
insights.push({
|
||||
type: 'pattern',
|
||||
description: `Common relationship pattern: ${pattern.description}`,
|
||||
confidence: pattern.confidence,
|
||||
affectedEntities: pattern.entities,
|
||||
recommendation: pattern.recommendation
|
||||
});
|
||||
});
|
||||
return insights;
|
||||
}
|
||||
/**
|
||||
* Helper methods for the neural system
|
||||
*/
|
||||
extractMainText(data) {
|
||||
// Extract the most relevant text from a data object
|
||||
const textFields = ['name', 'title', 'description', 'content', 'text', 'label'];
|
||||
for (const field of textFields) {
|
||||
if (data[field] && typeof data[field] === 'string') {
|
||||
return data[field];
|
||||
}
|
||||
}
|
||||
// Fallback: concatenate all string values
|
||||
return Object.values(data)
|
||||
.filter(v => typeof v === 'string')
|
||||
.join(' ')
|
||||
.substring(0, 200); // Limit length
|
||||
}
|
||||
generateSmartId(data, nounType, index) {
|
||||
const mainText = this.extractMainText(data);
|
||||
const cleanText = mainText.toLowerCase().replace(/[^a-z0-9]/g, '_').substring(0, 20);
|
||||
return `${nounType}_${cleanText}_${index}`;
|
||||
}
|
||||
extractRelationshipContext(source, target, allData) {
|
||||
// Extract context for relationship detection
|
||||
return [
|
||||
this.extractMainText(source),
|
||||
this.extractMainText(target),
|
||||
// Add more contextual information
|
||||
].join(' ');
|
||||
}
|
||||
calculateTypeCompatibility(sourceType, targetType, verbType) {
|
||||
// Define type compatibility matrix for relationships
|
||||
const compatibilityMatrix = {
|
||||
[NounType.Person]: {
|
||||
[NounType.Organization]: [VerbType.MemberOf, VerbType.WorksWith],
|
||||
[NounType.Project]: [VerbType.WorksWith, VerbType.Creates],
|
||||
[NounType.Person]: [VerbType.WorksWith, VerbType.Mentors, VerbType.ReportsTo]
|
||||
}
|
||||
// Add more compatibility rules
|
||||
};
|
||||
const sourceCompatibility = compatibilityMatrix[sourceType];
|
||||
if (sourceCompatibility && sourceCompatibility[targetType]) {
|
||||
return sourceCompatibility[targetType].includes(verbType) ? 1.0 : 0.3;
|
||||
}
|
||||
return 0.5; // Default compatibility
|
||||
}
|
||||
getVerbSpecificity(verbType) {
|
||||
// More specific verbs get higher scores
|
||||
const specificityScores = {
|
||||
[VerbType.RelatedTo]: 0.1, // Very generic
|
||||
[VerbType.WorksWith]: 0.7, // Specific
|
||||
[VerbType.Mentors]: 0.9, // Very specific
|
||||
[VerbType.ReportsTo]: 0.9, // Very specific
|
||||
[VerbType.Supervises]: 0.9 // Very specific
|
||||
};
|
||||
return specificityScores[verbType] || 0.5;
|
||||
}
|
||||
getRelevantFields(data, nounType) {
|
||||
// Implementation for finding relevant fields
|
||||
return [];
|
||||
}
|
||||
getMatchedPatterns(text, data, nounType) {
|
||||
// Implementation for finding matched patterns
|
||||
return [];
|
||||
}
|
||||
pruneRelationships(relationships) {
|
||||
// Remove duplicates and low-confidence relationships
|
||||
return relationships
|
||||
.sort((a, b) => b.confidence - a.confidence)
|
||||
.slice(0, 1000); // Limit to top 1000 relationships
|
||||
}
|
||||
detectHierarchies(relationships) {
|
||||
// Detect hierarchical structures
|
||||
return [];
|
||||
}
|
||||
detectClusters(entities, relationships) {
|
||||
// Detect entity clusters
|
||||
return [];
|
||||
}
|
||||
detectPatterns(relationships) {
|
||||
// Detect relationship patterns
|
||||
return [];
|
||||
}
|
||||
calculateOverallConfidence(entities, relationships) {
|
||||
if (entities.length === 0)
|
||||
return 0;
|
||||
const entityConfidence = entities.reduce((sum, e) => sum + e.confidence, 0) / entities.length;
|
||||
if (relationships.length === 0)
|
||||
return entityConfidence;
|
||||
const relationshipConfidence = relationships.reduce((sum, r) => sum + r.confidence, 0) / relationships.length;
|
||||
return (entityConfidence + relationshipConfidence) / 2;
|
||||
}
|
||||
async storeNeuralAnalysis(analysis) {
|
||||
// Store the full analysis result for later retrieval by Neural Import or other systems
|
||||
// This could be stored in the brainy instance metadata or a separate analysis store
|
||||
}
|
||||
getDataTypeFromPath(filePath) {
|
||||
const ext = path.extname(filePath).toLowerCase();
|
||||
switch (ext) {
|
||||
case '.json': return 'json';
|
||||
case '.csv': return 'csv';
|
||||
case '.yaml':
|
||||
case '.yml': return 'yaml';
|
||||
case '.txt': return 'text';
|
||||
default: return 'text';
|
||||
}
|
||||
}
|
||||
async generateRelationshipReasoning(source, target, verbType, context) {
|
||||
return `Neural analysis detected ${verbType} relationship based on semantic context`;
|
||||
}
|
||||
extractRelationshipMetadata(sourceData, targetData, verbType) {
|
||||
return {
|
||||
sourceType: typeof sourceData,
|
||||
targetType: typeof targetData,
|
||||
detectedBy: 'neural-import-sense',
|
||||
timestamp: new Date().toISOString()
|
||||
};
|
||||
}
|
||||
/**
|
||||
* Assess data quality metrics
|
||||
*/
|
||||
assessDataQuality(parsedData, analysis) {
|
||||
// Completeness: ratio of fields with data
|
||||
let totalFields = 0;
|
||||
let filledFields = 0;
|
||||
parsedData.forEach(item => {
|
||||
const fields = Object.keys(item);
|
||||
totalFields += fields.length;
|
||||
filledFields += fields.filter(field => item[field] !== null &&
|
||||
item[field] !== undefined &&
|
||||
item[field] !== '').length;
|
||||
});
|
||||
const completeness = totalFields > 0 ? filledFields / totalFields : 0;
|
||||
// Consistency: variance in field structure
|
||||
const fieldSets = parsedData.map(item => new Set(Object.keys(item)));
|
||||
const allFields = new Set(fieldSets.flatMap(set => Array.from(set)));
|
||||
let consistencyScore = 0;
|
||||
if (fieldSets.length > 0) {
|
||||
consistencyScore = Array.from(allFields).reduce((score, field) => {
|
||||
const hasField = fieldSets.filter(set => set.has(field)).length;
|
||||
return score + (hasField / fieldSets.length);
|
||||
}, 0) / allFields.size;
|
||||
}
|
||||
// Accuracy: average confidence of detected entities
|
||||
const accuracy = analysis.detectedEntities.length > 0 ?
|
||||
analysis.detectedEntities.reduce((sum, e) => sum + e.confidence, 0) / analysis.detectedEntities.length :
|
||||
0;
|
||||
return {
|
||||
completeness,
|
||||
consistency: consistencyScore,
|
||||
accuracy
|
||||
};
|
||||
}
|
||||
/**
|
||||
* Generate recommendations based on analysis
|
||||
*/
|
||||
generateRecommendations(parsedData, analysis, entityTypes, relationshipTypes) {
|
||||
const recommendations = [];
|
||||
// Low entity confidence recommendations
|
||||
const lowConfidenceEntities = entityTypes.filter(et => et.confidence < 0.7);
|
||||
if (lowConfidenceEntities.length > 0) {
|
||||
recommendations.push(`Consider improving field names for ${lowConfidenceEntities.map(e => e.type).join(', ')} entities`);
|
||||
}
|
||||
// Missing relationships recommendations
|
||||
if (relationshipTypes.length === 0 && entityTypes.length > 1) {
|
||||
recommendations.push('Add fields that describe how entities relate to each other');
|
||||
}
|
||||
// Data structure recommendations
|
||||
if (parsedData.length > 0) {
|
||||
const firstItem = parsedData[0];
|
||||
const fieldCount = Object.keys(firstItem).length;
|
||||
if (fieldCount < 3) {
|
||||
recommendations.push('Consider adding more descriptive fields to each entity');
|
||||
}
|
||||
if (fieldCount > 20) {
|
||||
recommendations.push('Consider grouping related fields or splitting complex entities');
|
||||
}
|
||||
}
|
||||
// Entity distribution recommendations
|
||||
const dominantEntityType = entityTypes.reduce((max, current) => current.count > max.count ? current : max, entityTypes[0] || { count: 0 });
|
||||
if (dominantEntityType && dominantEntityType.count > parsedData.length * 0.8) {
|
||||
recommendations.push(`Consider diversifying entity types - ${dominantEntityType.type} dominates the dataset`);
|
||||
}
|
||||
// Relationship quality recommendations
|
||||
const lowWeightRelationships = relationshipTypes.filter(rt => rt.confidence < 0.6);
|
||||
if (lowWeightRelationships.length > 0) {
|
||||
recommendations.push('Consider adding more contextual information to strengthen relationship detection');
|
||||
}
|
||||
return recommendations;
|
||||
}
|
||||
}
|
||||
//# sourceMappingURL=neuralImport.js.map
|
||||
Loading…
Add table
Add a link
Reference in a new issue