brainy/dist/augmentations/neuralImport.js

/**
 * Neural Import Augmentation - AI-Powered Data Understanding
 *
 * 🧠 Built-in AI augmentation for intelligent data processing
 * ⚛️ Always free, always included, always enabled
 *
 * This is the default AI-powered augmentation that comes with every Brainy installation.
 * It provides intelligent data understanding, entity detection, and relationship analysis.
 */
import { NounType, VerbType } from '../types/graphTypes.js';
import * as fs from '../universal/fs.js';
import * as path from '../universal/path.js';
/**
 * Neural Import SENSE Augmentation - The Brain's Perceptual System
 */
export class NeuralImportAugmentation {
    constructor(brainy, config = {}) {
        this.name = 'neural-import';
        this.description = 'Built-in AI-powered data understanding and entity detection';
        this.enabled = true;
        this.brainy = brainy;
        this.config = {
            confidenceThreshold: 0.7,
            enableWeights: true,
            skipDuplicates: true,
            ...config
        };
    }
    async initialize() {
        // Initialize the cortex analysis system
        console.log('🧠 Neural Import augmentation initialized');
    }
    async shutDown() {
        console.log('🧠 Neural Import SENSE augmentation shut down');
    }
    async getStatus() {
        return this.enabled ? 'active' : 'inactive';
    }
    /**
     * Process raw data into structured nouns and verbs using neural analysis
     */
    async processRawData(rawData, dataType, options) {
        try {
            // Merge options with config
            const mergedConfig = { ...this.config, ...options };
            // Parse the raw data based on type
            const parsedData = await this.parseRawData(rawData, dataType);
            // Perform neural analysis
            const analysis = await this.performNeuralAnalysis(parsedData, mergedConfig);
            // Extract nouns and verbs for the ISenseAugmentation interface
            const nouns = analysis.detectedEntities.map(entity => entity.suggestedId);
            const verbs = analysis.detectedRelationships.map(rel => `${rel.sourceId}->${rel.verbType}->${rel.targetId}`);
            // Store the full analysis for later retrieval
            await this.storeNeuralAnalysis(analysis);
            return {
                success: true,
                data: {
                    nouns,
                    verbs,
                    confidence: analysis.confidence,
                    insights: analysis.insights.map((insight) => ({
                        type: insight.type,
                        description: insight.description,
                        confidence: insight.confidence
                    })),
                    metadata: {
                        detectedEntities: analysis.detectedEntities.length,
                        detectedRelationships: analysis.detectedRelationships.length,
                        timestamp: new Date().toISOString(),
                        augmentation: 'neural-import-sense'
                    }
                }
            };
        }
        catch (error) {
            return {
                success: false,
                data: { nouns: [], verbs: [] },
                error: error instanceof Error ? error.message : 'Neural analysis failed'
            };
        }
    }
    /**
     * Listen to real-time data feeds and process them
     */
    async listenToFeed(feedUrl, callback) {
        // For file-based feeds, watch for changes
        if (feedUrl.startsWith('file://')) {
            const filePath = feedUrl.replace('file://', '');
            // Watch file for changes using Node.js fs.watch
            const fsWatch = require('fs');
            const watcher = fsWatch.watch(filePath, async (eventType) => {
                if (eventType === 'change') {
                    try {
                        const fileContent = await fs.readFile(filePath);
                        const result = await this.processRawData(fileContent, this.getDataTypeFromPath(filePath));
                        if (result.success) {
                            callback({
                                nouns: result.data.nouns,
                                verbs: result.data.verbs,
                                confidence: result.data.confidence
                            });
                        }
                    }
                    catch (error) {
                        console.error('Neural Import feed error:', error);
                    }
                }
            });
            return;
        }
        // For other feed types, implement appropriate listeners
        console.log(`🧠 Neural Import listening to feed: ${feedUrl}`);
    }
    /**
     * Analyze data structure without processing (preview mode)
     */
    async analyzeStructure(rawData, dataType, options) {
        try {
            // Parse the raw data
            const parsedData = await this.parseRawData(rawData, dataType);
            // Perform lightweight analysis for structure detection
            const analysis = await this.performNeuralAnalysis(parsedData, { ...this.config, ...options });
            // Summarize entity types
            const entityTypeCounts = new Map();
            analysis.detectedEntities.forEach(entity => {
                const existing = entityTypeCounts.get(entity.nounType) || { count: 0, totalConfidence: 0 };
                entityTypeCounts.set(entity.nounType, {
                    count: existing.count + 1,
                    totalConfidence: existing.totalConfidence + entity.confidence
                });
            });
            const entityTypes = Array.from(entityTypeCounts.entries()).map(([type, stats]) => ({
                type,
                count: stats.count,
                confidence: stats.totalConfidence / stats.count
            }));
            // Summarize relationship types
            const relationshipTypeCounts = new Map();
            analysis.detectedRelationships.forEach(rel => {
                const existing = relationshipTypeCounts.get(rel.verbType) || { count: 0, totalConfidence: 0 };
                relationshipTypeCounts.set(rel.verbType, {
                    count: existing.count + 1,
                    totalConfidence: existing.totalConfidence + rel.confidence
                });
            });
            const relationshipTypes = Array.from(relationshipTypeCounts.entries()).map(([type, stats]) => ({
                type,
                count: stats.count,
                confidence: stats.totalConfidence / stats.count
            }));
            // Assess data quality
            const dataQuality = this.assessDataQuality(parsedData, analysis);
            // Generate recommendations
            const recommendations = this.generateRecommendations(parsedData, analysis, entityTypes, relationshipTypes);
            return {
                success: true,
                data: {
                    entityTypes,
                    relationshipTypes,
                    dataQuality,
                    recommendations
                }
            };
        }
        catch (error) {
            return {
                success: false,
                data: {
                    entityTypes: [],
                    relationshipTypes: [],
                    dataQuality: { completeness: 0, consistency: 0, accuracy: 0 },
                    recommendations: []
                },
                error: error instanceof Error ? error.message : 'Structure analysis failed'
            };
        }
    }
    /**
     * Validate data compatibility with current knowledge base
     */
    async validateCompatibility(rawData, dataType) {
        try {
            // Parse the raw data
            const parsedData = await this.parseRawData(rawData, dataType);
            // Perform neural analysis
            const analysis = await this.performNeuralAnalysis(parsedData);
            const issues = [];
            const suggestions = [];
            // Check for low confidence entities
            const lowConfidenceEntities = analysis.detectedEntities.filter((e) => e.confidence < 0.5);
            if (lowConfidenceEntities.length > 0) {
                issues.push({
                    type: 'confidence',
                    description: `${lowConfidenceEntities.length} entities have low confidence scores`,
                    severity: 'medium'
                });
                suggestions.push('Consider reviewing field names and data structure for better entity detection');
            }
            // Check for missing relationships
            if (analysis.detectedRelationships.length === 0 && analysis.detectedEntities.length > 1) {
                issues.push({
                    type: 'relationships',
                    description: 'No relationships detected between entities',
                    severity: 'low'
                });
                suggestions.push('Consider adding contextual fields that describe entity relationships');
            }
            // Check for data type compatibility
            const supportedTypes = ['json', 'csv', 'yaml', 'text'];
            if (!supportedTypes.includes(dataType.toLowerCase())) {
                issues.push({
                    type: 'format',
                    description: `Data type '${dataType}' may not be fully supported`,
                    severity: 'high'
                });
                suggestions.push(`Convert data to one of: ${supportedTypes.join(', ')}`);
            }
            // Check for data completeness
            const incompleteEntities = analysis.detectedEntities.filter((e) => !e.originalData || Object.keys(e.originalData).length < 2);
            if (incompleteEntities.length > 0) {
                issues.push({
                    type: 'completeness',
                    description: `${incompleteEntities.length} entities have insufficient data`,
                    severity: 'medium'
                });
                suggestions.push('Ensure each entity has multiple descriptive fields');
            }
            const compatible = issues.filter(i => i.severity === 'high').length === 0;
            return {
                success: true,
                data: {
                    compatible,
                    issues,
                    suggestions
                }
            };
        }
        catch (error) {
            return {
                success: false,
                data: {
                    compatible: false,
                    issues: [{
                            type: 'error',
                            description: error instanceof Error ? error.message : 'Validation failed',
                            severity: 'high'
                        }],
                    suggestions: []
                },
                error: error instanceof Error ? error.message : 'Compatibility validation failed'
            };
        }
    }
    /**
     * Get the full neural analysis result (custom method for Cortex integration)
     */
    async getNeuralAnalysis(rawData, dataType) {
        const parsedData = await this.parseRawData(rawData, dataType);
        return await this.performNeuralAnalysis(parsedData);
    }
    /**
     * Parse raw data based on type
     */
    async parseRawData(rawData, dataType) {
        const content = typeof rawData === 'string' ? rawData : rawData.toString('utf8');
        switch (dataType.toLowerCase()) {
            case 'json':
                const jsonData = JSON.parse(content);
                return Array.isArray(jsonData) ? jsonData : [jsonData];
            case 'csv':
                return this.parseCSV(content);
            case 'yaml':
            case 'yml':
                // For now, basic YAML support - in full implementation would use yaml parser
                return JSON.parse(content); // Placeholder
            case 'txt':
            case 'text':
                // Split text into sentences/paragraphs for analysis
                return content.split(/\n+/).filter(line => line.trim()).map(line => ({ text: line }));
            default:
                throw new Error(`Unsupported data type: ${dataType}`);
        }
    }
    /**
     * Basic CSV parser
     */
    parseCSV(content) {
        const lines = content.split('\n').filter(line => line.trim());
        if (lines.length < 2)
            return [];
        const headers = lines[0].split(',').map(h => h.trim().replace(/"/g, ''));
        const data = [];
        for (let i = 1; i < lines.length; i++) {
            const values = lines[i].split(',').map(v => v.trim().replace(/"/g, ''));
            const row = {};
            headers.forEach((header, index) => {
                row[header] = values[index] || '';
            });
            data.push(row);
        }
        return data;
    }
    /**
     * Perform neural analysis on parsed data
     */
    async performNeuralAnalysis(parsedData, config = this.config) {
        // Phase 1: Neural Entity Detection
        const detectedEntities = await this.detectEntitiesWithNeuralAnalysis(parsedData, config);
        // Phase 2: Neural Relationship Detection  
        const detectedRelationships = await this.detectRelationshipsWithNeuralAnalysis(detectedEntities, parsedData, config);
        // Phase 3: Neural Insights Generation
        const insights = await this.generateNeuralInsights(detectedEntities, detectedRelationships);
        // Phase 4: Confidence Scoring
        const overallConfidence = this.calculateOverallConfidence(detectedEntities, detectedRelationships);
        return {
            detectedEntities,
            detectedRelationships,
            confidence: overallConfidence,
            insights
        };
    }
    /**
     * Neural Entity Detection - The Core AI Engine
     */
    async detectEntitiesWithNeuralAnalysis(rawData, config = this.config) {
        const entities = [];
        const nounTypes = Object.values(NounType);
        for (const [index, dataItem] of rawData.entries()) {
            const mainText = this.extractMainText(dataItem);
            const detections = [];
            // Test against all noun types using semantic similarity
            for (const nounType of nounTypes) {
                const confidence = await this.calculateEntityTypeConfidence(mainText, dataItem, nounType);
                if (confidence >= config.confidenceThreshold - 0.2) { // Allow slightly lower for alternatives
                    const reasoning = await this.generateEntityReasoning(mainText, dataItem, nounType);
                    detections.push({ type: nounType, confidence, reasoning });
                }
            }
            if (detections.length > 0) {
                // Sort by confidence
                detections.sort((a, b) => b.confidence - a.confidence);
                const primaryType = detections[0];
                const alternatives = detections.slice(1, 3); // Top 2 alternatives
                entities.push({
                    originalData: dataItem,
                    nounType: primaryType.type,
                    confidence: primaryType.confidence,
                    suggestedId: this.generateSmartId(dataItem, primaryType.type, index),
                    reasoning: primaryType.reasoning,
                    alternativeTypes: alternatives
                });
            }
        }
        return entities;
    }
    /**
     * Calculate entity type confidence using AI
     */
    async calculateEntityTypeConfidence(text, data, nounType) {
        // Base semantic similarity using search
        const searchResults = await this.brainy.search(text + ' ' + nounType, 1);
        const textSimilarity = searchResults.length > 0 ? searchResults[0].score : 0.5;
        // Field-based confidence boost
        const fieldBoost = this.calculateFieldBasedConfidence(data, nounType);
        // Pattern-based confidence boost  
        const patternBoost = this.calculatePatternBasedConfidence(text, data, nounType);
        // Combine confidences with weights
        const combined = (textSimilarity * 0.5) + (fieldBoost * 0.3) + (patternBoost * 0.2);
        return Math.min(combined, 1.0);
    }
    /**
     * Field-based confidence calculation
     */
    calculateFieldBasedConfidence(data, nounType) {
        const fields = Object.keys(data);
        let boost = 0;
        // Field patterns that boost confidence for specific noun types
        const fieldPatterns = {
            [NounType.Person]: ['name', 'email', 'phone', 'age', 'firstname', 'lastname', 'employee'],
            [NounType.Organization]: ['company', 'organization', 'corp', 'inc', 'ltd', 'department', 'team'],
            [NounType.Project]: ['project', 'task', 'deadline', 'status', 'milestone', 'deliverable'],
            [NounType.Location]: ['address', 'city', 'country', 'state', 'zip', 'location', 'coordinates'],
            [NounType.Product]: ['product', 'price', 'sku', 'inventory', 'category', 'brand'],
            [NounType.Event]: ['date', 'time', 'venue', 'event', 'meeting', 'conference', 'schedule']
        };
        const relevantPatterns = fieldPatterns[nounType] || [];
        for (const field of fields) {
            for (const pattern of relevantPatterns) {
                if (field.toLowerCase().includes(pattern)) {
                    boost += 0.1;
                }
            }
        }
        return Math.min(boost, 0.5);
    }
    /**
     * Pattern-based confidence calculation
     */
    calculatePatternBasedConfidence(text, data, nounType) {
        let boost = 0;
        // Content patterns that indicate entity types
        const patterns = {
            [NounType.Person]: [
                /@.*\.com/i, // Email pattern
                /\b[A-Z][a-z]+ [A-Z][a-z]+\b/, // Name pattern
                /Mr\.|Mrs\.|Dr\.|Prof\./i // Title pattern
            ],
            [NounType.Organization]: [
                /\bInc\.|Corp\.|LLC\.|Ltd\./i, // Corporate suffixes
                /Company|Corporation|Enterprise/i
            ],
            [NounType.Location]: [
                /\b\d{5}(-\d{4})?\b/, // ZIP code
                /Street|Ave|Road|Blvd/i
            ]
        };
        const relevantPatterns = patterns[nounType] || [];
        for (const pattern of relevantPatterns) {
            if (pattern.test(text)) {
                boost += 0.15;
            }
        }
        return Math.min(boost, 0.3);
    }
    /**
     * Generate reasoning for entity type selection
     */
    async generateEntityReasoning(text, data, nounType) {
        const reasons = [];
        // Semantic similarity reason
        const searchResults = await this.brainy.search(text + ' ' + nounType, 1);
        const similarity = searchResults.length > 0 ? searchResults[0].score : 0.5;
        if (similarity > 0.7) {
            reasons.push(`High semantic similarity (${(similarity * 100).toFixed(1)}%)`);
        }
        // Field-based reasons
        const relevantFields = this.getRelevantFields(data, nounType);
        if (relevantFields.length > 0) {
            reasons.push(`Contains ${nounType}-specific fields: ${relevantFields.join(', ')}`);
        }
        // Pattern-based reasons
        const matchedPatterns = this.getMatchedPatterns(text, data, nounType);
        if (matchedPatterns.length > 0) {
            reasons.push(`Matches ${nounType} patterns: ${matchedPatterns.join(', ')}`);
        }
        return reasons.length > 0 ? reasons.join('; ') : 'General semantic match';
    }
    /**
     * Neural Relationship Detection
     */
    async detectRelationshipsWithNeuralAnalysis(entities, rawData, config = this.config) {
        const relationships = [];
        const verbTypes = Object.values(VerbType);
        // For each pair of entities, test relationship possibilities
        for (let i = 0; i < entities.length; i++) {
            for (let j = i + 1; j < entities.length; j++) {
                const sourceEntity = entities[i];
                const targetEntity = entities[j];
                // Extract context for relationship detection
                const context = this.extractRelationshipContext(sourceEntity.originalData, targetEntity.originalData, rawData);
                // Test all verb types
                for (const verbType of verbTypes) {
                    const confidence = await this.calculateRelationshipConfidence(sourceEntity, targetEntity, verbType, context);
                    if (confidence >= config.confidenceThreshold - 0.1) { // Slightly lower threshold for relationships
                        const weight = config.enableWeights ?
                            this.calculateRelationshipWeight(sourceEntity, targetEntity, verbType, context) :
                            0.5;
                        const reasoning = await this.generateRelationshipReasoning(sourceEntity, targetEntity, verbType, context);
                        relationships.push({
                            sourceId: sourceEntity.suggestedId,
                            targetId: targetEntity.suggestedId,
                            verbType,
                            confidence,
                            weight,
                            reasoning,
                            context,
                            metadata: this.extractRelationshipMetadata(sourceEntity.originalData, targetEntity.originalData, verbType)
                        });
                    }
                }
            }
        }
        // Sort by confidence and remove duplicates/conflicts
        return this.pruneRelationships(relationships);
    }
    /**
     * Calculate relationship confidence
     */
    async calculateRelationshipConfidence(source, target, verbType, context) {
        // Semantic similarity between entities and verb type
        const relationshipText = `${this.extractMainText(source.originalData)} ${verbType} ${this.extractMainText(target.originalData)}`;
        const directResults = await this.brainy.search(relationshipText, 1);
        const directSimilarity = directResults.length > 0 ? directResults[0].score : 0.5;
        // Context-based similarity
        const contextResults = await this.brainy.search(context + ' ' + verbType, 1);
        const contextSimilarity = contextResults.length > 0 ? contextResults[0].score : 0.5;
        // Entity type compatibility
        const typeCompatibility = this.calculateTypeCompatibility(source.nounType, target.nounType, verbType);
        // Combine with weights
        return (directSimilarity * 0.4) + (contextSimilarity * 0.4) + (typeCompatibility * 0.2);
    }
    /**
     * Calculate relationship weight/strength
     */
    calculateRelationshipWeight(source, target, verbType, context) {
        let weight = 0.5; // Base weight
        // Context richness (more descriptive = stronger)
        const contextWords = context.split(' ').length;
        weight += Math.min(contextWords / 20, 0.2);
        // Entity importance (higher confidence entities = stronger relationships)
        const avgEntityConfidence = (source.confidence + target.confidence) / 2;
        weight += avgEntityConfidence * 0.2;
        // Verb type specificity (more specific verbs = stronger)
        const verbSpecificity = this.getVerbSpecificity(verbType);
        weight += verbSpecificity * 0.1;
        return Math.min(weight, 1.0);
    }
    /**
     * Generate Neural Insights - The Intelligence Layer
     */
    async generateNeuralInsights(entities, relationships) {
        const insights = [];
        // Detect hierarchies
        const hierarchies = this.detectHierarchies(relationships);
        hierarchies.forEach(hierarchy => {
            insights.push({
                type: 'hierarchy',
                description: `Detected ${hierarchy.type} hierarchy with ${hierarchy.levels} levels`,
                confidence: hierarchy.confidence,
                affectedEntities: hierarchy.entities,
                recommendation: `Consider visualizing the ${hierarchy.type} structure`
            });
        });
        // Detect clusters  
        const clusters = this.detectClusters(entities, relationships);
        clusters.forEach(cluster => {
            insights.push({
                type: 'cluster',
                description: `Found cluster of ${cluster.size} ${cluster.primaryType} entities`,
                confidence: cluster.confidence,
                affectedEntities: cluster.entities,
                recommendation: `These ${cluster.primaryType}s might form a natural grouping`
            });
        });
        // Detect patterns
        const patterns = this.detectPatterns(relationships);
        patterns.forEach(pattern => {
            insights.push({
                type: 'pattern',
                description: `Common relationship pattern: ${pattern.description}`,
                confidence: pattern.confidence,
                affectedEntities: pattern.entities,
                recommendation: pattern.recommendation
            });
        });
        return insights;
    }
    /**
     * Helper methods for the neural system
     */
    extractMainText(data) {
        // Extract the most relevant text from a data object
        const textFields = ['name', 'title', 'description', 'content', 'text', 'label'];
        for (const field of textFields) {
            if (data[field] && typeof data[field] === 'string') {
                return data[field];
            }
        }
        // Fallback: concatenate all string values
        return Object.values(data)
            .filter(v => typeof v === 'string')
            .join(' ')
            .substring(0, 200); // Limit length
    }
    generateSmartId(data, nounType, index) {
        const mainText = this.extractMainText(data);
        const cleanText = mainText.toLowerCase().replace(/[^a-z0-9]/g, '_').substring(0, 20);
        return `${nounType}_${cleanText}_${index}`;
    }
    extractRelationshipContext(source, target, allData) {
        // Extract context for relationship detection
        return [
            this.extractMainText(source),
            this.extractMainText(target),
            // Add more contextual information
        ].join(' ');
    }
    calculateTypeCompatibility(sourceType, targetType, verbType) {
        // Define type compatibility matrix for relationships
        const compatibilityMatrix = {
            [NounType.Person]: {
                [NounType.Organization]: [VerbType.MemberOf, VerbType.WorksWith],
                [NounType.Project]: [VerbType.WorksWith, VerbType.Creates],
                [NounType.Person]: [VerbType.WorksWith, VerbType.Mentors, VerbType.ReportsTo]
            }
            // Add more compatibility rules
        };
        const sourceCompatibility = compatibilityMatrix[sourceType];
        if (sourceCompatibility && sourceCompatibility[targetType]) {
            return sourceCompatibility[targetType].includes(verbType) ? 1.0 : 0.3;
        }
        return 0.5; // Default compatibility
    }
    getVerbSpecificity(verbType) {
        // More specific verbs get higher scores
        const specificityScores = {
            [VerbType.RelatedTo]: 0.1, // Very generic
            [VerbType.WorksWith]: 0.7, // Specific
            [VerbType.Mentors]: 0.9, // Very specific
            [VerbType.ReportsTo]: 0.9, // Very specific
            [VerbType.Supervises]: 0.9 // Very specific
        };
        return specificityScores[verbType] || 0.5;
    }
    getRelevantFields(data, nounType) {
        // Implementation for finding relevant fields
        return [];
    }
    getMatchedPatterns(text, data, nounType) {
        // Implementation for finding matched patterns
        return [];
    }
    pruneRelationships(relationships) {
        // Remove duplicates and low-confidence relationships
        return relationships
            .sort((a, b) => b.confidence - a.confidence)
            .slice(0, 1000); // Limit to top 1000 relationships
    }
    detectHierarchies(relationships) {
        // Detect hierarchical structures
        return [];
    }
    detectClusters(entities, relationships) {
        // Detect entity clusters
        return [];
    }
    detectPatterns(relationships) {
        // Detect relationship patterns
        return [];
    }
    calculateOverallConfidence(entities, relationships) {
        if (entities.length === 0)
            return 0;
        const entityConfidence = entities.reduce((sum, e) => sum + e.confidence, 0) / entities.length;
        if (relationships.length === 0)
            return entityConfidence;
        const relationshipConfidence = relationships.reduce((sum, r) => sum + r.confidence, 0) / relationships.length;
        return (entityConfidence + relationshipConfidence) / 2;
    }
    async storeNeuralAnalysis(analysis) {
        // Store the full analysis result for later retrieval by Neural Import or other systems
        // This could be stored in the brainy instance metadata or a separate analysis store
    }
    getDataTypeFromPath(filePath) {
        const ext = path.extname(filePath).toLowerCase();
        switch (ext) {
            case '.json': return 'json';
            case '.csv': return 'csv';
            case '.yaml':
            case '.yml': return 'yaml';
            case '.txt': return 'text';
            default: return 'text';
        }
    }
    async generateRelationshipReasoning(source, target, verbType, context) {
        return `Neural analysis detected ${verbType} relationship based on semantic context`;
    }
    extractRelationshipMetadata(sourceData, targetData, verbType) {
        return {
            sourceType: typeof sourceData,
            targetType: typeof targetData,
            detectedBy: 'neural-import-sense',
            timestamp: new Date().toISOString()
        };
    }
    /**
     * Assess data quality metrics
     */
    assessDataQuality(parsedData, analysis) {
        // Completeness: ratio of fields with data
        let totalFields = 0;
        let filledFields = 0;
        parsedData.forEach(item => {
            const fields = Object.keys(item);
            totalFields += fields.length;
            filledFields += fields.filter(field => item[field] !== null &&
                item[field] !== undefined &&
                item[field] !== '').length;
        });
        const completeness = totalFields > 0 ? filledFields / totalFields : 0;
        // Consistency: variance in field structure
        const fieldSets = parsedData.map(item => new Set(Object.keys(item)));
        const allFields = new Set(fieldSets.flatMap(set => Array.from(set)));
        let consistencyScore = 0;
        if (fieldSets.length > 0) {
            consistencyScore = Array.from(allFields).reduce((score, field) => {
                const hasField = fieldSets.filter(set => set.has(field)).length;
                return score + (hasField / fieldSets.length);
            }, 0) / allFields.size;
        }
        // Accuracy: average confidence of detected entities
        const accuracy = analysis.detectedEntities.length > 0 ?
            analysis.detectedEntities.reduce((sum, e) => sum + e.confidence, 0) / analysis.detectedEntities.length :
            0;
        return {
            completeness,
            consistency: consistencyScore,
            accuracy
        };
    }
    /**
     * Generate recommendations based on analysis
     */
    generateRecommendations(parsedData, analysis, entityTypes, relationshipTypes) {
        const recommendations = [];
        // Low entity confidence recommendations
        const lowConfidenceEntities = entityTypes.filter(et => et.confidence < 0.7);
        if (lowConfidenceEntities.length > 0) {
            recommendations.push(`Consider improving field names for ${lowConfidenceEntities.map(e => e.type).join(', ')} entities`);
        }
        // Missing relationships recommendations
        if (relationshipTypes.length === 0 && entityTypes.length > 1) {
            recommendations.push('Add fields that describe how entities relate to each other');
        }
        // Data structure recommendations
        if (parsedData.length > 0) {
            const firstItem = parsedData[0];
            const fieldCount = Object.keys(firstItem).length;
            if (fieldCount < 3) {
                recommendations.push('Consider adding more descriptive fields to each entity');
            }
            if (fieldCount > 20) {
                recommendations.push('Consider grouping related fields or splitting complex entities');
            }
        }
        // Entity distribution recommendations
        const dominantEntityType = entityTypes.reduce((max, current) => current.count > max.count ? current : max, entityTypes[0] || { count: 0 });
        if (dominantEntityType && dominantEntityType.count > parsedData.length * 0.8) {
            recommendations.push(`Consider diversifying entity types - ${dominantEntityType.type} dominates the dataset`);
        }
        // Relationship quality recommendations
        const lowWeightRelationships = relationshipTypes.filter(rt => rt.confidence < 0.6);
        if (lowWeightRelationships.length > 0) {
            recommendations.push('Consider adding more contextual information to strengthen relationship detection');
        }
        return recommendations;
    }
}
//# sourceMappingURL=neuralImport.js.map