brainy/examples/test-excel-import.ts

84 lines
2.8 KiB
TypeScript
Raw Permalink Normal View History

feat: add unified import system with auto-detection and dual storage Implemented a comprehensive unified import system that revolutionizes how data flows into Brainy: ## Core Features (Phase 1) - Auto-detection of file formats (Excel, PDF, CSV, JSON, Markdown) via magic bytes and content analysis - Dual storage architecture: creates both VFS files AND knowledge graph entities - Single unified API: brain.import() handles all formats automatically - Format-specific importers for optimal extraction from each file type - VFS structure generation with configurable grouping (by type, sheet, or flat) ## Entity Deduplication (Phase 2) - Embedding-based similarity matching to detect duplicate entities across imports - Intelligent merging with provenance tracking (records which imports contributed) - Fuzzy name matching using Levenshtein distance - Confidence score merging with weighted averages - Cross-import shared knowledge: same entity referenced in multiple datasets gets merged ## Streaming Support (Phase 3) - Chunked processing for memory-efficient handling of large datasets - Configurable chunk size for optimal performance - Progress tracking with real-time callbacks - Scales to millions of entities without memory issues ## Import History & Rollback (Phase 4) - Complete tracking of all imports with full metadata - Rollback capability to undo any import completely - Statistics and analytics across all imports - Persistent history stored in VFS ## Architecture - ImportCoordinator: orchestrates the entire import pipeline - FormatDetector: auto-detects file formats with high confidence - EntityDeduplicator: prevents duplicate entities across imports - ImportHistory: tracks and enables rollback of imports - Format-specific importers: SmartExcelImporter, SmartPDFImporter, etc. - VFSStructureGenerator: creates organized file hierarchies ## Usage ```typescript const result = await brain.import('/path/to/file.xlsx', { vfsPath: '/imports/data', groupBy: 'type', enableDeduplication: true, onProgress: (progress) => console.log(progress) }) ``` ## Production Ready - 5,500+ lines of production code - All integration tests passing - No mocks, stubs, or TODOs - Full TypeScript type safety - Comprehensive error handling - Memory efficient and scalable Closes requirements for unified data ingestion pipeline.
2025-10-08 16:55:30 -07:00
/**
* Test unified import with real Excel file
*/
import { Brainy } from '../src/brainy.js'
async function main() {
console.log('🧠 Testing Excel Import via Unified Import System\n')
const brain = new Brainy({
storage: { type: 'memory' as const }
})
await brain.init()
// Use environment variable or default sample file path
const excelFile = process.env.TEST_EXCEL_FILE || './sample-data.xlsx'
if (!require('fs').existsSync(excelFile)) {
console.log('⚠️ No Excel file found for testing')
console.log(' Set TEST_EXCEL_FILE environment variable or create ./sample-data.xlsx')
console.log(' Example: TEST_EXCEL_FILE=/path/to/your/file.xlsx npm run example')
return
}
console.log('📥 Importing:', excelFile)
console.log()
const result = await brain.import(excelFile, {
vfsPath: '/imports/excel-data',
groupBy: 'type',
onProgress: (progress) => {
if (progress.stage === 'extracting' && progress.processed && progress.total) {
if (progress.processed % 10 === 0 || progress.processed === progress.total) {
console.log(` [${progress.stage}] ${progress.processed}/${progress.total} rows processed`)
}
} else {
console.log(` [${progress.stage}] ${progress.message}`)
}
}
})
console.log()
console.log('✅ Import Complete!')
console.log('─'.repeat(60))
console.log(`Format Detected: ${result.format} (${result.formatConfidence * 100}% confidence)`)
console.log(`Entities Extracted: ${result.stats.entitiesExtracted}`)
console.log(`Graph Nodes Created: ${result.stats.graphNodesCreated}`)
console.log(`Graph Edges Created: ${result.stats.graphEdgesCreated}`)
console.log(`VFS Files Created: ${result.stats.vfsFilesCreated}`)
console.log(`VFS Directories: ${result.vfs.directories.length}`)
console.log(`Processing Time: ${result.stats.processingTime}ms`)
console.log('─'.repeat(60))
console.log()
console.log('📂 VFS Structure:')
result.vfs.directories.forEach(dir => {
console.log(` ${dir}`)
})
console.log()
console.log('🔍 Sample Entities:')
result.entities.slice(0, 5).forEach((entity, i) => {
console.log(` ${i + 1}. ${entity.name} (${entity.type})`)
console.log(` VFS: ${entity.vfsPath}`)
})
console.log()
console.log('🔗 Sample Relationships:')
result.relationships.slice(0, 5).forEach((rel, i) => {
const fromEntity = result.entities.find(e => e.id === rel.from)
const toEntity = result.entities.find(e => e.id === rel.to)
console.log(` ${i + 1}. ${fromEntity?.name || rel.from} --[${rel.type}]--> ${toEntity?.name || rel.to}`)
})
console.log()
console.log('✨ Test Complete!')
}
main().catch(err => {
console.error('❌ Error:', err.message)
process.exit(1)
})