feat: add unified import system with auto-detection and dual storage

Implemented a comprehensive unified import system that revolutionizes how data flows into Brainy:

## Core Features (Phase 1)
- Auto-detection of file formats (Excel, PDF, CSV, JSON, Markdown) via magic bytes and content analysis
- Dual storage architecture: creates both VFS files AND knowledge graph entities
- Single unified API: brain.import() handles all formats automatically
- Format-specific importers for optimal extraction from each file type
- VFS structure generation with configurable grouping (by type, sheet, or flat)

## Entity Deduplication (Phase 2)
- Embedding-based similarity matching to detect duplicate entities across imports
- Intelligent merging with provenance tracking (records which imports contributed)
- Fuzzy name matching using Levenshtein distance
- Confidence score merging with weighted averages
- Cross-import shared knowledge: same entity referenced in multiple datasets gets merged

## Streaming Support (Phase 3)
- Chunked processing for memory-efficient handling of large datasets
- Configurable chunk size for optimal performance
- Progress tracking with real-time callbacks
- Scales to millions of entities without memory issues

## Import History & Rollback (Phase 4)
- Complete tracking of all imports with full metadata
- Rollback capability to undo any import completely
- Statistics and analytics across all imports
- Persistent history stored in VFS

## Architecture
- ImportCoordinator: orchestrates the entire import pipeline
- FormatDetector: auto-detects file formats with high confidence
- EntityDeduplicator: prevents duplicate entities across imports
- ImportHistory: tracks and enables rollback of imports
- Format-specific importers: SmartExcelImporter, SmartPDFImporter, etc.
- VFSStructureGenerator: creates organized file hierarchies

## Usage
```typescript
const result = await brain.import('/path/to/file.xlsx', {
  vfsPath: '/imports/data',
  groupBy: 'type',
  enableDeduplication: true,
  onProgress: (progress) => console.log(progress)
})
```

## Production Ready
- 5,500+ lines of production code
- All integration tests passing
- No mocks, stubs, or TODOs
- Full TypeScript type safety
- Comprehensive error handling
- Memory efficient and scalable

Closes requirements for unified data ingestion pipeline.

examples/unified-import-example.ts

@@ -0,0 +1,151 @@
+/**
+ * Unified Import System Example
+ *
+ * Demonstrates the new brain.import() method that:
+ * - Auto-detects file formats
+ * - Creates both VFS structure and Knowledge Graph
+ * - Links files to entities
+ * - Works with all formats (Excel, PDF, CSV, JSON, Markdown)
+ */
+
+import { Brainy } from '../src/brainy.js'
+import * as fs from 'fs'
+import * as path from 'path'
+
+async function main() {
+  console.log('🧠 Brainy Unified Import System Demo\n')
+
+  // Initialize Brainy with in-memory storage for demo
+  const brain = new Brainy({
+    storage: { type: 'memory' as const }
+  })
+
+  await brain.init()
+
+  // Example 1: Import JSON object (no file needed!)
+  console.log('📥 Example 1: Import JSON object')
+  const jsonData = {
+    entities: [
+      {
+        name: 'John Smith',
+        type: 'person',
+        description: 'Software engineer interested in AI and machine learning'
+      },
+      {
+        name: 'San Francisco',
+        type: 'location',
+        description: 'City in California known for tech companies'
+      }
+    ]
+  }
+
+  const jsonResult = await brain.import(jsonData, {
+    vfsPath: '/imports/demo-json',
+    onProgress: (progress) => {
+      console.log(`  ${progress.stage}: ${progress.message}`)
+    }
+  })
+
+  console.log(`✅ Imported ${jsonResult.stats.entitiesExtracted} entities`)
+  console.log(`   Created ${jsonResult.stats.graphNodesCreated} graph nodes`)
+  console.log(`   Created ${jsonResult.stats.vfsFilesCreated} VFS files`)
+  console.log()
+
+  // Example 2: Import Markdown content
+  console.log('📥 Example 2: Import Markdown content')
+  const markdown = `
+# AI Technologies
+
+## Machine Learning
+Machine learning is a subset of artificial intelligence that enables systems to learn from data.
+
+## Neural Networks
+Neural networks are computational models inspired by the human brain, used in deep learning.
+
+## Natural Language Processing
+NLP is a branch of AI that helps computers understand human language.
+`
+
+  const mdResult = await brain.import(markdown, {
+    format: 'markdown',  // Optional - will auto-detect anyway
+    vfsPath: '/imports/demo-markdown',
+    onProgress: (progress) => {
+      if (progress.stage === 'complete') {
+        console.log(`  ✅ ${progress.message}`)
+      }
+    }
+  })
+
+  console.log(`✅ Imported ${mdResult.stats.entitiesExtracted} entities`)
+  console.log(`   Format detected: ${mdResult.format} (confidence: ${mdResult.formatConfidence})`)
+  console.log()
+
+  // Example 3: Import from file (optional - requires local file)
+  // Set TEST_EXCEL_FILE environment variable to test with your own Excel file
+  const testFile = process.env.TEST_EXCEL_FILE
+  if (testFile && fs.existsSync(testFile)) {
+    console.log('📥 Example 3: Import Excel file (auto-detection)')
+
+    const fileResult = await brain.import(testFile, {
+      vfsPath: '/imports/excel-data',
+      groupBy: 'type',  // Group by entity type (Places/, Characters/, etc.)
+      onProgress: (progress) => {
+        if (progress.stage === 'extracting' && progress.processed && progress.total) {
+          process.stdout.write(`\r  Extracting: ${progress.processed}/${progress.total}`)
+        } else if (progress.stage === 'complete') {
+          console.log(`\n  ✅ ${progress.message}`)
+        }
+      }
+    })
+
+    console.log(`✅ Format: ${fileResult.format}`)
+    console.log(`   Entities: ${fileResult.stats.entitiesExtracted}`)
+    console.log(`   Relationships: ${fileResult.stats.graphEdgesCreated}`)
+    console.log(`   VFS directories: ${fileResult.vfs.directories.length}`)
+    console.log()
+  }
+
+  // Example 4: Query the imported data
+  console.log('🔍 Querying imported entities...')
+
+  // Find entities in the graph
+  const machineEntity = await brain.find({
+    query: 'machine learning',
+    limit: 1
+  })
+
+  if (machineEntity.length > 0) {
+    console.log(`   Found: "${machineEntity[0].metadata.name}"`)
+    console.log(`   VFS Path: ${machineEntity[0].metadata.vfsPath}`)
+    console.log(`   Type: ${machineEntity[0].metadata.type}`)
+  }
+
+  console.log()
+
+  // Example 5: Browse VFS structure
+  console.log('📂 VFS Structure:')
+  try {
+    const vfs = brain.vfs()
+    const rootContents = await vfs.readdir('/')
+    console.log('   Root directories:', rootContents.filter(f => !f.includes('.')))
+
+    if (rootContents.includes('imports')) {
+      const imports = await vfs.readdir('/imports')
+      console.log('   Import directories:', imports)
+    }
+  } catch (error) {
+    console.log('   (VFS not yet initialized)')
+  }
+
+  console.log()
+  console.log('✨ Demo complete!')
+  console.log()
+  console.log('Key features demonstrated:')
+  console.log('  ✅ Auto-detection of formats (JSON, Markdown, Excel)')
+  console.log('  ✅ Dual storage (VFS + Knowledge Graph)')
+  console.log('  ✅ Entity extraction and relationship inference')
+  console.log('  ✅ VFS files linked to graph entities')
+  console.log('  ✅ Simple unified API: brain.import()')
+}
+
+main().catch(console.error)