diff --git a/docs/DEVELOPER_LEARNING_PATH.md b/docs/DEVELOPER_LEARNING_PATH.md
new file mode 100644
index 00000000..3436d922
--- /dev/null
+++ b/docs/DEVELOPER_LEARNING_PATH.md
@@ -0,0 +1,1239 @@
+# ๐ Brainy Developer Learning Path
+**From Zero to Hero in 5 Progressive Levels**
+
+> This guide takes you from your first Brainy query to production-scale neural database mastery. Follow each level in order for the best learning experience.
+
+---
+
+## ๐ Quick Navigation
+
+- [Level 1: Hello Brainy](#level-1-hello-brainy-15-minutes) - Your first neural database
+- [Level 2: Relationships & Batch Operations](#level-2-relationships--batch-operations-30-minutes) - Scale up your data
+- [Level 3: Advanced Search & Neural AI](#level-3-advanced-search--neural-ai-45-minutes) - Triple Intelligence
+- [Level 4: Virtual Filesystem](#level-4-virtual-filesystem-60-minutes) - Files as intelligent entities
+- [Level 5: Production Scale](#level-5-production-scale-90-minutes) - Planet-scale deployment
+
+---
+
+## Level 1: Hello Brainy (15 minutes)
+
+### What You'll Learn
+- Initialize Brainy
+- Add your first entity
+- Perform semantic search
+- Understand basic types
+
+### Prerequisites
+```bash
+npm install @soulcraft/brainy
+```
+
+### Your First Neural Database
+
+```typescript
+import { Brainy, NounType } from '@soulcraft/brainy'
+
+// Step 1: Create and initialize Brainy
+const brain = new Brainy({
+ storage: { type: 'memory' } // Start simple - no persistence needed
+})
+await brain.init()
+
+// Step 2: Add some data
+const johnId = await brain.add({
+ data: 'John Smith is a software engineer at TechCorp',
+ type: NounType.Person,
+ metadata: { role: 'Engineer', company: 'TechCorp' }
+})
+
+const aliceId = await brain.add({
+ data: 'Alice Johnson is a product manager at TechCorp',
+ type: NounType.Person,
+ metadata: { role: 'Manager', company: 'TechCorp' }
+})
+
+const projectId = await brain.add({
+ data: 'AI-powered customer support system using machine learning',
+ type: NounType.Project,
+ metadata: { status: 'active', priority: 'high' }
+})
+
+// Step 3: Semantic search (this is where magic happens!)
+console.log('\n๐ Searching for "engineers"...')
+const engineers = await brain.find({ query: 'engineers' })
+console.log(`Found ${engineers.length} engineers:`)
+for (const result of engineers) {
+ console.log(` - ${result.entity.data} (score: ${result.score.toFixed(2)})`)
+}
+
+// Step 4: Search with filters
+console.log('\n๐ Searching for "people at TechCorp"...')
+const techcorpPeople = await brain.find({
+ query: 'people',
+ type: NounType.Person,
+ where: { company: 'TechCorp' },
+ limit: 10
+})
+console.log(`Found ${techcorpPeople.length} people at TechCorp`)
+
+// Step 5: Get entity by ID
+const john = await brain.get(johnId)
+console.log('\n๐ค John\'s data:', {
+ type: john?.type,
+ data: john?.data,
+ metadata: john?.metadata
+})
+
+// Step 6: Clean up
+await brain.close()
+console.log('\nโ
Done! You just created your first neural database!')
+```
+
+### Key Concepts
+
+#### 1. **NounType** - Entity Classification
+Brainy has 31 built-in types including:
+- `Person`, `Organization`, `Location`
+- `Document`, `File`, `Content`
+- `Product`, `Service`, `Event`
+- `Project`, `Task`, `Concept`
+
+**Why it matters**: Proper typing enables intelligent search and organization.
+
+#### 2. **Semantic Search** - Understanding Meaning
+```typescript
+// Traditional search: exact keyword matching
+// "engineers" would NOT find "software developer"
+
+// Semantic search: understands meaning
+await brain.find({ query: 'engineers' })
+// โ
Finds: "software engineer", "developer", "programmer", "coder"
+```
+
+#### 3. **Metadata Filtering** - Precise Control
+```typescript
+// Combine semantic search with structured filters
+await brain.find({
+ query: 'machine learning', // Semantic: finds AI, ML, neural networks
+ where: { company: 'TechCorp' }, // Structured: exact match
+ type: NounType.Project // Type filter
+})
+```
+
+### Practice Exercises
+
+1. Create a small company directory with 5-10 people
+2. Search for "managers", "developers", "designers"
+3. Add projects and search for "active projects"
+4. Experiment with different metadata filters
+
+### Next Steps
+Once you're comfortable with basic operations, move to **Level 2** to learn about relationships and batch operations.
+
+---
+
+## Level 2: Relationships & Batch Operations (30 minutes)
+
+### What You'll Learn
+- Create relationships between entities
+- Batch add/update/delete operations
+- Query graph relationships
+- Understand VerbTypes
+
+### Building a Knowledge Graph
+
+```typescript
+import { Brainy, NounType, VerbType } from '@soulcraft/brainy'
+
+const brain = new Brainy({ storage: { type: 'memory' } })
+await brain.init()
+
+// Batch add multiple entities
+console.log('๐ฆ Adding team members...')
+const result = await brain.addMany({
+ items: [
+ { data: 'John Smith - Senior Engineer', type: NounType.Person, metadata: { role: 'Engineer' } },
+ { data: 'Alice Johnson - Product Manager', type: NounType.Person, metadata: { role: 'Manager' } },
+ { data: 'Bob Wilson - Designer', type: NounType.Person, metadata: { role: 'Designer' } },
+ { data: 'TechCorp - Software Company', type: NounType.Organization },
+ { data: 'AI Assistant Project', type: NounType.Project, metadata: { status: 'active' } }
+ ],
+ parallel: true,
+ onProgress: (done, total) => console.log(` Progress: ${done}/${total}`)
+})
+
+console.log(`โ
Added ${result.successful.length} entities`)
+const [johnId, aliceId, bobId, techcorpId, projectId] = result.successful
+
+// Create relationships (building the graph!)
+console.log('\n๐ Creating relationships...')
+await brain.relateMany({
+ relations: [
+ // People work for organization
+ { from: johnId, to: techcorpId, type: VerbType.WorksWith },
+ { from: aliceId, to: techcorpId, type: VerbType.WorksWith },
+ { from: bobId, to: techcorpId, type: VerbType.WorksWith },
+
+ // People work on project
+ { from: johnId, to: projectId, type: VerbType.WorksOn },
+ { from: bobId, to: projectId, type: VerbType.WorksOn },
+
+ // Alice manages the project
+ { from: aliceId, to: projectId, type: VerbType.Manages },
+
+ // Team collaboration
+ { from: johnId, to: aliceId, type: VerbType.CollaboratesWith, bidirectional: true },
+ { from: bobId, to: johnId, type: VerbType.CollaboratesWith, bidirectional: true }
+ ]
+})
+
+console.log('โ
Created relationships')
+
+// Query relationships
+console.log('\n๐ Querying relationships...')
+
+// Who works for TechCorp?
+const techcorpEmployees = await brain.getRelations({
+ to: techcorpId,
+ type: VerbType.WorksWith
+})
+console.log(`TechCorp has ${techcorpEmployees.length} employees`)
+
+// Who works on the AI project?
+const projectContributors = await brain.getRelations({
+ to: projectId,
+ type: [VerbType.WorksOn, VerbType.Manages]
+})
+console.log(`AI Project has ${projectContributors.length} contributors`)
+
+// Who does John collaborate with?
+const johnsCollaborators = await brain.getRelations({
+ from: johnId,
+ type: VerbType.CollaboratesWith
+})
+console.log(`John collaborates with ${johnsCollaborators.length} people`)
+
+// Get graph statistics
+const stats = brain.getStats()
+console.log('\n๐ Graph Statistics:', {
+ entities: stats.entities.total,
+ relationships: stats.relationships.totalRelationships,
+ density: stats.density.toFixed(2)
+})
+
+// Batch update
+console.log('\n๐ Updating all team members...')
+await brain.updateMany({
+ items: [johnId, aliceId, bobId].map(id => ({
+ id,
+ metadata: { team: 'AI Team', updated: new Date().toISOString() },
+ merge: true // Merge with existing metadata (don't replace!)
+ }))
+})
+
+console.log('โ
Updated team metadata')
+
+await brain.close()
+```
+
+### Key Concepts
+
+#### 1. **VerbType** - Relationship Types
+Brainy has 40 relationship types including:
+- Work: `WorksWith`, `WorksOn`, `Manages`, `Supervises`
+- Structure: `PartOf`, `Contains`, `BelongsTo`
+- Knowledge: `RelatedTo`, `DependsOn`, `Requires`
+- Creation: `Creates`, `Modifies`, `Transforms`
+
+#### 2. **Bidirectional Relationships**
+```typescript
+await brain.relate({
+ from: personA,
+ to: personB,
+ type: VerbType.CollaboratesWith,
+ bidirectional: true // Creates AโB AND BโA
+})
+```
+
+#### 3. **Batch Operations = Performance**
+```typescript
+// โ Slow: 100 individual operations
+for (const item of items) {
+ await brain.add(item) // 100 round trips!
+}
+
+// โ
Fast: 1 batch operation
+await brain.addMany({ items }) // 1 round trip!
+```
+
+#### 4. **Metadata Merging**
+```typescript
+// Initial metadata
+await brain.add({
+ data: 'John',
+ metadata: { role: 'Engineer', level: 3 }
+})
+
+// Update with merge: true (default)
+await brain.update({
+ id: johnId,
+ metadata: { team: 'AI Team' },
+ merge: true // Result: { role: 'Engineer', level: 3, team: 'AI Team' }
+})
+
+// Update with merge: false
+await brain.update({
+ id: johnId,
+ metadata: { team: 'AI Team' },
+ merge: false // Result: { team: 'AI Team' } - role and level lost!
+})
+```
+
+### Practice Exercises
+
+1. Create an organizational hierarchy (CEO โ Managers โ Engineers)
+2. Build a project dependency graph
+3. Model a social network with CollaboratesWith relationships
+4. Query "Who reports to Alice?" using getRelations()
+5. Batch update all projects to add a "year: 2024" field
+
+### Next Steps
+Ready for AI-powered search and clustering? Move to **Level 3**.
+
+---
+
+## Level 3: Advanced Search & Neural AI (45 minutes)
+
+### What You'll Learn
+- Triple Intelligence (Vector + Metadata + Graph)
+- Semantic similarity
+- Automatic clustering
+- Outlier detection
+- Natural language queries
+
+### Triple Intelligence in Action
+
+```typescript
+import { Brainy, NounType, VerbType } from '@soulcraft/brainy'
+
+const brain = new Brainy({ storage: { type: 'memory' } })
+await brain.init()
+
+// Create a realistic dataset
+console.log('๐ฆ Creating knowledge base...')
+const knowledgeBase = await brain.addMany({
+ items: [
+ // Research papers
+ { data: 'Deep Learning for Computer Vision using Convolutional Neural Networks',
+ type: NounType.Document,
+ metadata: { category: 'AI', year: 2024, citations: 150 } },
+ { data: 'Natural Language Processing with Transformer Models',
+ type: NounType.Document,
+ metadata: { category: 'AI', year: 2024, citations: 200 } },
+ { data: 'Reinforcement Learning for Robotics Applications',
+ type: NounType.Document,
+ metadata: { category: 'AI', year: 2023, citations: 80 } },
+
+ // Different domain
+ { data: 'Climate Change Impact on Ocean Ecosystems',
+ type: NounType.Document,
+ metadata: { category: 'Climate', year: 2024, citations: 120 } },
+ { data: 'Renewable Energy Solutions for Urban Planning',
+ type: NounType.Document,
+ metadata: { category: 'Energy', year: 2024, citations: 95 } },
+
+ // Code projects
+ { data: 'AI-powered code completion tool using GPT',
+ type: NounType.Project,
+ metadata: { category: 'Tools', status: 'active' } },
+ { data: 'Neural network visualization dashboard',
+ type: NounType.Project,
+ metadata: { category: 'Tools', status: 'active' } }
+ ]
+})
+
+console.log(`โ
Created ${knowledgeBase.successful.length} entities\n`)
+
+// 1. VECTOR INTELLIGENCE: Semantic similarity
+console.log('๐ 1. VECTOR INTELLIGENCE: Semantic Search')
+const aiResults = await brain.find({
+ query: 'machine learning and neural networks', // User's natural language
+ limit: 3
+})
+
+console.log('Top 3 semantically similar documents:')
+aiResults.forEach((r, i) => {
+ console.log(` ${i + 1}. [${r.score.toFixed(3)}] ${r.entity.data?.substring(0, 50)}...`)
+})
+
+// 2. METADATA INTELLIGENCE: Structured filtering
+console.log('\n๐ 2. METADATA INTELLIGENCE: Precise Filtering')
+const recentHighCitations = await brain.find({
+ query: 'artificial intelligence',
+ where: {
+ year: 2024,
+ citations: { $gte: 100 } // Brainy Field Operator: greater than or equal
+ },
+ limit: 10
+})
+
+console.log(`Found ${recentHighCitations.length} highly-cited AI papers from 2024`)
+
+// 3. GRAPH INTELLIGENCE: Relationship-aware search
+console.log('\n๐ 3. GRAPH INTELLIGENCE: Relationship-Aware Search')
+
+// First, create some relationships
+const [paper1, paper2] = knowledgeBase.successful
+await brain.relate({
+ from: paper1,
+ to: paper2,
+ type: VerbType.References
+})
+
+// Search with graph constraints
+const connectedDocs = await brain.find({
+ query: 'deep learning',
+ connected: {
+ to: paper2,
+ via: VerbType.References
+ }
+})
+
+console.log(`Found ${connectedDocs.length} papers that reference the NLP paper`)
+
+// 4. FUSION: Combine all three intelligences!
+console.log('\n๐ 4. TRIPLE INTELLIGENCE FUSION')
+const fusionResults = await brain.find({
+ query: 'AI research', // Vector: semantic understanding
+ where: { year: 2024 }, // Metadata: structured filter
+ type: NounType.Document, // Type constraint
+ fusion: {
+ strategy: 'adaptive', // Let Brainy optimize weights
+ weights: {
+ vector: 0.5, // 50% semantic similarity
+ field: 0.3, // 30% metadata match
+ graph: 0.2 // 20% relationship strength
+ }
+ },
+ explain: true // See how the score was calculated
+})
+
+console.log('Fusion search results with score explanations:')
+fusionResults.forEach(r => {
+ console.log(`\n ${r.entity.data?.substring(0, 60)}...`)
+ console.log(` Total score: ${r.score.toFixed(3)}`)
+ if (r.explanation) {
+ console.log(` Vector: ${r.explanation.vector.toFixed(3)}`)
+ console.log(` Metadata: ${r.explanation.metadata.toFixed(3)}`)
+ console.log(` Graph: ${r.explanation.graph.toFixed(3)}`)
+ }
+})
+
+// 5. NEURAL API: Automatic clustering
+console.log('\n\n๐ค NEURAL API: Automatic Clustering')
+const neural = brain.neural()
+
+const clusters = await neural.clusters({
+ maxClusters: 3,
+ minClusterSize: 1
+})
+
+console.log(`Found ${clusters.length} semantic clusters:`)
+clusters.forEach((cluster, i) => {
+ console.log(`\n Cluster ${i + 1}: ${cluster.label || cluster.id}`)
+ console.log(` Members: ${cluster.members.length}`)
+ console.log(` Centroid topics: ${cluster.metadata?.topics?.join(', ') || 'N/A'}`)
+})
+
+// 6. SIMILARITY: Find similar documents
+console.log('\n\n๐ SIMILARITY: Find Similar Documents')
+const similarTo = await brain.similar({
+ to: paper1, // Entity ID of first AI paper
+ limit: 3,
+ threshold: 0.5, // Minimum similarity score
+ type: NounType.Document
+})
+
+console.log(`Documents similar to "${knowledgeBase.successful[0]}":`)
+similarTo.forEach(r => {
+ console.log(` [${r.score.toFixed(3)}] ${r.entity.data?.substring(0, 50)}...`)
+})
+
+// 7. OUTLIER DETECTION
+console.log('\n\n๐จ OUTLIER DETECTION')
+const outliers = await neural.outliers({
+ method: 'statistical',
+ threshold: 2.0 // 2 standard deviations
+})
+
+console.log(`Found ${outliers.length} outlier documents:`)
+outliers.forEach(o => {
+ const entity = await brain.get(o.id)
+ console.log(` [Anomaly score: ${o.score.toFixed(3)}] ${entity?.data?.substring(0, 50)}...`)
+})
+
+await brain.close()
+```
+
+### Key Concepts
+
+#### 1. **Triple Intelligence Explained**
+
+```
+Traditional Database: WHERE category = 'AI' (exact match only)
+ โ Misses: "artificial intelligence", "machine learning"
+
+Vector Search: semantic("AI research") (meaning-based)
+ โ
Finds: AI, ML, neural networks, deep learning
+ โ No filtering by year, citations, etc.
+
+Brainy Triple: semantic("AI") + WHERE year=2024 + CONNECTED TO paper123
+ โ
Finds semantically similar + filters + graph aware
+```
+
+#### 2. **Score Explanations**
+```typescript
+const results = await brain.find({
+ query: 'AI',
+ explain: true // Get score breakdown
+})
+
+// result.explanation shows:
+// {
+// vector: 0.85, // 85% semantic match
+// metadata: 0.90, // 90% field match
+// graph: 0.70, // 70% graph relevance
+// final: 0.82 // Weighted combination
+// }
+```
+
+#### 3. **Fusion Strategies**
+```typescript
+// 'adaptive' - Brainy automatically adjusts weights based on query
+fusion: { strategy: 'adaptive' }
+
+// 'balanced' - Equal weights to all signals
+fusion: { strategy: 'balanced' }
+
+// 'custom' - You control the weights
+fusion: {
+ strategy: 'custom',
+ weights: { vector: 0.7, field: 0.2, graph: 0.1 }
+}
+```
+
+#### 4. **Brainy Field Operators (BFO)**
+```typescript
+where: {
+ age: { $gte: 18, $lte: 65 }, // Range
+ role: { $in: ['Engineer', 'Manager'] }, // One of
+ name: { $contains: 'John' }, // Substring
+ active: true, // Exact match
+ tags: { $includes: 'AI' } // Array contains
+}
+```
+
+### Practice Exercises
+
+1. Create a document collection and find semantically similar items
+2. Use fusion search with custom weights
+3. Cluster your data and examine the clusters
+4. Find outliers in a dataset
+5. Compare results with/without explain: true
+
+### Next Steps
+Want to treat files as intelligent entities? Learn the **Virtual Filesystem** in Level 4.
+
+---
+
+## Level 4: Virtual Filesystem (60 minutes)
+
+### What You'll Learn
+- VFS as knowledge operating system
+- Files with semantic understanding
+- Semantic file search
+- Cross-boundary relationships (VFS โ Knowledge)
+- VFS filtering architecture
+
+### Files as Intelligent Entities
+
+```typescript
+import { Brainy, NounType, VerbType } from '@soulcraft/brainy'
+
+const brain = new Brainy({ storage: { type: 'memory' } })
+await brain.init()
+
+// Initialize VFS
+const vfs = brain.vfs()
+await vfs.init()
+
+console.log('๐ Creating semantic filesystem...\n')
+
+// 1. BASIC FILE OPERATIONS (POSIX-like)
+await vfs.mkdir('/projects', { recursive: true })
+await vfs.mkdir('/projects/ai-assistant')
+await vfs.mkdir('/docs')
+
+await vfs.writeFile('/projects/ai-assistant/README.md', `
+# AI Assistant Project
+
+A neural-powered assistant using transformer models for natural language understanding.
+
+## Features
+- Semantic search
+- Context-aware responses
+- Multi-turn conversations
+`)
+
+await vfs.writeFile('/projects/ai-assistant/architecture.md', `
+# Architecture
+
+## Components
+- NLP Engine: Transformer-based language model
+- Knowledge Graph: Brainy neural database
+- API Layer: RESTful endpoints
+`)
+
+await vfs.writeFile('/docs/installation.md', `
+# Installation Guide
+
+\`\`\`bash
+npm install ai-assistant
+\`\`\`
+`)
+
+console.log('โ
Created 3 files\n')
+
+// 2. VFS-ONLY SEMANTIC SEARCH
+console.log('๐ Searching VFS for "neural networks"...')
+const vfsFiles = await vfs.search('neural networks', { limit: 5 })
+console.log(`Found ${vfsFiles.length} VFS files:`)
+vfsFiles.forEach(f => {
+ console.log(` [${f.score.toFixed(3)}] ${f.path}`)
+})
+
+// 3. VFS FILTERING IN KNOWLEDGE QUERIES
+console.log('\n๐ Understanding VFS filtering (v4.4.0)...\n')
+
+// Create some knowledge entities
+const conceptId = await brain.add({
+ data: 'Neural networks are computational models inspired by biological neurons',
+ type: NounType.Concept,
+ metadata: { topic: 'AI' }
+})
+
+const projectId = await brain.add({
+ data: 'AI Assistant - conversational AI using transformers',
+ type: NounType.Project,
+ metadata: { status: 'active' }
+})
+
+console.log('Created 2 knowledge entities\n')
+
+// DEFAULT: Knowledge queries exclude VFS (clean separation!)
+console.log('๐ brain.find() - DEFAULT behavior (excludes VFS):')
+const knowledgeOnly = await brain.find({ query: 'neural networks' })
+console.log(` Found ${knowledgeOnly.length} entities`)
+console.log(` VFS files: ${knowledgeOnly.filter(r => r.metadata?.isVFS).length}`) // 0
+console.log(` Knowledge: ${knowledgeOnly.filter(r => !r.metadata?.isVFS).length}`)
+
+// OPT-IN: Include VFS when needed
+console.log('\n๐ brain.find() with includeVFS: true:')
+const everything = await brain.find({
+ query: 'neural networks',
+ includeVFS: true // Opt-in to include VFS files
+})
+console.log(` Found ${everything.length} entities`)
+console.log(` VFS files: ${everything.filter(r => r.metadata?.isVFS).length}`)
+console.log(` Knowledge: ${everything.filter(r => !r.metadata?.isVFS).length}`)
+
+// VFS-ONLY: Search only files
+console.log('\n๐ Searching ONLY VFS files:')
+const filesOnly = await brain.find({
+ where: { vfsType: 'file', extension: '.md' },
+ includeVFS: true // Required to find VFS entities
+})
+console.log(` Found ${filesOnly.length} markdown files`)
+
+// 4. CROSS-BOUNDARY RELATIONSHIPS
+console.log('\n\n๐ Creating cross-boundary relationships...')
+
+// Link concept to documentation file
+const readmeEntity = await brain.find({
+ where: { path: '/projects/ai-assistant/README.md' },
+ includeVFS: true,
+ limit: 1
+})
+
+if (readmeEntity.length > 0) {
+ await brain.relate({
+ from: conceptId,
+ to: readmeEntity[0].id,
+ type: VerbType.DocumentedBy,
+ metadata: { section: 'Features' }
+ })
+ console.log('โ
Linked concept to README.md')
+}
+
+// Query relationships
+const conceptDocs = await brain.getRelations({
+ from: conceptId,
+ type: VerbType.DocumentedBy
+})
+console.log(`Concept is documented by ${conceptDocs.length} files`)
+
+// 5. VFS SEMANTIC FEATURES
+console.log('\n\n๐ VFS Semantic Features:')
+
+// Find similar files
+const similarFiles = await vfs.findSimilar('/projects/ai-assistant/README.md', {
+ limit: 3,
+ threshold: 0.5
+})
+console.log(`\nFiles similar to README.md: ${similarFiles.length}`)
+similarFiles.forEach(f => {
+ console.log(` [${f.score.toFixed(3)}] ${f.path}`)
+})
+
+// Get file stats
+const stats = await vfs.stat('/projects/ai-assistant/README.md')
+console.log('\nREADME.md stats:', {
+ size: stats.size,
+ type: stats.vfsType,
+ extension: stats.metadata?.extension,
+ created: new Date(stats.metadata?.createdAt || 0).toLocaleString()
+})
+
+// Read directory
+console.log('\n๐ Directory contents of /projects/ai-assistant:')
+const entries = await vfs.readdir('/projects/ai-assistant')
+console.log(entries)
+
+// 6. METADATA & EXTENDED ATTRIBUTES
+console.log('\n\n๐ Metadata & Extended Attributes:')
+
+await vfs.setMetadata('/projects/ai-assistant/README.md', {
+ author: 'John Smith',
+ version: '1.0.0',
+ tags: ['AI', 'documentation', 'project']
+})
+
+const metadata = await vfs.getMetadata('/projects/ai-assistant/README.md')
+console.log('README metadata:', metadata)
+
+// Extended attributes (like file properties)
+await vfs.setxattr('/projects/ai-assistant/README.md', 'priority', 'high')
+await vfs.setxattr('/projects/ai-assistant/README.md', 'reviewStatus', 'approved')
+
+const xattrs = await vfs.listxattr('/projects/ai-assistant/README.md')
+console.log('Extended attributes:', xattrs)
+
+// 7. FILE OPERATIONS
+console.log('\n\n๐ Advanced File Operations:')
+
+// Copy file
+await vfs.copy('/docs/installation.md', '/projects/ai-assistant/INSTALL.md')
+console.log('โ
Copied installation.md')
+
+// Rename
+await vfs.rename('/projects/ai-assistant/INSTALL.md', '/projects/ai-assistant/setup.md')
+console.log('โ
Renamed to setup.md')
+
+// Check existence
+const exists = await vfs.exists('/projects/ai-assistant/setup.md')
+console.log(`setup.md exists: ${exists}`)
+
+console.log('\n\nโ
VFS Tutorial Complete!')
+console.log('\n๐ Key Takeaways:')
+console.log(' 1. VFS files have semantic understanding (search by meaning)')
+console.log(' 2. brain.find() excludes VFS by default (clean knowledge queries)')
+console.log(' 3. Use includeVFS: true to include VFS in knowledge queries')
+console.log(' 4. vfs.search() ONLY searches VFS files (never knowledge entities)')
+console.log(' 5. Cross-boundary relationships link files to concepts')
+console.log(' 6. Every file is a full Brainy entity with vector, metadata, and graph')
+
+await vfs.close()
+await brain.close()
+```
+
+### Key Concepts
+
+#### 1. **VFS Filtering Architecture (v4.4.0)**
+
+```typescript
+// ๐ฏ DEFAULT BEHAVIOR: Clean Separation
+//
+// Knowledge queries stay clean (no VFS pollution)
+const concepts = await brain.find({ query: 'AI' })
+// Returns: Only NounType.Concept, NounType.Document, etc.
+// Excludes: VFS files (no .path property)
+
+// VFS queries work with VFS only
+const files = await vfs.search('documentation')
+// Returns: Only VFS files with .path property
+// Excludes: Knowledge entities
+
+// ๐ CROSS-BOUNDARY: Opt-in when needed
+const everything = await brain.find({
+ query: 'machine learning',
+ includeVFS: true // Include both knowledge AND VFS
+})
+// Returns: Knowledge entities + VFS files
+
+// ๐ VFS-ONLY via brain.find()
+const markdownFiles = await brain.find({
+ where: { vfsType: 'file', extension: '.md' },
+ includeVFS: true // Required to find VFS entities
+})
+```
+
+#### 2. **Cross-Boundary Relationships**
+
+```typescript
+// Files can relate to knowledge entities
+await brain.relate({
+ from: conceptId, // Knowledge: NounType.Concept
+ to: fileId, // VFS: File entity
+ type: VerbType.DocumentedBy
+})
+
+// Query across boundaries
+const conceptDocs = await brain.getRelations({
+ from: conceptId,
+ type: VerbType.DocumentedBy
+})
+// Returns: VFS files that document the concept
+```
+
+#### 3. **VFS vs Traditional Filesystem**
+
+| Feature | Traditional FS | Brainy VFS |
+|---------|---------------|------------|
+| Search | Filename only | Semantic content search |
+| Organization | Hierarchy only | Hierarchy + Graph |
+| Metadata | Limited (size, dates) | Unlimited custom metadata |
+| Relationships | None | Full graph relationships |
+| Similarity | None | Find similar files |
+| Understanding | None | Vector embeddings |
+
+#### 4. **When to Use What**
+
+```typescript
+// Use vfs.* methods for file operations
+await vfs.writeFile('/path/to/file.txt', content)
+await vfs.readFile('/path/to/file.txt')
+await vfs.search('semantic query')
+
+// Use brain.* methods for knowledge operations
+await brain.add({ data: 'concept', type: NounType.Concept })
+await brain.find({ query: 'concept' }) // Excludes VFS by default
+
+// Use includeVFS for cross-boundary queries
+await brain.find({
+ query: 'documentation',
+ includeVFS: true // Search both knowledge AND files
+})
+```
+
+### Practice Exercises
+
+1. Create a project structure with docs, source code, tests
+2. Add semantic tags to files
+3. Search for "API documentation" and see VFS filtering in action
+4. Create relationships between code files and design documents
+5. Find files similar to a specific README
+6. Compare results with/without includeVFS
+
+### Next Steps
+Ready for production deployment? Level 5 covers **planet-scale architecture**.
+
+---
+
+## Level 5: Production Scale (90 minutes)
+
+### What You'll Learn
+- Cloud storage (GCS, S3, R2)
+- Performance optimization
+- Batch imports (CSV, Excel, PDF)
+- Metadata query optimization
+- Production best practices
+
+### Production-Ready Deployment
+
+```typescript
+import { Brainy, NounType } from '@soulcraft/brainy'
+
+// 1. PRODUCTION STORAGE - Google Cloud Storage (Native SDK)
+console.log('โ๏ธ Initializing production storage...\n')
+
+const brain = new Brainy({
+ storage: {
+ type: 'gcs-native', // Native GCS SDK (recommended)
+ gcsNativeStorage: {
+ bucketName: 'my-brainy-production',
+ // ADC (Application Default Credentials) - zero config in Cloud Run/GCE!
+ // Or provide credentials:
+ // keyFilename: '/path/to/service-account.json'
+ }
+ },
+
+ // Performance tuning
+ cache: {
+ maxSize: 10000, // Cache up to 10K entities
+ ttl: 600000 // 10 minute TTL
+ },
+
+ // Monitoring
+ verbose: process.env.NODE_ENV === 'development'
+})
+
+await brain.init()
+console.log('โ
Brainy initialized with GCS Native storage\n')
+
+// 2. BATCH IMPORT - CSV File
+console.log('๐ Importing CSV data...\n')
+
+const csvResult = await brain.import('./data/customers-1000.csv', {
+ vfsPath: '/imports/customers.csv', // Store in VFS
+ createEntities: true, // Create knowledge entities
+ batchSize: 100, // Process in batches of 100
+ onProgress: (done, total) => {
+ console.log(` Progress: ${done}/${total} (${(done/total*100).toFixed(1)}%)`)
+ }
+})
+
+console.log('\n๐ Import Results:')
+console.log(` Entities created: ${csvResult.stats.graphNodesCreated}`)
+console.log(` VFS files created: ${csvResult.stats.vfsFilesCreated}`)
+console.log(` Duration: ${csvResult.stats.duration}ms`)
+
+// 3. METADATA QUERY OPTIMIZATION
+console.log('\n\n๐ Metadata Query Optimization:\n')
+
+// Discover what fields are available
+const fields = await brain.getAvailableFields()
+console.log(`Available metadata fields: ${fields.length}`)
+console.log(` Top fields: ${fields.slice(0, 10).join(', ')}`)
+
+// Get field statistics (cardinality, types)
+const fieldStats = await brain.getFieldStatistics()
+console.log(`\nField statistics:`)
+const topFields = Array.from(fieldStats.entries()).slice(0, 5)
+topFields.forEach(([field, stats]) => {
+ console.log(` ${field}: ${stats.cardinality} unique values`)
+})
+
+// Get optimal query plan
+const queryPlan = await brain.getOptimalQueryPlan({
+ status: 'active',
+ year: 2024
+})
+console.log(`\nQuery plan:`)
+console.log(` Estimated results: ${queryPlan.estimatedResults}`)
+console.log(` Index usage: ${queryPlan.indexUsage.join(', ')}`)
+console.log(` Execution time: ~${queryPlan.estimatedMs}ms`)
+
+// 4. LARGE-SCALE BATCH OPERATIONS
+console.log('\n\n๐ฆ Large-Scale Batch Operations:\n')
+
+// Generate test data
+const testItems = Array.from({ length: 1000 }, (_, i) => ({
+ data: `Test entity ${i} - Machine learning and artificial intelligence`,
+ type: NounType.Document,
+ metadata: {
+ index: i,
+ category: i % 5 === 0 ? 'AI' : 'General',
+ priority: Math.random() > 0.5 ? 'high' : 'normal',
+ year: 2024
+ }
+}))
+
+console.log(`Adding 1000 entities...`)
+const startTime = Date.now()
+
+const batchResult = await brain.addMany({
+ items: testItems,
+ parallel: true,
+ chunkSize: 100,
+ onProgress: (done, total) => {
+ if (done % 200 === 0) console.log(` ${done}/${total}`)
+ }
+})
+
+const duration = Date.now() - startTime
+console.log(`\nโ
Batch add complete:`)
+console.log(` Success: ${batchResult.successful.length}`)
+console.log(` Failed: ${batchResult.failed.length}`)
+console.log(` Duration: ${duration}ms`)
+console.log(` Throughput: ${(batchResult.successful.length / (duration / 1000)).toFixed(0)} entities/sec`)
+
+// 5. ADVANCED CLUSTERING (Large Dataset)
+console.log('\n\n๐ค Clustering 1000 entities...\n')
+
+const neural = brain.neural()
+
+// Use fast clustering for large datasets
+const clusters = await neural.clusterFast({
+ maxClusters: 5
+})
+
+console.log(`Found ${clusters.length} clusters:`)
+clusters.forEach((cluster, i) => {
+ console.log(`\n Cluster ${i + 1}: ${cluster.label || cluster.id}`)
+ console.log(` Size: ${cluster.members.length} members`)
+ console.log(` Density: ${(cluster.density || 0).toFixed(3)}`)
+ if (cluster.metadata?.keywords) {
+ console.log(` Keywords: ${cluster.metadata.keywords.slice(0, 5).join(', ')}`)
+ }
+})
+
+// 6. PRODUCTION STATISTICS
+console.log('\n\n๐ Production Statistics:\n')
+
+const stats = brain.getStats()
+console.log(`Total Entities: ${stats.entities.total.toLocaleString()}`)
+console.log(`Total Relationships: ${stats.relationships.totalRelationships.toLocaleString()}`)
+console.log(`Graph Density: ${stats.density.toFixed(4)}`)
+
+console.log(`\nEntities by Type:`)
+Object.entries(stats.entities.byType)
+ .sort(([, a], [, b]) => (b as number) - (a as number))
+ .slice(0, 5)
+ .forEach(([type, count]) => {
+ console.log(` ${type}: ${(count as number).toLocaleString()}`)
+ })
+
+// 7. QUERY PERFORMANCE MONITORING
+console.log('\n\nโก Query Performance:\n')
+
+const perfStart = Date.now()
+const searchResults = await brain.find({
+ query: 'artificial intelligence machine learning',
+ where: { category: 'AI' },
+ limit: 100,
+ explain: true
+})
+const perfDuration = Date.now() - perfStart
+
+console.log(`Query completed in ${perfDuration}ms`)
+console.log(` Results: ${searchResults.length}`)
+console.log(` Avg score: ${(searchResults.reduce((sum, r) => sum + r.score, 0) / searchResults.length).toFixed(3)}`)
+
+// Show top result explanation
+if (searchResults[0]?.explanation) {
+ console.log(`\n Top result score breakdown:`)
+ console.log(` Vector: ${searchResults[0].explanation.vector?.toFixed(3) || 'N/A'}`)
+ console.log(` Metadata: ${searchResults[0].explanation.metadata?.toFixed(3) || 'N/A'}`)
+ console.log(` Graph: ${searchResults[0].explanation.graph?.toFixed(3) || 'N/A'}`)
+}
+
+// 8. CLEANUP & BEST PRACTICES
+console.log('\n\n๐งน Production Best Practices:\n')
+
+// Always flush before shutdown
+await brain.flush()
+console.log('โ
Flushed all data to storage')
+
+// Get final stats
+const finalStats = brain.getStats()
+console.log(`โ
Final entity count: ${finalStats.entities.total.toLocaleString()}`)
+
+// Clean shutdown
+await brain.close()
+console.log('โ
Brain closed cleanly')
+
+console.log('\n\n๐ Production Deployment Complete!')
+console.log('\n๐ Key Production Learnings:')
+console.log(' 1. Use native cloud storage (GCS, S3, R2) for persistence')
+console.log(' 2. Batch operations = 100x faster than individual ops')
+console.log(' 3. Metadata query optimization for complex filters')
+console.log(' 4. Monitor query performance with explain: true')
+console.log(' 5. Always flush() before shutdown')
+console.log(' 6. Use getStats() for O(1) counts (no expensive scans)')
+console.log(' 7. Stream large imports with progress callbacks')
+```
+
+### Key Concepts
+
+#### 1. **Storage Options Comparison**
+
+| Storage | Use Case | Performance | Cost | Setup |
+|---------|----------|-------------|------|-------|
+| Memory | Dev/testing | Fastest | Free | Zero config |
+| Filesystem | Local prod | Fast | Free | Local path |
+| GCS Native | GCP prod | Fast | $$$ | Service account |
+| S3 | AWS prod | Fast | $$$ | Access keys |
+| R2 | Cloudflare | Fast | $ | Access keys |
+
+#### 2. **GCS Native vs S3-Compatible**
+
+```typescript
+// โ
RECOMMENDED: GCS Native SDK
+{
+ storage: {
+ type: 'gcs-native',
+ gcsNativeStorage: {
+ bucketName: 'my-bucket'
+ // ADC handles auth automatically in Cloud Run/GCE!
+ }
+ }
+}
+
+// โ ๏ธ LEGACY: S3-compatible mode
+{
+ storage: {
+ type: 'gcs',
+ gcsStorage: {
+ bucketName: 'my-bucket',
+ accessKeyId: process.env.GCS_ACCESS_KEY,
+ secretAccessKey: process.env.GCS_SECRET_KEY
+ }
+ }
+}
+```
+
+#### 3. **Performance Optimization**
+
+```typescript
+// 1. Use batch operations
+await brain.addMany({ items, parallel: true, chunkSize: 100 })
+
+// 2. Enable caching
+const brain = new Brainy({
+ cache: { maxSize: 10000, ttl: 600000 }
+})
+
+// 3. Use metadata indexes for filtering
+await brain.find({
+ where: { status: 'active' }, // Uses MetadataIndexManager
+ limit: 100
+})
+
+// 4. Optimize query plans
+const plan = await brain.getOptimalQueryPlan(filters)
+// Use plan to choose best query strategy
+
+// 5. Use writeOnly for bulk imports
+await brain.add({
+ data,
+ type,
+ writeOnly: true // Skip validation for speed
+})
+```
+
+#### 4. **Import Strategies**
+
+```typescript
+// Small files (<10MB) - Direct import
+await brain.import('./data.csv')
+
+// Large files (>10MB) - Stream with progress
+await brain.import('./large-data.csv', {
+ batchSize: 1000,
+ onProgress: (done, total) => {
+ console.log(`${(done/total*100).toFixed(1)}%`)
+ }
+})
+
+// Very large files (>100MB) - External pipeline
+// Use streaming pipeline API for max control
+```
+
+#### 5. **Monitoring & Observability**
+
+```typescript
+// 1. Track query performance
+const start = Date.now()
+const results = await brain.find({ query, explain: true })
+const duration = Date.now() - start
+console.log(`Query: ${duration}ms, Results: ${results.length}`)
+
+// 2. Monitor graph statistics
+const stats = brain.getStats()
+console.log(`Density: ${stats.density}`) // Relationships per entity
+
+// 3. Track field cardinality
+const fieldStats = await brain.getFieldStatistics()
+// High cardinality fields = good for filtering
+
+// 4. Enable verbose logging in dev
+const brain = new Brainy({ verbose: true })
+```
+
+### Production Checklist
+
+#### Before Deployment
+
+- [ ] Choose cloud storage (GCS/S3/R2)
+- [ ] Set up authentication (service account/access keys)
+- [ ] Configure caching
+- [ ] Test batch operations
+- [ ] Benchmark query performance
+- [ ] Set up monitoring
+
+#### During Operation
+
+- [ ] Monitor query latency
+- [ ] Track entity/relationship counts
+- [ ] Watch for outliers
+- [ ] Optimize slow queries
+- [ ] Regular backups
+
+#### Scaling Considerations
+
+- [ ] Shard data by service/tenant
+- [ ] Use read replicas for queries
+- [ ] Implement rate limiting
+- [ ] Monitor storage costs
+- [ ] Plan for growth
+
+### Practice Exercises
+
+1. Deploy Brainy with GCS Native storage
+2. Import a 10,000 row CSV file
+3. Measure query performance for different filters
+4. Optimize a slow query using getOptimalQueryPlan()
+5. Set up monitoring dashboard
+6. Create backup/restore scripts
+
+---
+
+## ๐ Graduation: You're a Brainy Expert!
+
+### What You've Mastered
+
+โ
**Level 1**: Basic operations (add, find, search)
+โ
**Level 2**: Relationships & batch operations
+โ
**Level 3**: Triple Intelligence & Neural AI
+โ
**Level 4**: Virtual Filesystem
+โ
**Level 5**: Production deployment
+
+### Next Steps
+
+#### Advanced Topics
+
+- **Distributed Systems**: Sharding, replication, coordination
+- **Custom Augmentations**: Extend Brainy with plugins
+- **Streaming Pipelines**: Real-time data ingestion
+- **Security**: Encryption, access control, audit logs
+- **Framework Integration**: React, Vue, Next.js, Nuxt
+
+#### Resources
+
+- ๐ [API Reference](../API_REFERENCE.md) - Complete API documentation
+- ๐ [VFS Guide](../vfs/VFS_API_GUIDE.md) - Virtual Filesystem deep dive
+- ๐ค [Neural API](../guides/neural-api.md) - Advanced neural operations
+- ๐ [Distributed Guide](../guides/distributed-system.md) - Planet-scale architecture
+- ๐ฌ [Discord Community](https://discord.gg/brainy) - Get help, share projects
+
+#### Share Your Success
+
+Built something cool with Brainy? Share it with the community!
+
+- GitHub: https://github.com/soulcraft/brainy
+- Twitter: @brainydb
+- Discord: https://discord.gg/brainy
+
+---
+
+**Congratulations! You're now a Brainy expert ready to build production neural database applications! ๐**
diff --git a/docs/guides/import-progress-examples.md b/docs/guides/import-progress-examples.md
new file mode 100644
index 00000000..6489d19a
--- /dev/null
+++ b/docs/guides/import-progress-examples.md
@@ -0,0 +1,370 @@
+# Import Progress - Usage Examples
+
+**How to Use Progress Tracking in Your Applications**
+
+Brainy v4.5.0+ provides real-time progress tracking for **all 7 supported file formats** (CSV, PDF, Excel, JSON, Markdown, YAML, DOCX).
+
+> **โ ๏ธ KEY FEATURE:** The progress API is **100% standardized**. Write your progress handler ONCE and it works for ALL formats with zero format-specific code! See [Standard Import Progress API](./standard-import-progress.md) for the complete interface documentation.
+
+---
+
+## ๐ Quick Start
+
+### Basic Progress Tracking
+
+```typescript
+import { Brainy } from '@soulcraft/brainy'
+import * as fs from 'fs'
+
+const brain = await Brainy.create()
+
+// Import with progress tracking
+const result = await brain.import(fs.readFileSync('large-file.xlsx'), {
+ onProgress: (progress) => {
+ console.log(`Progress: ${progress.stage}`)
+ console.log(` Message: ${progress.message}`)
+ console.log(` Entities: ${progress.entities || 0}`)
+ console.log(` Relationships: ${progress.relationships || 0}`)
+ }
+})
+
+console.log(`Import complete: ${result.entities.length} entities created`)
+```
+
+**Expected Output:**
+```
+Progress: detecting
+ Message: Detecting format...
+ Entities: 0
+ Relationships: 0
+Progress: extracting
+ Message: Loading Excel workbook...
+ Entities: 0
+ Relationships: 0
+Progress: extracting
+ Message: Reading sheet: Sales (1/3)
+ Entities: 0
+ Relationships: 0
+Progress: extracting
+ Message: Parsing Excel (33%)
+ Entities: 0
+ Relationships: 0
+Progress: extracting
+ Message: Reading sheet: Products (2/3)
+ Entities: 0
+ Relationships: 0
+... (more progress updates)
+Progress: complete
+ Message: Import complete
+ Entities: 1523
+ Relationships: 892
+Import complete: 1523 entities created
+```
+
+---
+
+## ๐ฏ Universal Progress Handler (Works for ALL Formats)
+
+The examples below show format-specific messages, but **you don't need format-specific code**! The `ImportProgress` interface is the same for all formats:
+
+```typescript
+// ONE HANDLER FOR ALL FORMATS!
+function universalProgressHandler(progress) {
+ console.log(`[${progress.stage}] ${progress.message}`)
+
+ if (progress.processed && progress.total) {
+ console.log(` Progress: ${progress.processed}/${progress.total}`)
+ }
+
+ if (progress.entities || progress.relationships) {
+ console.log(` Extracted: ${progress.entities || 0} entities, ${progress.relationships || 0} relationships`)
+ }
+
+ if (progress.throughput && progress.eta) {
+ console.log(` Rate: ${progress.throughput.toFixed(1)}/sec, ETA: ${Math.round(progress.eta/1000)}s`)
+ }
+}
+
+// Use it for ANY format!
+await brain.import(csvBuffer, { onProgress: universalProgressHandler })
+await brain.import(pdfBuffer, { onProgress: universalProgressHandler })
+await brain.import(excelBuffer, { onProgress: universalProgressHandler })
+await brain.import(jsonBuffer, { onProgress: universalProgressHandler })
+await brain.import(markdownString, { onProgress: universalProgressHandler })
+await brain.import(yamlBuffer, { onProgress: universalProgressHandler })
+await brain.import(docxBuffer, { onProgress: universalProgressHandler })
+```
+
+---
+
+## ๐ What Different Formats Look Like (Same Handler!)
+
+The examples below show **what messages look like** for different formats using the **same universal handler** above.
+
+### CSV Import (Row-by-Row Progress)
+
+```typescript
+await brain.import(csvBuffer, {
+ format: 'csv',
+ onProgress: (progress) => {
+ if (progress.stage === 'extracting') {
+ // CSV reports: "Parsing CSV (45%)", "Extracted 1000 rows", etc.
+ console.log(progress.message)
+ }
+ }
+})
+```
+
+**CSV Progress Messages:**
+- โ
"Detecting CSV encoding and delimiter..."
+- โ
"Parsing CSV rows (delimiter: ",")"
+- โ
"Parsed 75%" (via bytes processed)
+- โ
"Extracted 1000 rows"
+- โ
"Converting types: 5000/10000 rows..."
+- โ
"CSV processing complete: 10000 rows"
+
+---
+
+### PDF Import (Page-by-Page Progress)
+
+```typescript
+await brain.import(pdfBuffer, {
+ format: 'pdf',
+ onProgress: (progress) => {
+ // PDF reports exact page numbers
+ console.log(progress.message)
+ // Example: "Processing page 5 of 23"
+ }
+})
+```
+
+**PDF Progress Messages:**
+- โ
"Loading PDF document..."
+- โ
"Processing 23 pages..."
+- โ
"Processing page 5 of 23"
+- โ
"Parsed 22%" (via bytes processed)
+- โ
"Extracted 156 items from PDF"
+- โ
"PDF complete: 23 pages, 156 items extracted"
+
+---
+
+### Excel Import (Sheet-by-Sheet Progress)
+
+```typescript
+await brain.import(excelBuffer, {
+ format: 'excel',
+ onProgress: (progress) => {
+ // Excel reports sheet names
+ console.log(progress.message)
+ // Example: "Reading sheet: Q2 Sales (2/5)"
+ }
+})
+```
+
+**Excel Progress Messages:**
+- โ
"Loading Excel workbook..."
+- โ
"Processing 3 sheets..."
+- โ
"Reading sheet: Sales (1/3)"
+- โ
"Parsing Excel (33%)" (via bytes processed)
+- โ
"Extracted 5234 rows from Excel"
+- โ
"Excel complete: 3 sheets, 5234 rows"
+
+---
+
+### JSON Import (Node Traversal)
+
+```typescript
+await brain.import(jsonBuffer, {
+ format: 'json',
+ onProgress: (progress) => {
+ // JSON reports every 10 nodes
+ console.log(`Processed ${progress.processed} nodes, found ${progress.entities} entities`)
+ }
+})
+```
+
+---
+
+### Markdown Import (Section-by-Section)
+
+```typescript
+await brain.import(markdownString, {
+ format: 'markdown',
+ onProgress: (progress) => {
+ console.log(`Section ${progress.processed}/${progress.total}`)
+ }
+})
+```
+
+---
+
+## ๐ฏ Building Progress UI Components
+
+### React Progress Bar
+
+```typescript
+function ImportProgress({ file }: { file: File }) {
+ const [progress, setProgress] = useState({
+ stage: 'idle',
+ message: '',
+ percent: 0,
+ entities: 0,
+ relationships: 0
+ })
+
+ const handleImport = async () => {
+ const buffer = await file.arrayBuffer()
+
+ await brain.import(Buffer.from(buffer), {
+ onProgress: (p) => {
+ setProgress({
+ stage: p.stage,
+ message: p.message,
+ // Estimate percentage from stage
+ percent: {
+ detecting: 10,
+ extracting: 50,
+ 'storing-vfs': 80,
+ 'storing-graph': 90,
+ complete: 100
+ }[p.stage] || 0,
+ entities: p.entities || 0,
+ relationships: p.relationships || 0
+ })
+ }
+ })
+ }
+
+ return (
+
+
+
{progress.message}
+
Entities: {progress.entities} | Relationships: {progress.relationships}
+
+ )
+}
+```
+
+---
+
+### CLI Progress Spinner
+
+```typescript
+import ora from 'ora'
+
+const spinner = ora('Starting import...').start()
+
+await brain.import(buffer, {
+ onProgress: (progress) => {
+ spinner.text = progress.message
+
+ if (progress.stage === 'complete') {
+ spinner.succeed(`Import complete: ${progress.entities} entities`)
+ }
+ }
+})
+```
+
+**CLI Output:**
+```
+โ Detecting format...
+โ Loading Excel workbook...
+โ น Reading sheet: Sales (1/3)
+โ ธ Parsing Excel (33%)
+โ ผ Reading sheet: Products (2/3)
+...
+โ Import complete: 1523 entities
+```
+
+---
+
+### Progress Dashboard with ETA
+
+```typescript
+let startTime = Date.now()
+let lastUpdate = startTime
+
+await brain.import(buffer, {
+ onProgress: (progress) => {
+ const elapsed = Date.now() - startTime
+ const rate = progress.entities / (elapsed / 1000) // entities/sec
+
+ console.clear()
+ console.log('Import Progress Dashboard')
+ console.log('========================')
+ console.log(`Stage: ${progress.stage}`)
+ console.log(`Status: ${progress.message}`)
+ console.log(`Entities: ${progress.entities}`)
+ console.log(`Relationships: ${progress.relationships}`)
+ console.log(`Rate: ${rate.toFixed(1)} entities/sec`)
+ console.log(`Elapsed: ${(elapsed / 1000).toFixed(1)}s`)
+ }
+})
+```
+
+---
+
+## ๐ง Advanced: Format-Specific Optimization
+
+### Detecting Format to Show Appropriate Progress
+
+```typescript
+const formatMessages = {
+ csv: (p) => `CSV: ${p.message}`,
+ pdf: (p) => `PDF: ${p.message}`,
+ excel: (p) => `Excel: ${p.message}`,
+ json: (p) => `JSON: ${p.processed} nodes, ${p.entities} entities`,
+ markdown: (p) => `Markdown: Section ${p.processed}/${p.total}`,
+ yaml: (p) => `YAML: ${p.processed} nodes`,
+ docx: (p) => `DOCX: ${p.processed} paragraphs`
+}
+
+await brain.import(buffer, {
+ onProgress: (progress) => {
+ // Format is available in progress.stage metadata
+ const message = formatMessages[detectedFormat]?.(progress) || progress.message
+ console.log(message)
+ }
+})
+```
+
+---
+
+## โก Performance Tips
+
+### Throttle UI Updates
+
+```typescript
+let lastUIUpdate = 0
+const THROTTLE_MS = 100 // Update UI max once per 100ms
+
+await brain.import(buffer, {
+ onProgress: (progress) => {
+ const now = Date.now()
+ if (now - lastUIUpdate < THROTTLE_MS && progress.stage !== 'complete') {
+ return // Skip this update
+ }
+
+ lastUIUpdate = now
+ updateUI(progress) // Only update every 100ms
+ }
+})
+```
+
+**Note:** Brainy already throttles progress callbacks internally, but additional UI throttling can help with heavy rendering.
+
+---
+
+## ๐ Summary
+
+โ
**All 7 formats** have consistent progress reporting
+โ
**Real-time updates** during long imports (no more "0%" hangs)
+โ
**Contextual messages** show exactly what's happening
+โ
**Build reliable tools** with standardized progress callbacks
+โ
**Workshop team problem SOLVED** - users see progress throughout import
+
+**Files Modified:**
+- 3 handlers: `csvHandler.ts`, `pdfHandler.ts`, `excelHandler.ts`
+- 7 importers: `SmartCSVImporter.ts`, `SmartPDFImporter.ts`, `SmartExcelImporter.ts`, `SmartJSONImporter.ts`, `SmartMarkdownImporter.ts`, `SmartYAMLImporter.ts`, `SmartDOCXImporter.ts`
+
+**Result:** Comprehensive, consistent progress tracking across ALL import formats!
diff --git a/docs/guides/import-progress-implementation.md b/docs/guides/import-progress-implementation.md
new file mode 100644
index 00000000..2b6ac5b7
--- /dev/null
+++ b/docs/guides/import-progress-implementation.md
@@ -0,0 +1,734 @@
+# Import Progress Implementation Guide
+**For Developers: How to Add Progress Tracking to ANY File Handler**
+
+> This guide shows the **standard pattern** for implementing rich progress tracking in Brainy import handlers. Follow this template for **all 7 supported formats** (CSV, PDF, Excel, JSON, Markdown, YAML, DOCX) or any future file format.
+
+---
+
+## ๐ Supported Formats & Consistent Progress Reporting
+
+> **โ ๏ธ IMPORTANT FOR DEVELOPERS:** The public API (`ImportProgress`) is 100% standardized across all formats. You can build ONE progress handler that works for CSV, PDF, Excel, JSON, Markdown, YAML, and DOCX with **zero format-specific code**. See [Standard Import Progress API](./standard-import-progress.md) for details.
+
+**ALL 7 formats now have consistent, standardized progress reporting** for building reliable import tools:
+
+| Format | Category | Progress Points | File Location | Status |
+|--------|----------|-----------------|---------------|--------|
+| **CSV** | Tabular | Parsing โ Row extraction โ Type conversion โ Complete | `handlers/csvHandler.ts` + `SmartCSVImporter.ts` | โ
Complete |
+| **PDF** | Document | Loading โ Page-by-page โ Item extraction โ Complete | `handlers/pdfHandler.ts` + `SmartPDFImporter.ts` | โ
Complete |
+| **Excel** | Tabular | Loading โ Sheet-by-sheet โ Row extraction โ Type conversion โ Complete | `handlers/excelHandler.ts` + `SmartExcelImporter.ts` | โ
Complete |
+| **JSON** | Structured | Parsing โ Node traversal (every 10 nodes) โ Complete | `SmartJSONImporter.ts` | โ
Complete |
+| **Markdown** | Document | Parsing โ Section-by-section โ Complete | `SmartMarkdownImporter.ts` | โ
Complete |
+| **YAML** | Structured | Parsing โ Node traversal (every 10 nodes) โ Complete | `SmartYAMLImporter.ts` | โ
Complete |
+| **DOCX** | Document | Parsing โ Paragraph-by-paragraph (every 10) โ Complete | `SmartDOCXImporter.ts` | โ
Complete |
+
+### The Standard Public API
+
+**Developers calling `brain.import()` see ONE standardized interface** regardless of format:
+
+```typescript
+// THE PUBLIC API - Same for ALL 7 formats!
+brain.import(buffer, {
+ onProgress: (progress: ImportProgress) => {
+ // These fields work for CSV, PDF, Excel, JSON, Markdown, YAML, DOCX
+ progress.stage // 'detecting' | 'extracting' | 'storing-vfs' | 'storing-graph' | 'complete'
+ progress.message // Human-readable status (varies by format, always readable)
+ progress.processed // Items processed (optional)
+ progress.total // Total items (optional)
+ progress.entities // Entities extracted (optional)
+ progress.relationships // Relationships inferred (optional)
+ progress.throughput // Items/sec (optional, during extraction)
+ progress.eta // Time remaining in ms (optional)
+ }
+})
+```
+
+**Internal Implementation** (for developers adding new format handlers):
+
+The table below shows how formats implement progress *internally*. Normal developers don't need to know this - they just use the standard `ImportProgress` interface above!
+
+```typescript
+// Internal: Binary formats use handler hooks (you added these!)
+interface FormatHandlerProgressHooks {
+ onBytesProcessed?: (bytes: number) => void
+ onCurrentItem?: (message: string) => void
+ onDataExtracted?: (count: number, total?: number) => void
+}
+
+// Internal: Text formats use importer callbacks
+interface ImporterProgressCallback {
+ onProgress?: (stats: { processed, total, entities, relationships }) => void
+}
+
+// Both are converted to ImportProgress by ImportCoordinator!
+```
+
+### Developer Benefits
+
+โ
**Consistent API** - Same pattern across all 7 formats
+โ
**Throttled Updates** - Progress reported every 10-1000 items (no spam)
+โ
**Contextual Messages** - "Processing page 5 of 23", "Reading sheet: Sales (2/5)"
+โ
**Real-time Estimates** - Users see progress during long imports
+โ
**Build Monitoring Tools** - Reliable progress data for UIs, dashboards, CLI tools
+
+---
+
+## ๐ฏ Overview
+
+As of v4.5.0, Brainy supports comprehensive, multi-dimensional progress tracking for imports:
+- **Bytes processed** (always available, most deterministic)
+- **Entities extracted** (AI extraction phase)
+- **Stage-specific metrics** (parsing: MB/s, extraction: entities/s)
+- **Time estimates** (remaining time, total time)
+- **Context information** ("Processing page 5 of 23")
+
+All handlers follow a simple, consistent pattern using **progress hooks**.
+
+---
+
+## ๐ The Progress Hooks Pattern
+
+### 1. Progress Hooks Interface
+
+```typescript
+export interface FormatHandlerProgressHooks {
+ /**
+ * Report bytes processed
+ * Call this as you read/parse the file
+ */
+ onBytesProcessed?: (bytes: number) => void
+
+ /**
+ * Set current processing context
+ * Examples: "Processing page 5", "Reading sheet: Q2 Sales"
+ */
+ onCurrentItem?: (item: string) => void
+
+ /**
+ * Report structured data extraction progress
+ * Examples: "Extracted 100 rows", "Parsed 50 paragraphs"
+ */
+ onDataExtracted?: (count: number, total?: number) => void
+}
+```
+
+### 2. Handler Options (Automatic)
+
+Progress hooks are automatically passed to your handler via `FormatHandlerOptions`:
+
+```typescript
+export interface FormatHandlerOptions {
+ // ... existing options ...
+
+ /**
+ * Progress hooks (v4.5.0)
+ * Handlers call these to report progress during processing
+ */
+ progressHooks?: FormatHandlerProgressHooks
+
+ /**
+ * Total file size in bytes (v4.5.0)
+ * Used for progress percentage calculation
+ */
+ totalBytes?: number
+}
+```
+
+**You don't need to modify FormatHandlerOptions** - it's already done!
+
+### 3. Standard Implementation Pattern
+
+Every handler follows these 5 steps:
+
+```typescript
+async process(data: Buffer | string, options: FormatHandlerOptions): Promise {
+ const progressHooks = options.progressHooks // Step 1: Get hooks
+
+ // Step 2: Report initial progress
+ if (progressHooks?.onCurrentItem) {
+ progressHooks.onCurrentItem('Starting import...')
+ }
+
+ // Step 3: Report bytes as you process
+ const buffer = Buffer.isBuffer(data) ? data : Buffer.from(data)
+ if (progressHooks?.onBytesProcessed) {
+ progressHooks.onBytesProcessed(0) // Start
+ }
+
+ // ... do parsing ...
+
+ if (progressHooks?.onBytesProcessed) {
+ progressHooks.onBytesProcessed(buffer.length) // Complete
+ }
+
+ // Step 4: Report data extraction
+ if (progressHooks?.onDataExtracted) {
+ progressHooks.onDataExtracted(data.length, data.length)
+ }
+
+ // Step 5: Report completion
+ if (progressHooks?.onCurrentItem) {
+ progressHooks.onCurrentItem(`Complete: ${data.length} items processed`)
+ }
+
+ return { format, data, metadata }
+}
+```
+
+---
+
+## ๐ Complete Example: CSV Handler
+
+Here's the **ACTUAL implementation** from CSV handler showing all the key progress points:
+
+```typescript
+async process(data: Buffer | string, options: FormatHandlerOptions): Promise {
+ const startTime = Date.now()
+ const progressHooks = options.progressHooks // โ
Step 1
+
+ // Convert to buffer if string
+ const buffer = Buffer.isBuffer(data) ? data : Buffer.from(data, 'utf-8')
+ const totalBytes = buffer.length
+
+ // โ
Step 2: Report start
+ if (progressHooks?.onBytesProcessed) {
+ progressHooks.onBytesProcessed(0)
+ }
+ if (progressHooks?.onCurrentItem) {
+ progressHooks.onCurrentItem('Detecting CSV encoding and delimiter...')
+ }
+
+ // Detect encoding
+ const detectedEncoding = options.encoding || this.detectEncodingSafe(buffer)
+ const text = buffer.toString(detectedEncoding as BufferEncoding)
+
+ // Detect delimiter
+ const delimiter = options.csvDelimiter || this.detectDelimiter(text)
+
+ // โ
Progress update: Parsing phase
+ if (progressHooks?.onCurrentItem) {
+ progressHooks.onCurrentItem(`Parsing CSV rows (delimiter: "${delimiter}")...`)
+ }
+
+ // Parse CSV
+ const records = parse(text, { /* options */ })
+
+ // โ
Step 3: Report bytes processed (entire file parsed)
+ if (progressHooks?.onBytesProcessed) {
+ progressHooks.onBytesProcessed(totalBytes)
+ }
+
+ const data = Array.isArray(records) ? records : [records]
+
+ // โ
Step 4: Report data extraction
+ if (progressHooks?.onDataExtracted) {
+ progressHooks.onDataExtracted(data.length, data.length)
+ }
+ if (progressHooks?.onCurrentItem) {
+ progressHooks.onCurrentItem(`Extracted ${data.length} rows, inferring types...`)
+ }
+
+ // Type inference and conversion
+ const fields = data.length > 0 ? Object.keys(data[0]) : []
+ const types = this.inferFieldTypes(data)
+
+ const convertedData = data.map((row, index) => {
+ const converted = this.convertRow(row, types)
+
+ // โ
Progress update every 1000 rows (avoid spam)
+ if (progressHooks?.onCurrentItem && index > 0 && index % 1000 === 0) {
+ progressHooks.onCurrentItem(`Converting types: ${index}/${data.length} rows...`)
+ }
+
+ return converted
+ })
+
+ // โ
Step 5: Report completion
+ if (progressHooks?.onCurrentItem) {
+ progressHooks.onCurrentItem(`CSV processing complete: ${convertedData.length} rows`)
+ }
+
+ return {
+ format: this.format,
+ data: convertedData,
+ metadata: { /* ... */ }
+ }
+}
+```
+
+### Key Progress Points in CSV Handler
+
+| Progress Point | Hook Used | Message Example |
+|----------------|-----------|-----------------|
+| **Start** | `onCurrentItem` | "Detecting CSV encoding and delimiter..." |
+| **Start bytes** | `onBytesProcessed(0)` | 0 bytes |
+| **Parsing** | `onCurrentItem` | "Parsing CSV rows (delimiter: \",\")..." |
+| **Bytes complete** | `onBytesProcessed(totalBytes)` | All bytes read |
+| **Data extracted** | `onDataExtracted(count, total)` | Number of rows extracted |
+| **Type conversion** | `onCurrentItem` (every 1000 rows) | "Converting types: 5000/10000 rows..." |
+| **Complete** | `onCurrentItem` | "CSV processing complete: 10000 rows" |
+
+---
+
+## ๐ Implementation Guide by File Type
+
+### Supported Formats
+
+Brainy supports **7 file formats** with full progress tracking:
+
+**Binary Formats** (use handlers):
+1. **CSV** - Row-by-row parsing with type inference
+2. **PDF** - Page-by-page extraction with table detection
+3. **Excel** - Sheet-by-sheet processing with formula evaluation
+
+**Text/Structured Formats** (parse inline):
+4. **JSON** - Recursive traversal of nested structures
+5. **Markdown** - Section-by-section with heading extraction
+6. **YAML** - Hierarchical traversal with relationship inference
+7. **DOCX** - Paragraph-by-paragraph with structure analysis
+
+---
+
+### PDF Handler (Multi-Page)
+
+```typescript
+async process(data: Buffer, options: FormatHandlerOptions): Promise {
+ const progressHooks = options.progressHooks
+ const totalBytes = data.length
+
+ // Report start
+ if (progressHooks?.onCurrentItem) {
+ progressHooks.onCurrentItem('Loading PDF document...')
+ }
+
+ const pdfDoc = await loadPDF(data)
+ const totalPages = pdfDoc.numPages
+
+ const extractedData: any[] = []
+ let bytesProcessed = 0
+
+ for (let pageNum = 1; pageNum <= totalPages; pageNum++) {
+ // โ
Report current page
+ if (progressHooks?.onCurrentItem) {
+ progressHooks.onCurrentItem(`Processing page ${pageNum} of ${totalPages}`)
+ }
+
+ const page = await pdfDoc.getPage(pageNum)
+ const text = await page.getTextContent()
+ extractedData.push(this.processPageText(text))
+
+ // โ
Estimate bytes processed (pages are sequential)
+ bytesProcessed = Math.floor((pageNum / totalPages) * totalBytes)
+ if (progressHooks?.onBytesProcessed) {
+ progressHooks.onBytesProcessed(bytesProcessed)
+ }
+
+ // โ
Report extraction progress
+ if (progressHooks?.onDataExtracted) {
+ progressHooks.onDataExtracted(pageNum, totalPages)
+ }
+ }
+
+ // Final progress
+ if (progressHooks?.onBytesProcessed) {
+ progressHooks.onBytesProcessed(totalBytes)
+ }
+ if (progressHooks?.onCurrentItem) {
+ progressHooks.onCurrentItem(`PDF complete: ${totalPages} pages processed`)
+ }
+
+ return { format: 'pdf', data: extractedData, metadata: { /* ... */ } }
+}
+```
+
+### Excel Handler (Multi-Sheet)
+
+```typescript
+async process(data: Buffer, options: FormatHandlerOptions): Promise {
+ const progressHooks = options.progressHooks
+ const totalBytes = data.length
+
+ // Load workbook
+ if (progressHooks?.onCurrentItem) {
+ progressHooks.onCurrentItem('Loading Excel workbook...')
+ }
+
+ const workbook = XLSX.read(data)
+ const sheetNames = options.excelSheets === 'all'
+ ? workbook.SheetNames
+ : (options.excelSheets || [workbook.SheetNames[0]])
+
+ const allData: any[] = []
+ let bytesProcessed = 0
+
+ for (let i = 0; i < sheetNames.length; i++) {
+ const sheetName = sheetNames[i]
+
+ // โ
Report current sheet
+ if (progressHooks?.onCurrentItem) {
+ progressHooks.onCurrentItem(`Reading sheet: ${sheetName} (${i + 1}/${sheetNames.length})`)
+ }
+
+ const sheet = workbook.Sheets[sheetName]
+ const sheetData = XLSX.utils.sheet_to_json(sheet)
+ allData.push(...sheetData)
+
+ // โ
Estimate bytes processed (sheets processed sequentially)
+ bytesProcessed = Math.floor(((i + 1) / sheetNames.length) * totalBytes)
+ if (progressHooks?.onBytesProcessed) {
+ progressHooks.onBytesProcessed(bytesProcessed)
+ }
+
+ // โ
Report data extraction
+ if (progressHooks?.onDataExtracted) {
+ progressHooks.onDataExtracted(allData.length, undefined) // Total unknown until done
+ }
+ }
+
+ // Final progress
+ if (progressHooks?.onBytesProcessed) {
+ progressHooks.onBytesProcessed(totalBytes)
+ }
+ if (progressHooks?.onCurrentItem) {
+ progressHooks.onCurrentItem(`Excel complete: ${sheetNames.length} sheets, ${allData.length} rows`)
+ }
+
+ return { format: 'xlsx', data: allData, metadata: { /* ... */ } }
+}
+```
+
+### JSON Importer (Recursive Traversal)
+
+```typescript
+async extract(data: any, options: SmartJSONOptions = {}): Promise {
+ // โ
Report parsing start
+ options.onProgress?.({ processed: 0, entities: 0, relationships: 0 })
+
+ // Parse JSON if string
+ let jsonData = typeof data === 'string' ? JSON.parse(data) : data
+
+ // โ
Report parsing complete
+ options.onProgress?.({ processed: 0, entities: 0, relationships: 0 })
+
+ // Traverse and extract (reports progress every 10 nodes)
+ const entities: ExtractedJSONEntity[] = []
+ const relationships: ExtractedJSONRelationship[] = []
+ let nodesProcessed = 0
+
+ await this.traverseJSON(
+ jsonData,
+ entities,
+ relationships,
+ () => {
+ nodesProcessed++
+ if (nodesProcessed % 10 === 0) {
+ options.onProgress?.({
+ processed: nodesProcessed,
+ entities: entities.length,
+ relationships: relationships.length
+ })
+ }
+ }
+ )
+
+ // โ
Report completion
+ options.onProgress?.({
+ processed: nodesProcessed,
+ entities: entities.length,
+ relationships: relationships.length
+ })
+
+ return { nodesProcessed, entitiesExtracted: entities.length, ... }
+}
+```
+
+### Markdown Importer (Section-Based)
+
+```typescript
+async extract(markdown: string, options: SmartMarkdownOptions = {}): Promise {
+ // โ
Report parsing start
+ options.onProgress?.({ processed: 0, total: 0, entities: 0, relationships: 0 })
+
+ // Parse markdown into sections
+ const parsedSections = this.parseMarkdown(markdown, options)
+
+ // โ
Report parsing complete
+ options.onProgress?.({ processed: 0, total: parsedSections.length, entities: 0, relationships: 0 })
+
+ // Process each section (reports progress after each section)
+ const sections: MarkdownSection[] = []
+ for (let i = 0; i < parsedSections.length; i++) {
+ const section = await this.processSection(parsedSections[i], options)
+ sections.push(section)
+
+ options.onProgress?.({
+ processed: i + 1,
+ total: parsedSections.length,
+ entities: sections.reduce((sum, s) => sum + s.entities.length, 0),
+ relationships: sections.reduce((sum, s) => sum + s.relationships.length, 0)
+ })
+ }
+
+ // โ
Report completion
+ options.onProgress?.({
+ processed: sections.length,
+ total: sections.length,
+ entities: sections.reduce((sum, s) => sum + s.entities.length, 0),
+ relationships: sections.reduce((sum, s) => sum + s.relationships.length, 0)
+ })
+
+ return { sectionsProcessed: sections.length, ... }
+}
+```
+
+### YAML Importer (Hierarchical)
+
+```typescript
+async extract(yamlContent: string | Buffer, options: SmartYAMLOptions = {}): Promise {
+ // โ
Report parsing start
+ options.onProgress?.({ processed: 0, entities: 0, relationships: 0 })
+
+ // Parse YAML
+ const yamlString = typeof yamlContent === 'string' ? yamlContent : yamlContent.toString('utf-8')
+ const data = yaml.load(yamlString)
+
+ // โ
Report parsing complete
+ options.onProgress?.({ processed: 0, entities: 0, relationships: 0 })
+
+ // Traverse YAML structure (reports progress every 10 nodes)
+ // ... similar to JSON traversal ...
+
+ // โ
Report completion (already implemented)
+ options.onProgress?.({
+ processed: nodesProcessed,
+ entities: entities.length,
+ relationships: relationships.length
+ })
+
+ return { nodesProcessed, entitiesExtracted: entities.length, ... }
+}
+```
+
+### DOCX Importer (Paragraph-Based)
+
+```typescript
+async extract(buffer: Buffer, options: SmartDOCXOptions = {}): Promise {
+ // โ
Report parsing start
+ options.onProgress?.({ processed: 0, entities: 0, relationships: 0 })
+
+ // Extract text and HTML using Mammoth
+ const textResult = await mammoth.extractRawText({ buffer })
+ const htmlResult = await mammoth.convertToHtml({ buffer })
+
+ // โ
Report parsing complete
+ options.onProgress?.({ processed: 0, entities: 0, relationships: 0 })
+
+ // Process paragraphs (reports progress every 10 paragraphs)
+ const paragraphs = textResult.value.split(/\n\n+/).filter(p => p.trim().length >= minLength)
+
+ for (let i = 0; i < paragraphs.length; i++) {
+ await this.processParagraph(paragraphs[i])
+
+ if (i % 10 === 0) {
+ options.onProgress?.({
+ processed: i + 1,
+ entities: entities.length,
+ relationships: relationships.length
+ })
+ }
+ }
+
+ // โ
Report completion (already implemented)
+ options.onProgress?.({
+ processed: paragraphs.length,
+ entities: entities.length,
+ relationships: relationships.length
+ })
+
+ return { paragraphsProcessed: paragraphs.length, ... }
+}
+```
+
+---
+
+## ๐ฏ Best Practices
+
+### 1. Always Check if Hooks Exist
+
+Progress hooks are **optional**. Always check before calling:
+
+```typescript
+// โ
Good - safe
+if (progressHooks?.onBytesProcessed) {
+ progressHooks.onBytesProcessed(bytes)
+}
+
+// โ Bad - will crash if hooks undefined
+progressHooks.onBytesProcessed(bytes) // TypeError!
+```
+
+### 2. Report Bytes at Start and End
+
+```typescript
+// โ
Good - clear start and end
+progressHooks?.onBytesProcessed(0) // Start
+// ... processing ...
+progressHooks?.onBytesProcessed(totalBytes) // End
+
+// โ Bad - no clear boundaries
+// ... just start processing without reporting start
+```
+
+### 3. Throttle Frequent Updates
+
+```typescript
+// โ
Good - report every 1000 items
+for (let i = 0; i < items.length; i++) {
+ processItem(items[i])
+
+ if (i > 0 && i % 1000 === 0) {
+ progressHooks?.onCurrentItem(`Processing: ${i}/${items.length}`)
+ }
+}
+
+// โ Bad - report EVERY item (spam!)
+for (let i = 0; i < items.length; i++) {
+ processItem(items[i])
+ progressHooks?.onCurrentItem(`Processing: ${i}/${items.length}`) // 1M callbacks!
+}
+```
+
+### 4. Provide Contextual Messages
+
+```typescript
+// โ
Good - specific and helpful
+progressHooks?.onCurrentItem('Parsing CSV rows (delimiter: ",")')
+progressHooks?.onCurrentItem('Processing page 5 of 23')
+progressHooks?.onCurrentItem('Reading sheet: Q2 Sales Data')
+
+// โ Bad - vague
+progressHooks?.onCurrentItem('Processing...')
+progressHooks?.onCurrentItem('Working...')
+```
+
+### 5. Report Data Extraction with Totals (if known)
+
+```typescript
+// โ
Good - total known
+progressHooks?.onDataExtracted(100, 1000) // 100 of 1000 rows
+
+// โ
Also good - total unknown (streaming)
+progressHooks?.onDataExtracted(100, undefined) // 100 rows so far
+
+// โ
Also good - complete
+progressHooks?.onDataExtracted(1000, 1000) // All 1000 rows
+```
+
+---
+
+## ๐ง Testing Your Handler
+
+### Manual Test
+
+```typescript
+import { CSVHandler } from './csvHandler.js'
+import * as fs from 'fs'
+
+const handler = new CSVHandler()
+const data = fs.readFileSync('./test.csv')
+
+const result = await handler.process(data, {
+ filename: 'test.csv',
+ progressHooks: {
+ onBytesProcessed: (bytes) => {
+ console.log(`Bytes: ${bytes}`)
+ },
+ onCurrentItem: (item) => {
+ console.log(`Status: ${item}`)
+ },
+ onDataExtracted: (count, total) => {
+ console.log(`Extracted: ${count}${total ? `/${total}` : ''}`)
+ }
+ }
+})
+
+console.log(`Complete: ${result.data.length} rows`)
+```
+
+### Expected Output
+
+```
+Status: Detecting CSV encoding and delimiter...
+Bytes: 0
+Status: Parsing CSV rows (delimiter: ",")...
+Bytes: 52438
+Extracted: 1000/1000
+Status: Extracted 1000 rows, inferring types...
+Status: CSV processing complete: 1000 rows
+Complete: 1000 rows
+```
+
+---
+
+## ๐ Progress Flow Diagram
+
+```
+User Imports File
+ โ
+ImportManager
+ โ
+Creates ProgressTracker
+ โ
+Calls Handler.process() with progressHooks
+ โ
+Handler Reports Progress:
+ โโ onBytesProcessed(0) โ ProgressTracker โ overall_progress calculated
+ โโ onCurrentItem("Parsing...") โ ProgressTracker โ stage_message updated
+ โโ onBytesProcessed(bytes) โ ProgressTracker โ bytes_per_second calculated
+ โโ onDataExtracted(count) โ ProgressTracker โ entities_extracted updated
+ โโ onCurrentItem("Complete") โ ProgressTracker โ final progress
+ โ
+ProgressTracker emits to callback (throttled 100ms)
+ โ
+User sees:
+ "Overall: 45% | PARSING | 12.5 MB/s | Parsing CSV rows..."
+```
+
+---
+
+## โ
Checklist for New Handlers
+
+When implementing a new file format handler:
+
+- [ ] Get `progressHooks` from `options`
+- [ ] Get `totalBytes` (if available)
+- [ ] Report `onBytesProcessed(0)` at start
+- [ ] Report `onCurrentItem()` for key stages
+- [ ] Report `onBytesProcessed()` as you process
+- [ ] Report `onDataExtracted()` when you extract data
+- [ ] Throttle frequent updates (every 1000 items max)
+- [ ] Report `onBytesProcessed(totalBytes)` at end
+- [ ] Report final `onCurrentItem()` with summary
+- [ ] Test with progress callback to verify output
+
+---
+
+## ๐ Summary
+
+**The Pattern (5 Steps)**:
+1. Get `progressHooks` from options
+2. Report start (`onBytesProcessed(0)`, `onCurrentItem("Starting...")`)
+3. Report progress as you process (`onBytesProcessed(bytes)`, `onCurrentItem("Page 5...")`)
+4. Report data extraction (`onDataExtracted(count, total)`)
+5. Report completion (`onBytesProcessed(totalBytes)`, `onCurrentItem("Complete")`)
+
+**Always Check**: `progressHooks?.method()`
+
+**Throttle**: Report every N items, not every single item
+
+**Context**: Provide specific, helpful messages
+
+**Testing**: Use manual test with console.log callbacks
+
+---
+
+**This pattern makes it trivial to add progress tracking to ANY file format. Copy this template and adapt for your handler!**
diff --git a/docs/guides/standard-import-progress.md b/docs/guides/standard-import-progress.md
new file mode 100644
index 00000000..d57f8803
--- /dev/null
+++ b/docs/guides/standard-import-progress.md
@@ -0,0 +1,453 @@
+# Standard Import Progress API
+
+## โ
Build Once, Works for ALL Formats
+
+**Brainy provides a 100% standardized progress API** - write your UI/tool once, and it works for all 7 supported formats (CSV, PDF, Excel, JSON, Markdown, YAML, DOCX) with **zero format-specific code**.
+
+---
+
+## ๐ฏ The Standard Interface
+
+### One Interface for Everything
+
+```typescript
+import { Brainy } from '@soulcraft/brainy'
+
+const brain = await Brainy.create()
+
+// THIS CODE WORKS FOR ALL 7 FORMATS - NO FORMAT-SPECIFIC LOGIC NEEDED!
+await brain.import(anyBuffer, {
+ onProgress: (progress) => {
+ // Standard fields - ALWAYS available regardless of format
+ console.log(progress.stage) // Current stage
+ console.log(progress.message) // Human-readable status
+
+ // Optional fields - available when relevant
+ console.log(progress.processed) // Items processed so far
+ console.log(progress.total) // Total items (if known)
+ console.log(progress.entities) // Entities extracted
+ console.log(progress.relationships) // Relationships inferred
+ console.log(progress.throughput) // Items/sec (during extraction)
+ console.log(progress.eta) // Time remaining in ms
+ }
+})
+```
+
+### The Complete Interface
+
+```typescript
+interface ImportProgress {
+ // === ALWAYS PRESENT ===
+
+ /** High-level stage (5 stages for all formats) */
+ stage: 'detecting' | 'extracting' | 'storing-vfs' | 'storing-graph' | 'complete'
+
+ /** Human-readable status message */
+ message: string
+
+ // === AVAILABLE WHEN RELEVANT ===
+
+ /** Items processed (rows, pages, nodes, etc.) */
+ processed?: number
+
+ /** Total items to process (if known ahead of time) */
+ total?: number
+
+ /** Entities extracted so far */
+ entities?: number
+
+ /** Relationships inferred so far */
+ relationships?: number
+
+ /** Processing rate (items per second) */
+ throughput?: number
+
+ /** Estimated time remaining (milliseconds) */
+ eta?: number
+
+ /** Whether data is queryable at this point (v4.2.0+) */
+ queryable?: boolean
+}
+```
+
+---
+
+## ๐จ Generic UI Components
+
+### React Progress Component (Works for ALL Formats)
+
+```typescript
+import { useState } from 'react'
+import { Brainy } from '@soulcraft/brainy'
+
+function UniversalImportProgress({ file }: { file: File }) {
+ const [progress, setProgress] = useState({
+ stage: 'idle',
+ message: 'Ready to import',
+ percent: 0,
+ entities: 0,
+ relationships: 0
+ })
+
+ const handleImport = async () => {
+ const buffer = await file.arrayBuffer()
+ const brain = await Brainy.create()
+
+ await brain.import(Buffer.from(buffer), {
+ // THIS WORKS FOR CSV, PDF, EXCEL, JSON, MARKDOWN, YAML, DOCX!
+ onProgress: (p) => {
+ setProgress({
+ stage: p.stage,
+ message: p.message,
+
+ // Calculate percentage from stage + processed/total
+ percent: calculatePercent(p),
+
+ entities: p.entities || 0,
+ relationships: p.relationships || 0
+ })
+ }
+ })
+ }
+
+ // Helper: Calculate percentage from progress
+ function calculatePercent(p: ImportProgress): number {
+ // Use processed/total if available
+ if (p.processed && p.total) {
+ return Math.round((p.processed / p.total) * 100)
+ }
+
+ // Otherwise estimate from stage
+ const stagePercents = {
+ detecting: 5,
+ extracting: 50,
+ 'storing-vfs': 80,
+ 'storing-graph': 90,
+ complete: 100
+ }
+ return stagePercents[p.stage] || 0
+ }
+
+ return (
+
+ {/* Stage Indicator */}
+
+ {['detecting', 'extracting', 'storing-vfs', 'storing-graph', 'complete'].map(s => (
+
+ {s}
+
+ ))}
+
+
+ {/* Progress Bar */}
+
+
+ {/* Status Message (format-specific but always readable) */}
+
{progress.message}
+
+ {/* Counts */}
+
+ Entities: {progress.entities}
+ Relationships: {progress.relationships}
+
+
+ )
+}
+```
+
+**This component works perfectly for:**
+- โ
CSV files with 10,000 rows
+- โ
PDF documents with 200 pages
+- โ
Excel workbooks with 5 sheets
+- โ
JSON files with nested structures
+- โ
Markdown documents with sections
+- โ
YAML configuration files
+- โ
DOCX documents with paragraphs
+
+**No format detection needed. No format-specific rendering. Just works.**
+
+---
+
+### CLI Progress Indicator (Works for ALL Formats)
+
+```typescript
+import ora from 'ora'
+import { Brainy } from '@soulcraft/brainy'
+
+async function importWithProgress(filePath: string) {
+ const spinner = ora('Starting import...').start()
+ const brain = await Brainy.create()
+
+ try {
+ await brain.import(filePath, {
+ // THIS WORKS FOR ALL 7 FORMATS!
+ onProgress: (p) => {
+ // Update spinner text with current message
+ spinner.text = p.message
+
+ // Add counts if available
+ if (p.entities || p.relationships) {
+ spinner.text += ` (${p.entities || 0} entities, ${p.relationships || 0} relationships)`
+ }
+
+ // Add throughput/ETA if available (during extraction)
+ if (p.throughput && p.eta) {
+ const etaSec = Math.round(p.eta / 1000)
+ spinner.text += ` [${p.throughput.toFixed(1)}/sec, ETA: ${etaSec}s]`
+ }
+
+ // Change spinner when complete
+ if (p.stage === 'complete') {
+ spinner.succeed(p.message)
+ }
+ }
+ })
+ } catch (error) {
+ spinner.fail(`Import failed: ${error.message}`)
+ }
+}
+
+// Works for ANY format!
+await importWithProgress('data.csv')
+await importWithProgress('document.pdf')
+await importWithProgress('workbook.xlsx')
+await importWithProgress('config.yaml')
+```
+
+**CLI Output (same code, different formats):**
+
+```bash
+# CSV Import
+โ Detecting format...
+โ Parsing CSV rows (delimiter: ",")
+โ น Extracting entities from csv (45 rows/sec, ETA: 120s) (150 entities, 45 relationships)
+โ ธ Extracting entities from csv (45 rows/sec, ETA: 60s) (750 entities, 223 relationships)
+โ Import complete (1350 entities, 401 relationships)
+
+# PDF Import
+โ Detecting format...
+โ Loading PDF document...
+โ น Processing page 5 of 23
+โ ธ Extracting entities from pdf (2.5 pages/sec, ETA: 30s) (45 entities, 12 relationships)
+โ Import complete (156 entities, 89 relationships)
+
+# Excel Import
+โ Detecting format...
+โ Loading Excel workbook...
+โ น Reading sheet: Sales (2/5)
+โ ธ Extracting entities from excel (120 rows/sec, ETA: 45s) (500 entities, 234 relationships)
+โ Import complete (2340 entities, 892 relationships)
+```
+
+**Same code. Different formats. Perfect progress for all.**
+
+---
+
+### Dashboard with Real-Time Stats (Works for ALL Formats)
+
+```typescript
+function ImportDashboard() {
+ const [stats, setStats] = useState({
+ stage: '',
+ message: '',
+ elapsed: 0,
+ entities: 0,
+ relationships: 0,
+ throughput: 0,
+ eta: 0
+ })
+
+ const startTime = Date.now()
+
+ const handleImport = async (file: File) => {
+ await brain.import(await file.arrayBuffer(), {
+ // UNIVERSAL PROGRESS HANDLER - WORKS FOR ALL FORMATS!
+ onProgress: (p) => {
+ setStats({
+ stage: p.stage,
+ message: p.message,
+ elapsed: Date.now() - startTime,
+ entities: p.entities || 0,
+ relationships: p.relationships || 0,
+ throughput: p.throughput || 0,
+ eta: p.eta || 0
+ })
+ }
+ })
+ }
+
+ return (
+
+
Import Progress
+
+
+
+ {stats.stage}
+
+
+
+
+ {stats.message}
+
+
+
+
+ {(stats.elapsed / 1000).toFixed(1)}s
+
+
+
+
+ {stats.entities.toLocaleString()}
+
+
+
+
+ {stats.relationships.toLocaleString()}
+
+
+ {stats.throughput > 0 && (
+
+
+ {stats.throughput.toFixed(1)} items/sec
+
+ )}
+
+ {stats.eta > 0 && (
+
+
+ {(stats.eta / 1000).toFixed(0)}s
+
+ )}
+
+ )
+}
+```
+
+**This dashboard shows live stats for ANY format** - CSV, PDF, Excel, JSON, Markdown, YAML, DOCX.
+
+---
+
+## ๐ What Messages Look Like (Format-Specific Text, Standard Fields)
+
+While the **fields are standardized**, the **message text** varies by format to be most helpful:
+
+```typescript
+// CSV Import Messages
+"Detecting format..."
+"Parsing CSV rows (delimiter: ",")"
+"Extracted 1000 rows, inferring types..."
+"Extracting entities from csv (45 rows/sec, ETA: 120s)..."
+"Creating VFS structure..."
+"Import complete"
+
+// PDF Import Messages
+"Detecting format..."
+"Loading PDF document..."
+"Processing page 5 of 23"
+"Extracting entities from pdf (2.5 pages/sec, ETA: 30s)..."
+"Creating VFS structure..."
+"Import complete"
+
+// Excel Import Messages
+"Detecting format..."
+"Loading Excel workbook..."
+"Reading sheet: Sales (2/5)"
+"Extracting entities from excel (120 rows/sec, ETA: 45s)..."
+"Creating VFS structure..."
+"Import complete"
+```
+
+**Key Point:** You can display `progress.message` directly in your UI **without parsing it**. It's always human-readable and contextually appropriate.
+
+---
+
+## โ
The 5 Standard Stages (Same for ALL Formats)
+
+Every import goes through these 5 stages in order:
+
+| Stage | Duration | Description | Fields Available |
+|-------|----------|-------------|------------------|
+| **detecting** | ~1% | Format detection | `stage`, `message` |
+| **extracting** | ~70% | Parse file + AI extraction | `stage`, `message`, `processed`, `total`, `entities`, `relationships`, `throughput`, `eta` |
+| **storing-vfs** | ~5% | Create file structure | `stage`, `message` |
+| **storing-graph** | ~20% | Create graph nodes | `stage`, `message`, `entities`, `relationships` |
+| **complete** | ~1% | Finalize | `stage`, `message`, `entities`, `relationships` |
+
+**These 5 stages are the same whether you're importing:**
+- A 10MB CSV file with 50,000 rows
+- A 200-page PDF document
+- A 5-sheet Excel workbook
+- A nested JSON structure
+- A Markdown document
+- A YAML configuration
+- A DOCX document
+
+---
+
+## ๐ฏ Why This Matters
+
+### Build Tools That Work for Everything
+
+```typescript
+// ONE progress handler for your entire application
+function universalProgressHandler(progress: ImportProgress) {
+ // Update UI (works for all formats)
+ updateProgressBar(progress)
+ updateStatusText(progress.message)
+ updateCounts(progress.entities, progress.relationships)
+
+ // Log to analytics (works for all formats)
+ analytics.track('import_progress', {
+ stage: progress.stage,
+ processed: progress.processed,
+ total: progress.total
+ })
+
+ // Send to monitoring (works for all formats)
+ monitoring.gauge('import.entities', progress.entities)
+ monitoring.gauge('import.throughput', progress.throughput)
+}
+
+// Use it everywhere
+await brain.import(csvFile, { onProgress: universalProgressHandler })
+await brain.import(pdfFile, { onProgress: universalProgressHandler })
+await brain.import(excelFile, { onProgress: universalProgressHandler })
+await brain.import(jsonFile, { onProgress: universalProgressHandler })
+```
+
+### No Format Detection Needed
+
+```typescript
+// โ DON'T DO THIS (format-specific handling)
+if (format === 'csv') {
+ // CSV-specific progress code
+} else if (format === 'pdf') {
+ // PDF-specific progress code
+} else if (format === 'excel') {
+ // Excel-specific progress code
+}
+
+// โ
DO THIS (universal handling)
+onProgress: (p) => {
+ // Works for ALL formats!
+ updateUI(p.stage, p.message, p.entities, p.relationships)
+}
+```
+
+---
+
+## ๐ Summary
+
+โ
**100% Standardized** - Same `ImportProgress` interface for all 7 formats
+โ
**Build Once** - Your progress UI works for CSV, PDF, Excel, JSON, Markdown, YAML, DOCX
+โ
**No Format Detection** - No need to check file type in your progress handler
+โ
**Human-Readable Messages** - Display `progress.message` directly, no parsing needed
+โ
**Standard Fields** - `stage`, `processed`, `total`, `entities`, `relationships` work everywhere
+โ
**Optional Enhancements** - `throughput`, `eta` available during extraction (all formats)
+
+**The Workshop team (and any developer) can now build monitoring tools, dashboards, CLIs, and UIs that work perfectly for all import formats with zero format-specific code!**
diff --git a/src/augmentations/intelligentImport/handlers/csvHandler.ts b/src/augmentations/intelligentImport/handlers/csvHandler.ts
index 40a70d2d..87951ad4 100644
--- a/src/augmentations/intelligentImport/handlers/csvHandler.ts
+++ b/src/augmentations/intelligentImport/handlers/csvHandler.ts
@@ -34,9 +34,19 @@ export class CSVHandler extends BaseFormatHandler {
async process(data: Buffer | string, options: FormatHandlerOptions): Promise {
const startTime = Date.now()
+ const progressHooks = options.progressHooks
// Convert to buffer if string
const buffer = Buffer.isBuffer(data) ? data : Buffer.from(data, 'utf-8')
+ const totalBytes = buffer.length
+
+ // v4.5.0: Report total bytes for progress tracking
+ if (progressHooks?.onBytesProcessed) {
+ progressHooks.onBytesProcessed(0)
+ }
+ if (progressHooks?.onCurrentItem) {
+ progressHooks.onCurrentItem('Detecting CSV encoding and delimiter...')
+ }
// Detect encoding
const detectedEncoding = options.encoding || this.detectEncodingSafe(buffer)
@@ -45,6 +55,11 @@ export class CSVHandler extends BaseFormatHandler {
// Detect delimiter if not specified
const delimiter = options.csvDelimiter || this.detectDelimiter(text)
+ // v4.5.0: Report progress - parsing started
+ if (progressHooks?.onCurrentItem) {
+ progressHooks.onCurrentItem(`Parsing CSV rows (delimiter: "${delimiter}")...`)
+ }
+
// Parse CSV
const hasHeaders = options.csvHeaders !== false
const maxRows = options.maxRows
@@ -60,23 +75,47 @@ export class CSVHandler extends BaseFormatHandler {
cast: false // We'll do type inference ourselves
})
+ // v4.5.0: Report bytes processed (entire file parsed)
+ if (progressHooks?.onBytesProcessed) {
+ progressHooks.onBytesProcessed(totalBytes)
+ }
+
// Convert to array of objects
const data = Array.isArray(records) ? records : [records]
+ // v4.5.0: Report data extraction progress
+ if (progressHooks?.onDataExtracted) {
+ progressHooks.onDataExtracted(data.length, data.length)
+ }
+ if (progressHooks?.onCurrentItem) {
+ progressHooks.onCurrentItem(`Extracted ${data.length} rows, inferring types...`)
+ }
+
// Infer types and convert values
const fields = data.length > 0 ? Object.keys(data[0]) : []
const types = this.inferFieldTypes(data)
- const convertedData = data.map(row => {
+ const convertedData = data.map((row, index) => {
const converted: Record = {}
for (const [key, value] of Object.entries(row)) {
converted[key] = this.convertValue(value, types[key] || 'string')
}
+
+ // v4.5.0: Report progress every 1000 rows
+ if (progressHooks?.onCurrentItem && index > 0 && index % 1000 === 0) {
+ progressHooks.onCurrentItem(`Converting types: ${index}/${data.length} rows...`)
+ }
+
return converted
})
const processingTime = Date.now() - startTime
+ // v4.5.0: Final progress update
+ if (progressHooks?.onCurrentItem) {
+ progressHooks.onCurrentItem(`CSV processing complete: ${convertedData.length} rows`)
+ }
+
return {
format: this.format,
data: convertedData,
diff --git a/src/augmentations/intelligentImport/handlers/excelHandler.ts b/src/augmentations/intelligentImport/handlers/excelHandler.ts
index 0b32609d..f5f7e314 100644
--- a/src/augmentations/intelligentImport/handlers/excelHandler.ts
+++ b/src/augmentations/intelligentImport/handlers/excelHandler.ts
@@ -21,9 +21,19 @@ export class ExcelHandler extends BaseFormatHandler {
async process(data: Buffer | string, options: FormatHandlerOptions): Promise {
const startTime = Date.now()
+ const progressHooks = options.progressHooks
// Convert to buffer if string (though Excel should always be binary)
const buffer = Buffer.isBuffer(data) ? data : Buffer.from(data, 'binary')
+ const totalBytes = buffer.length
+
+ // v4.5.0: Report start
+ if (progressHooks?.onBytesProcessed) {
+ progressHooks.onBytesProcessed(0)
+ }
+ if (progressHooks?.onCurrentItem) {
+ progressHooks.onCurrentItem('Loading Excel workbook...')
+ }
try {
// Read workbook
@@ -37,11 +47,25 @@ export class ExcelHandler extends BaseFormatHandler {
// Determine which sheets to process
const sheetsToProcess = this.getSheetsToProcess(workbook, options)
+ // v4.5.0: Report workbook loaded
+ if (progressHooks?.onCurrentItem) {
+ progressHooks.onCurrentItem(`Processing ${sheetsToProcess.length} sheets...`)
+ }
+
// Extract data from sheets
const allData: Array> = []
const sheetMetadata: Record = {}
- for (const sheetName of sheetsToProcess) {
+ for (let sheetIndex = 0; sheetIndex < sheetsToProcess.length; sheetIndex++) {
+ const sheetName = sheetsToProcess[sheetIndex]
+
+ // v4.5.0: Report current sheet
+ if (progressHooks?.onCurrentItem) {
+ progressHooks.onCurrentItem(
+ `Reading sheet: ${sheetName} (${sheetIndex + 1}/${sheetsToProcess.length})`
+ )
+ }
+
const sheet = workbook.Sheets[sheetName]
if (!sheet) continue
@@ -92,6 +116,25 @@ export class ExcelHandler extends BaseFormatHandler {
columnCount: headers.length,
headers
}
+
+ // v4.5.0: Estimate bytes processed (sheets are sequential)
+ const bytesProcessed = Math.floor(((sheetIndex + 1) / sheetsToProcess.length) * totalBytes)
+ if (progressHooks?.onBytesProcessed) {
+ progressHooks.onBytesProcessed(bytesProcessed)
+ }
+
+ // v4.5.0: Report extraction progress
+ if (progressHooks?.onDataExtracted) {
+ progressHooks.onDataExtracted(allData.length, undefined) // Total unknown until complete
+ }
+ }
+
+ // v4.5.0: Report data extraction complete
+ if (progressHooks?.onCurrentItem) {
+ progressHooks.onCurrentItem(`Extracted ${allData.length} rows, inferring types...`)
+ }
+ if (progressHooks?.onDataExtracted) {
+ progressHooks.onDataExtracted(allData.length, allData.length)
}
// Infer types (excluding _sheet field)
@@ -99,7 +142,7 @@ export class ExcelHandler extends BaseFormatHandler {
const types = this.inferFieldTypes(allData)
// Convert values to appropriate types
- const convertedData = allData.map(row => {
+ const convertedData = allData.map((row, index) => {
const converted: Record = {}
for (const [key, value] of Object.entries(row)) {
if (key === '_sheet') {
@@ -108,11 +151,29 @@ export class ExcelHandler extends BaseFormatHandler {
converted[key] = this.convertValue(value, types[key] || 'string')
}
}
+
+ // v4.5.0: Report progress every 1000 rows (avoid spam)
+ if (progressHooks?.onCurrentItem && index > 0 && index % 1000 === 0) {
+ progressHooks.onCurrentItem(`Converting types: ${index}/${allData.length} rows...`)
+ }
+
return converted
})
+ // v4.5.0: Final progress - all bytes processed
+ if (progressHooks?.onBytesProcessed) {
+ progressHooks.onBytesProcessed(totalBytes)
+ }
+
const processingTime = Date.now() - startTime
+ // v4.5.0: Report completion
+ if (progressHooks?.onCurrentItem) {
+ progressHooks.onCurrentItem(
+ `Excel complete: ${sheetsToProcess.length} sheets, ${convertedData.length} rows`
+ )
+ }
+
return {
format: this.format,
data: convertedData,
diff --git a/src/augmentations/intelligentImport/handlers/pdfHandler.ts b/src/augmentations/intelligentImport/handlers/pdfHandler.ts
index ed2863a8..f8bc9838 100644
--- a/src/augmentations/intelligentImport/handlers/pdfHandler.ts
+++ b/src/augmentations/intelligentImport/handlers/pdfHandler.ts
@@ -46,9 +46,19 @@ export class PDFHandler extends BaseFormatHandler {
async process(data: Buffer | string, options: FormatHandlerOptions): Promise {
const startTime = Date.now()
+ const progressHooks = options.progressHooks
// Convert to buffer
const buffer = Buffer.isBuffer(data) ? data : Buffer.from(data, 'binary')
+ const totalBytes = buffer.length
+
+ // v4.5.0: Report start
+ if (progressHooks?.onBytesProcessed) {
+ progressHooks.onBytesProcessed(0)
+ }
+ if (progressHooks?.onCurrentItem) {
+ progressHooks.onCurrentItem('Loading PDF document...')
+ }
try {
// Load PDF document
@@ -64,12 +74,22 @@ export class PDFHandler extends BaseFormatHandler {
const metadata = await pdfDoc.getMetadata()
const numPages = pdfDoc.numPages
+ // v4.5.0: Report document loaded
+ if (progressHooks?.onCurrentItem) {
+ progressHooks.onCurrentItem(`Processing ${numPages} pages...`)
+ }
+
// Extract text and structure from all pages
const allData: Array> = []
let totalTextLength = 0
let detectedTables = 0
for (let pageNum = 1; pageNum <= numPages; pageNum++) {
+ // v4.5.0: Report current page
+ if (progressHooks?.onCurrentItem) {
+ progressHooks.onCurrentItem(`Processing page ${pageNum} of ${numPages}`)
+ }
+
const page = await pdfDoc.getPage(pageNum)
const textContent = await page.getTextContent()
@@ -110,10 +130,36 @@ export class PDFHandler extends BaseFormatHandler {
})
}
}
+
+ // v4.5.0: Estimate bytes processed (pages are sequential)
+ const bytesProcessed = Math.floor((pageNum / numPages) * totalBytes)
+ if (progressHooks?.onBytesProcessed) {
+ progressHooks.onBytesProcessed(bytesProcessed)
+ }
+
+ // v4.5.0: Report extraction progress
+ if (progressHooks?.onDataExtracted) {
+ progressHooks.onDataExtracted(allData.length, undefined) // Total unknown until complete
+ }
+ }
+
+ // v4.5.0: Final progress - all bytes processed
+ if (progressHooks?.onBytesProcessed) {
+ progressHooks.onBytesProcessed(totalBytes)
+ }
+ if (progressHooks?.onDataExtracted) {
+ progressHooks.onDataExtracted(allData.length, allData.length)
}
const processingTime = Date.now() - startTime
+ // v4.5.0: Report completion
+ if (progressHooks?.onCurrentItem) {
+ progressHooks.onCurrentItem(
+ `PDF complete: ${numPages} pages, ${allData.length} items extracted`
+ )
+ }
+
// Get all unique fields (excluding metadata fields)
const fields = allData.length > 0
? Object.keys(allData[0]).filter(k => !k.startsWith('_'))
diff --git a/src/augmentations/intelligentImport/types.ts b/src/augmentations/intelligentImport/types.ts
index 054174b3..2a08ec24 100644
--- a/src/augmentations/intelligentImport/types.ts
+++ b/src/augmentations/intelligentImport/types.ts
@@ -3,6 +3,34 @@
* Handles Excel, PDF, and CSV import with intelligent extraction
*/
+import { ImportProgressTracker } from '../../utils/import-progress-tracker.js'
+
+/**
+ * Progress hooks for format handlers
+ *
+ * Handlers call these hooks to report progress during processing.
+ * This enables real-time progress tracking for any file format.
+ */
+export interface FormatHandlerProgressHooks {
+ /**
+ * Report bytes processed
+ * Call this as you read/parse the file
+ */
+ onBytesProcessed?: (bytes: number) => void
+
+ /**
+ * Set current processing context
+ * Examples: "Processing page 5", "Reading sheet: Q2 Sales"
+ */
+ onCurrentItem?: (item: string) => void
+
+ /**
+ * Report structured data extraction progress
+ * Examples: "Extracted 100 rows", "Parsed 50 paragraphs"
+ */
+ onDataExtracted?: (count: number, total?: number) => void
+}
+
export interface FormatHandler {
/**
* Format name (e.g., 'csv', 'xlsx', 'pdf')
@@ -58,6 +86,18 @@ export interface FormatHandlerOptions {
/** Whether to stream large files */
streaming?: boolean
+
+ /**
+ * Progress hooks (v4.5.0)
+ * Handlers call these to report progress during processing
+ */
+ progressHooks?: FormatHandlerProgressHooks
+
+ /**
+ * Total file size in bytes (v4.5.0)
+ * Used for progress percentage calculation
+ */
+ totalBytes?: number
}
export interface ProcessedData {
diff --git a/src/brainy.ts b/src/brainy.ts
index aa101335..ef2de0ae 100644
--- a/src/brainy.ts
+++ b/src/brainy.ts
@@ -2039,9 +2039,27 @@ export class Brainy implements BrainyInterface {
* groupBy: 'type', // Organize by entity type
* preserveSource: true, // Keep original file
*
- * // Progress tracking
- * onProgress: (p) => console.log(p.message)
+ * // Progress tracking (v4.5.0 - STANDARDIZED FOR ALL 7 FORMATS!)
+ * onProgress: (p) => {
+ * console.log(`[${p.stage}] ${p.message}`)
+ * console.log(`Entities: ${p.entities || 0}, Rels: ${p.relationships || 0}`)
+ * if (p.throughput) console.log(`Rate: ${p.throughput.toFixed(1)}/sec`)
+ * }
* })
+ * // THIS SAME HANDLER WORKS FOR CSV, PDF, Excel, JSON, Markdown, YAML, DOCX!
+ * ```
+ *
+ * @example Universal Progress Handler (v4.5.0)
+ * ```typescript
+ * // ONE handler for ALL 7 formats - no format-specific code needed!
+ * const universalProgress = (p) => {
+ * updateUI(p.stage, p.message, p.entities, p.relationships)
+ * }
+ *
+ * await brain.import(csvBuffer, { onProgress: universalProgress })
+ * await brain.import(pdfBuffer, { onProgress: universalProgress })
+ * await brain.import(excelBuffer, { onProgress: universalProgress })
+ * // Works for JSON, Markdown, YAML, DOCX too!
* ```
*
* @example Performance Tuning (Large Files)
@@ -2066,6 +2084,7 @@ export class Brainy implements BrainyInterface {
*
* @see {@link https://brainy.dev/docs/api/import API Documentation}
* @see {@link https://brainy.dev/docs/guides/migrating-to-v4 Migration Guide}
+ * @see {@link https://brainy.dev/docs/guides/standard-import-progress Standard Progress API (v4.5.0)}
*
* @remarks
* **โ ๏ธ Breaking Changes from v3.x:**
@@ -2098,7 +2117,7 @@ export class Brainy implements BrainyInterface {
async import(
source: Buffer | string | object,
options?: {
- format?: 'excel' | 'pdf' | 'csv' | 'json' | 'markdown'
+ format?: 'excel' | 'pdf' | 'csv' | 'json' | 'markdown' | 'yaml' | 'docx'
vfsPath?: string
groupBy?: 'type' | 'sheet' | 'flat' | 'custom'
customGrouping?: (entity: any) => string
diff --git a/src/cli/commands/core.ts b/src/cli/commands/core.ts
index 66c6f4c8..40ebb119 100644
--- a/src/cli/commands/core.ts
+++ b/src/cli/commands/core.ts
@@ -21,6 +21,8 @@ interface AddOptions extends CoreOptions {
id?: string
metadata?: string
type?: string
+ confidence?: string
+ weight?: string
}
interface SearchOptions extends CoreOptions {
@@ -35,6 +37,7 @@ interface SearchOptions extends CoreOptions {
via?: string
explain?: boolean
includeRelations?: boolean
+ includeVfs?: boolean
fusion?: string
vectorWeight?: string
graphWeight?: string
@@ -163,24 +166,40 @@ export const coreCommands = {
}
// Add with explicit type
- const result = await brain.add({
+ const addParams: any = {
data: text,
type: nounType,
metadata
- })
-
+ }
+
+ // v4.3.x: Add confidence and weight if provided
+ if (options.confidence) {
+ addParams.confidence = parseFloat(options.confidence)
+ }
+ if (options.weight) {
+ addParams.weight = parseFloat(options.weight)
+ }
+
+ const result = await brain.add(addParams)
+
spinner.succeed('Added successfully')
-
+
if (!options.json) {
console.log(chalk.green(`โ Added with ID: ${result}`))
if (options.type) {
console.log(chalk.dim(` Type: ${options.type}`))
}
+ if (options.confidence) {
+ console.log(chalk.dim(` Confidence: ${options.confidence}`))
+ }
+ if (options.weight) {
+ console.log(chalk.dim(` Weight: ${options.weight}`))
+ }
if (Object.keys(metadata).length > 0) {
console.log(chalk.dim(` Metadata: ${JSON.stringify(metadata)}`))
}
} else {
- formatOutput({ id: result, metadata }, options)
+ formatOutput({ id: result, metadata, confidence: addParams.confidence, weight: addParams.weight }, options)
}
} catch (error: any) {
if (spinner) spinner.fail('Failed to add data')
@@ -328,6 +347,11 @@ export const coreCommands = {
searchParams.includeRelations = true
}
+ // Include VFS files (v4.4.0 - find excludes VFS by default)
+ if (options.includeVfs) {
+ searchParams.includeVFS = true
+ }
+
// Triple Intelligence Fusion - custom weighting
if (options.fusion || options.vectorWeight || options.graphWeight || options.fieldWeight) {
searchParams.fusion = {
diff --git a/src/cli/commands/import.ts b/src/cli/commands/import.ts
index 7240725a..7d257025 100644
--- a/src/cli/commands/import.ts
+++ b/src/cli/commands/import.ts
@@ -149,23 +149,28 @@ export const importCommands = {
options.recursive = answer.recursive
}
- spinner = ora('Initializing neural import...').start()
+ spinner = ora('Initializing import...').start()
const brain = getBrainy()
- // Load UniversalImportAPI
- const { UniversalImportAPI } = await import('../../api/UniversalImportAPI.js')
- const universalImport = new UniversalImportAPI(brain)
- await universalImport.init()
-
- spinner.text = 'Processing import...'
-
// Handle different source types
let result: any
if (isURL) {
- // URL import
+ // URL import - fetch first
spinner.text = `Fetching from ${source}...`
- result = await universalImport.importFromURL(source)
+ const response = await fetch(source)
+ const buffer = Buffer.from(await response.arrayBuffer())
+
+ spinner.text = 'Importing...'
+ result = await brain.import(buffer, {
+ enableNeuralExtraction: options.extractEntities !== false,
+ enableRelationshipInference: options.detectRelationships !== false,
+ enableConceptExtraction: options.extractConcepts || false,
+ confidenceThreshold: options.confidence ? parseFloat(options.confidence) : 0.6,
+ onProgress: options.progress ? (p) => {
+ spinner.text = `${p.message}${p.entities ? ` (${p.entities} entities)` : ''}`
+ } : undefined
+ })
} else if (isDirectory) {
// Directory import - process each file
spinner.text = `Scanning directory: ${source}...`
@@ -202,34 +207,45 @@ export const importCommands = {
spinner.succeed(`Found ${files.length} files`)
- // Process files in batches
- const batchSize = options.batchSize ? parseInt(options.batchSize) : 100
+ // Process files with progress
let totalEntities = 0
let totalRelationships = 0
let filesProcessed = 0
- for (let i = 0; i < files.length; i += batchSize) {
- const batch = files.slice(i, i + batchSize)
+ for (const file of files) {
+ try {
+ if (options.progress) {
+ spinner = ora(`[${filesProcessed + 1}/${files.length}] Importing ${file}...`).start()
+ }
- if (options.progress) {
- spinner = ora(`Processing batch ${Math.floor(i / batchSize) + 1}/${Math.ceil(files.length / batchSize)} (${filesProcessed}/${files.length} files)...`).start()
- }
+ const fileResult = await brain.import(file, {
+ enableNeuralExtraction: options.extractEntities !== false,
+ enableRelationshipInference: options.detectRelationships !== false,
+ enableConceptExtraction: options.extractConcepts || false,
+ confidenceThreshold: options.confidence ? parseFloat(options.confidence) : 0.6,
+ onProgress: options.progress ? (p) => {
+ spinner.text = `[${filesProcessed + 1}/${files.length}] ${p.message}`
+ } : undefined
+ })
- for (const file of batch) {
- try {
- const fileResult = await universalImport.importFromFile(file)
- totalEntities += fileResult.stats.entitiesCreated
- totalRelationships += fileResult.stats.relationshipsCreated
- filesProcessed++
- } catch (error: any) {
- if (options.verbose) {
- console.log(chalk.yellow(`โ ๏ธ Failed to import ${file}: ${error.message}`))
- }
+ totalEntities += fileResult.entities.length
+ totalRelationships += fileResult.relationships.length
+ filesProcessed++
+
+ if (options.progress) {
+ spinner.succeed(`[${filesProcessed}/${files.length}] ${file}`)
+ }
+ } catch (error: any) {
+ if (options.verbose) {
+ if (spinner) spinner.fail(`Failed: ${file}`)
+ console.log(chalk.yellow(`โ ๏ธ ${error.message}`))
}
}
}
result = {
+ entities: [],
+ relationships: [],
stats: {
filesProcessed,
entitiesCreated: totalEntities,
@@ -238,10 +254,22 @@ export const importCommands = {
}
}
- spinner.succeed('Directory import complete')
+ spinner = ora().succeed(`Directory import complete: ${filesProcessed} files`)
} else {
- // File import
- result = await universalImport.importFromFile(source)
+ // File import with progress
+ result = await brain.import(source, {
+ format: options.format as any,
+ enableNeuralExtraction: options.extractEntities !== false,
+ enableRelationshipInference: options.detectRelationships !== false,
+ enableConceptExtraction: options.extractConcepts || false,
+ confidenceThreshold: options.confidence ? parseFloat(options.confidence) : 0.6,
+ onProgress: options.progress ? (p) => {
+ spinner.text = `${p.message}${p.entities ? ` (${p.entities} entities, ${p.relationships || 0} relationships)` : ''}`
+ if (p.throughput && p.eta) {
+ spinner.text += ` - ${p.throughput.toFixed(1)}/sec, ETA: ${Math.round(p.eta / 1000)}s`
+ }
+ } : undefined
+ })
}
spinner.succeed('Import complete')
@@ -323,18 +351,26 @@ export const importCommands = {
console.log(chalk.cyan('\n๐ Import Results:\n'))
console.log(chalk.bold('Statistics:'))
- console.log(` Entities created: ${chalk.green(result.stats.entitiesCreated)}`)
+ const entitiesCount = result.stats?.entitiesCreated || result.entities?.length || 0
+ const relationshipsCount = result.stats?.relationshipsCreated || result.relationships?.length || 0
- if (result.stats.relationshipsCreated > 0) {
- console.log(` Relationships created: ${chalk.green(result.stats.relationshipsCreated)}`)
+ console.log(` Entities created: ${chalk.green(entitiesCount)}`)
+
+ if (relationshipsCount > 0) {
+ console.log(` Relationships created: ${chalk.green(relationshipsCount)}`)
}
- if (result.stats.filesProcessed) {
+ if (result.stats?.filesProcessed) {
console.log(` Files processed: ${chalk.green(result.stats.filesProcessed)}`)
}
- console.log(` Average confidence: ${chalk.yellow((result.stats.averageConfidence * 100).toFixed(1))}%`)
- console.log(` Processing time: ${chalk.dim(result.stats.processingTimeMs)}ms`)
+ if (result.stats?.averageConfidence) {
+ console.log(` Average confidence: ${chalk.yellow((result.stats.averageConfidence * 100).toFixed(1))}%`)
+ }
+
+ if (result.stats?.processingTimeMs) {
+ console.log(` Processing time: ${chalk.dim(result.stats.processingTimeMs)}ms`)
+ }
if (options.verbose && result.entities && result.entities.length > 0) {
console.log(chalk.bold('\n๐ฆ Imported Entities (first 10):'))
diff --git a/src/importers/SmartCSVImporter.ts b/src/importers/SmartCSVImporter.ts
index 6adf9e16..576bb979 100644
--- a/src/importers/SmartCSVImporter.ts
+++ b/src/importers/SmartCSVImporter.ts
@@ -169,10 +169,44 @@ export class SmartCSVImporter {
}
// Parse CSV using existing handler
+ // v4.5.0: Pass progress hooks to handler for file parsing progress
const processedData = await this.csvHandler.process(buffer, {
...options,
csvDelimiter: opts.csvDelimiter,
- csvHeaders: opts.csvHeaders
+ csvHeaders: opts.csvHeaders,
+ totalBytes: buffer.length,
+ progressHooks: {
+ onBytesProcessed: (bytes) => {
+ // Handler reports bytes processed during parsing
+ opts.onProgress?.({
+ processed: 0,
+ total: 0,
+ entities: 0,
+ relationships: 0,
+ phase: `Parsing CSV (${Math.round((bytes / buffer.length) * 100)}%)`
+ })
+ },
+ onCurrentItem: (message) => {
+ // Handler reports current processing step
+ opts.onProgress?.({
+ processed: 0,
+ total: 0,
+ entities: 0,
+ relationships: 0,
+ phase: message
+ })
+ },
+ onDataExtracted: (count, total) => {
+ // Handler reports rows extracted
+ opts.onProgress?.({
+ processed: 0,
+ total: total || count,
+ entities: 0,
+ relationships: 0,
+ phase: `Extracted ${count} rows`
+ })
+ }
+ }
})
const rows = processedData.data
diff --git a/src/importers/SmartDOCXImporter.ts b/src/importers/SmartDOCXImporter.ts
index 0999093b..02a017c9 100644
--- a/src/importers/SmartDOCXImporter.ts
+++ b/src/importers/SmartDOCXImporter.ts
@@ -187,12 +187,26 @@ export class SmartDOCXImporter {
await this.init()
}
+ // v4.5.0: Report parsing start
+ options.onProgress?.({
+ processed: 0,
+ entities: 0,
+ relationships: 0
+ })
+
// Extract raw text for entity extraction
const textResult = await mammoth.extractRawText({ buffer })
// Extract HTML for structure analysis (headings, tables)
const htmlResult = await mammoth.convertToHtml({ buffer })
+ // v4.5.0: Report parsing complete
+ options.onProgress?.({
+ processed: 0,
+ entities: 0,
+ relationships: 0
+ })
+
// Process the document
const result = await this.extractFromContent(
textResult.value,
diff --git a/src/importers/SmartExcelImporter.ts b/src/importers/SmartExcelImporter.ts
index b996772f..08f5591d 100644
--- a/src/importers/SmartExcelImporter.ts
+++ b/src/importers/SmartExcelImporter.ts
@@ -184,7 +184,43 @@ export class SmartExcelImporter {
}
// Parse Excel using existing handler
- const processedData = await this.excelHandler.process(buffer, options)
+ // v4.5.0: Pass progress hooks to handler for file parsing progress
+ const processedData = await this.excelHandler.process(buffer, {
+ ...options,
+ totalBytes: buffer.length,
+ progressHooks: {
+ onBytesProcessed: (bytes) => {
+ // Handler reports bytes processed during parsing
+ opts.onProgress?.({
+ processed: 0,
+ total: 0,
+ entities: 0,
+ relationships: 0,
+ phase: `Parsing Excel (${Math.round((bytes / buffer.length) * 100)}%)`
+ })
+ },
+ onCurrentItem: (message) => {
+ // Handler reports current processing step (e.g., "Reading sheet: Sales (1/3)")
+ opts.onProgress?.({
+ processed: 0,
+ total: 0,
+ entities: 0,
+ relationships: 0,
+ phase: message
+ })
+ },
+ onDataExtracted: (count, total) => {
+ // Handler reports rows extracted
+ opts.onProgress?.({
+ processed: 0,
+ total: total || count,
+ entities: 0,
+ relationships: 0,
+ phase: `Extracted ${count} rows from Excel`
+ })
+ }
+ }
+ })
const rows = processedData.data
if (rows.length === 0) {
diff --git a/src/importers/SmartJSONImporter.ts b/src/importers/SmartJSONImporter.ts
index bad98c6b..fd5a79e8 100644
--- a/src/importers/SmartJSONImporter.ts
+++ b/src/importers/SmartJSONImporter.ts
@@ -167,6 +167,13 @@ export class SmartJSONImporter {
...options
}
+ // v4.5.0: Report parsing start
+ opts.onProgress({
+ processed: 0,
+ entities: 0,
+ relationships: 0
+ })
+
// Parse JSON if string
let jsonData: any
if (typeof data === 'string') {
@@ -179,6 +186,13 @@ export class SmartJSONImporter {
jsonData = data
}
+ // v4.5.0: Report parsing complete, starting traversal
+ opts.onProgress({
+ processed: 0,
+ entities: 0,
+ relationships: 0
+ })
+
// Traverse and extract
const entities: ExtractedJSONEntity[] = []
const relationships: ExtractedJSONRelationship[] = []
@@ -214,6 +228,13 @@ export class SmartJSONImporter {
}
)
+ // v4.5.0: Report completion
+ opts.onProgress({
+ processed: nodesProcessed,
+ entities: entities.length,
+ relationships: relationships.length
+ })
+
return {
nodesProcessed,
entitiesExtracted: entities.length,
diff --git a/src/importers/SmartMarkdownImporter.ts b/src/importers/SmartMarkdownImporter.ts
index 6a85213f..44471f65 100644
--- a/src/importers/SmartMarkdownImporter.ts
+++ b/src/importers/SmartMarkdownImporter.ts
@@ -174,9 +174,25 @@ export class SmartMarkdownImporter {
...options
}
+ // v4.5.0: Report parsing start
+ opts.onProgress({
+ processed: 0,
+ total: 0,
+ entities: 0,
+ relationships: 0
+ })
+
// Parse markdown into sections
const parsedSections = this.parseMarkdown(markdown, opts)
+ // v4.5.0: Report parsing complete
+ opts.onProgress({
+ processed: 0,
+ total: parsedSections.length,
+ entities: 0,
+ relationships: 0
+ })
+
// Process each section
const sections: MarkdownSection[] = []
const entityMap = new Map()
@@ -202,10 +218,20 @@ export class SmartMarkdownImporter {
})
}
+ // v4.5.0: Report completion
+ const totalEntities = sections.reduce((sum, s) => sum + s.entities.length, 0)
+ const totalRelationships = sections.reduce((sum, s) => sum + s.relationships.length, 0)
+ opts.onProgress({
+ processed: sections.length,
+ total: sections.length,
+ entities: totalEntities,
+ relationships: totalRelationships
+ })
+
return {
sectionsProcessed: sections.length,
- entitiesExtracted: sections.reduce((sum, s) => sum + s.entities.length, 0),
- relationshipsInferred: sections.reduce((sum, s) => sum + s.relationships.length, 0),
+ entitiesExtracted: totalEntities,
+ relationshipsInferred: totalRelationships,
sections,
entityMap,
processingTime: Date.now() - startTime,
diff --git a/src/importers/SmartPDFImporter.ts b/src/importers/SmartPDFImporter.ts
index 7ddc3a7e..18dc716c 100644
--- a/src/importers/SmartPDFImporter.ts
+++ b/src/importers/SmartPDFImporter.ts
@@ -181,7 +181,43 @@ export class SmartPDFImporter {
}
// Parse PDF using existing handler
- const processedData = await this.pdfHandler.process(buffer, options)
+ // v4.5.0: Pass progress hooks to handler for file parsing progress
+ const processedData = await this.pdfHandler.process(buffer, {
+ ...options,
+ totalBytes: buffer.length,
+ progressHooks: {
+ onBytesProcessed: (bytes) => {
+ // Handler reports bytes processed during parsing
+ opts.onProgress?.({
+ processed: 0,
+ total: 0,
+ entities: 0,
+ relationships: 0,
+ phase: `Parsing PDF (${Math.round((bytes / buffer.length) * 100)}%)`
+ })
+ },
+ onCurrentItem: (message) => {
+ // Handler reports current processing step (e.g., "Processing page 5 of 23")
+ opts.onProgress?.({
+ processed: 0,
+ total: 0,
+ entities: 0,
+ relationships: 0,
+ phase: message
+ })
+ },
+ onDataExtracted: (count, total) => {
+ // Handler reports items extracted (paragraphs + tables)
+ opts.onProgress?.({
+ processed: 0,
+ total: total || count,
+ entities: 0,
+ relationships: 0,
+ phase: `Extracted ${count} items from PDF`
+ })
+ }
+ }
+ })
const data = processedData.data
const pdfMetadata = processedData.metadata.additionalInfo?.pdfMetadata || {}
diff --git a/src/importers/SmartYAMLImporter.ts b/src/importers/SmartYAMLImporter.ts
index 0811a39a..73d6fc7d 100644
--- a/src/importers/SmartYAMLImporter.ts
+++ b/src/importers/SmartYAMLImporter.ts
@@ -159,6 +159,13 @@ export class SmartYAMLImporter {
): Promise {
const startTime = Date.now()
+ // v4.5.0: Report parsing start
+ options.onProgress?.({
+ processed: 0,
+ entities: 0,
+ relationships: 0
+ })
+
// Parse YAML to JavaScript object
const yamlString = typeof yamlContent === 'string'
? yamlContent
@@ -171,6 +178,13 @@ export class SmartYAMLImporter {
throw new Error(`Failed to parse YAML: ${error.message}`)
}
+ // v4.5.0: Report parsing complete
+ options.onProgress?.({
+ processed: 0,
+ entities: 0,
+ relationships: 0
+ })
+
// Process as JSON-like structure
const result = await this.extractFromData(data, options)
result.processingTime = Date.now() - startTime
diff --git a/src/types/brainy.types.ts b/src/types/brainy.types.ts
index 1f54622a..8bea452f 100644
--- a/src/types/brainy.types.ts
+++ b/src/types/brainy.types.ts
@@ -385,6 +385,138 @@ export interface BatchResult {
duration: number // Time taken in ms
}
+// ============= Import Progress (v4.5.0) =============
+
+/**
+ * Import stage enumeration
+ */
+export type ImportStage =
+ | 'detecting' // Detecting file format
+ | 'reading' // Reading file from disk/network
+ | 'parsing' // Parsing file structure (CSV rows, PDF pages, Excel sheets)
+ | 'extracting' // Extracting entities using AI
+ | 'indexing' // Creating graph nodes and relationships
+ | 'completing' // Final cleanup and stats
+
+/**
+ * Overall import status
+ */
+export type ImportStatus =
+ | 'starting' // Initializing import
+ | 'processing' // Actively importing
+ | 'completing' // Finalizing
+ | 'done' // Complete
+
+/**
+ * Comprehensive import progress information
+ *
+ * Provides multi-dimensional progress tracking:
+ * - Bytes processed (always deterministic)
+ * - Entities extracted and indexed
+ * - Stage-specific progress
+ * - Time estimates
+ * - Performance metrics
+ *
+ * @since v4.5.0
+ */
+export interface ImportProgress {
+ // Overall Progress
+ overall_progress: number // 0-100 weighted estimate across all stages
+ overall_status: ImportStatus // High-level status
+
+ // Current Stage
+ stage: ImportStage // What's happening now
+ stage_progress: number // 0-100 within current stage (0 if unknown)
+ stage_message: string // Human-readable: "Extracting entities from PDF..."
+
+ // Bytes (Always Available - most deterministic metric)
+ bytes_processed: number // Bytes read/processed so far
+ total_bytes: number // Total file size (0 if streaming/unknown)
+ bytes_percentage: number // Convenience: bytes_processed / total_bytes * 100
+ bytes_per_second?: number // Processing rate (relevant during parsing)
+
+ // Entities (Available when extraction starts)
+ entities_extracted: number // Entities found during AI extraction
+ entities_indexed: number // Entities added to Brainy graph
+ entities_per_second?: number // Entities/sec (relevant during extraction/indexing)
+ estimated_total_entities?: number // Estimated final count
+ estimation_confidence?: number // 0-1 confidence in estimation
+
+ // Timing
+ elapsed_ms: number // Time since import started
+ estimated_remaining_ms?: number // Estimated time remaining
+ estimated_total_ms?: number // Estimated total time
+
+ // Context (helps users understand what's happening)
+ current_item?: string // "Processing page 5 of 23"
+ current_file?: string // "Sheet: Q2 Sales Data"
+ file_number?: number // 3 (when importing multiple files)
+ total_files?: number // 10
+
+ // Performance Metrics (for debugging/optimization)
+ metrics?: {
+ parsing_rate_mbps?: number // MB/s during parsing
+ extraction_rate_entities_per_sec?: number // Entities/s during extraction
+ indexing_rate_entities_per_sec?: number // Entities/s during indexing
+ memory_usage_mb?: number // Current memory usage
+ peak_memory_mb?: number // Peak memory usage
+ }
+
+ // Backwards Compatibility (for legacy code)
+ current: number // Alias for entities_indexed
+ total: number // Alias for estimated_total_entities or 0
+}
+
+/**
+ * Import progress callback - backwards compatible
+ *
+ * Supports both legacy (current, total) and new (ImportProgress object) signatures
+ */
+export type ImportProgressCallback =
+ | ((progress: ImportProgress) => void)
+ | ((current: number, total: number) => void)
+
+/**
+ * Stage weight configuration for overall progress calculation
+ *
+ * These weights reflect the typical time distribution across stages.
+ * Extraction is typically the slowest stage (60% of time).
+ */
+export interface StageWeights {
+ detecting: number // Default: 0.01 (1%)
+ reading: number // Default: 0.05 (5%)
+ parsing: number // Default: 0.10 (10%)
+ extracting: number // Default: 0.60 (60% - slowest!)
+ indexing: number // Default: 0.20 (20%)
+ completing: number // Default: 0.04 (4%)
+}
+
+/**
+ * Import result statistics
+ */
+export interface ImportStats {
+ graphNodesCreated: number // Entities added to graph
+ graphEdgesCreated: number // Relationships created
+ vfsFilesCreated: number // VFS files created
+ duration: number // Total time in ms
+ bytesProcessed: number // Total bytes read
+ averageRate: number // Average entities/sec
+ peakMemoryMB?: number // Peak memory usage
+}
+
+/**
+ * Import operation result
+ */
+export interface ImportResult {
+ success: boolean
+ stats: ImportStats
+ errors?: Array<{
+ stage: ImportStage
+ message: string
+ error?: any
+ }>
+}
+
// ============= Advanced Operations =============
/**
diff --git a/src/utils/import-progress-tracker.ts b/src/utils/import-progress-tracker.ts
new file mode 100644
index 00000000..7090eee1
--- /dev/null
+++ b/src/utils/import-progress-tracker.ts
@@ -0,0 +1,541 @@
+/**
+ * Import Progress Tracker (v4.5.0)
+ *
+ * Comprehensive progress tracking for imports with:
+ * - Multi-dimensional progress (bytes, entities, stages, timing)
+ * - Smart estimation (entity count, time remaining)
+ * - Stage-specific metrics (bytes/sec vs entities/sec)
+ * - Throttled callbacks (avoid spam)
+ * - Weighted overall progress
+ *
+ * @since v4.5.0
+ */
+
+import {
+ ImportProgress,
+ ImportStage,
+ ImportStatus,
+ StageWeights,
+ ImportProgressCallback
+} from '../types/brainy.types.js'
+
+/**
+ * Default stage weights (reflect typical time distribution)
+ */
+const DEFAULT_STAGE_WEIGHTS: StageWeights = {
+ detecting: 0.01, // 1% - very fast
+ reading: 0.05, // 5% - reading file
+ parsing: 0.10, // 10% - parsing structure
+ extracting: 0.60, // 60% - AI extraction (slowest!)
+ indexing: 0.20, // 20% - creating graph
+ completing: 0.04 // 4% - cleanup
+}
+
+/**
+ * Stage ordering for progress calculation
+ */
+const STAGE_ORDER: ImportStage[] = [
+ 'detecting',
+ 'reading',
+ 'parsing',
+ 'extracting',
+ 'indexing',
+ 'completing'
+]
+
+/**
+ * Progress tracker for imports
+ */
+export class ImportProgressTracker {
+ // Configuration
+ private readonly stageWeights: StageWeights
+ private readonly throttleMs: number
+ private readonly callback?: ImportProgressCallback
+
+ // Tracking state
+ private startTime: number
+ private lastEmitTime: number = 0
+ private currentStage: ImportStage = 'detecting'
+ private completedStages: Set = new Set()
+
+ // Metrics
+ private totalBytes: number = 0
+ private bytesProcessed: number = 0
+ private entitiesExtracted: number = 0
+ private entitiesIndexed: number = 0
+ private parseStartTime?: number
+ private extractStartTime?: number
+ private indexStartTime?: number
+
+ // Estimation
+ private lastBytesCheckpoint: number = 0
+ private lastBytesCheckpointTime: number = 0
+ private lastEntitiesCheckpoint: number = 0
+ private lastEntitiesCheckpointTime: number = 0
+
+ // Context
+ private currentItem?: string
+ private currentFile?: string
+ private fileNumber?: number
+ private totalFiles?: number
+
+ // Memory tracking
+ private peakMemoryMB: number = 0
+
+ constructor(options: {
+ totalBytes?: number
+ stageWeights?: Partial
+ throttleMs?: number
+ callback?: ImportProgressCallback
+ } = {}) {
+ this.stageWeights = { ...DEFAULT_STAGE_WEIGHTS, ...options.stageWeights }
+ this.throttleMs = options.throttleMs ?? 100 // 100ms default
+ this.callback = options.callback
+ this.totalBytes = options.totalBytes ?? 0
+ this.startTime = Date.now()
+ this.lastBytesCheckpointTime = this.startTime
+ this.lastEntitiesCheckpointTime = this.startTime
+ }
+
+ /**
+ * Set total file size (if known later)
+ */
+ setTotalBytes(bytes: number): void {
+ this.totalBytes = bytes
+ }
+
+ /**
+ * Update current stage
+ */
+ setStage(stage: ImportStage, message?: string): void {
+ // Mark previous stage as complete
+ if (this.currentStage !== stage) {
+ this.completedStages.add(this.currentStage)
+ }
+
+ this.currentStage = stage
+ if (message) {
+ this.setStageMessage(message)
+ }
+
+ // Track stage start times
+ const now = Date.now()
+ switch (stage) {
+ case 'parsing':
+ this.parseStartTime = now
+ break
+ case 'extracting':
+ this.extractStartTime = now
+ break
+ case 'indexing':
+ this.indexStartTime = now
+ break
+ }
+
+ // Force emit on stage change
+ this.emit(true)
+ }
+
+ /**
+ * Update bytes processed
+ */
+ updateBytes(bytes: number): void {
+ this.bytesProcessed = bytes
+ this.emit()
+ }
+
+ /**
+ * Increment bytes processed
+ */
+ addBytes(bytes: number): void {
+ this.bytesProcessed += bytes
+ this.emit()
+ }
+
+ /**
+ * Update entities extracted
+ */
+ updateEntitiesExtracted(count: number): void {
+ this.entitiesExtracted = count
+ this.emit()
+ }
+
+ /**
+ * Increment entities extracted
+ */
+ addEntitiesExtracted(count: number): void {
+ this.entitiesExtracted += count
+ this.emit()
+ }
+
+ /**
+ * Update entities indexed
+ */
+ updateEntitiesIndexed(count: number): void {
+ this.entitiesIndexed = count
+ this.emit()
+ }
+
+ /**
+ * Increment entities indexed
+ */
+ addEntitiesIndexed(count: number): void {
+ this.entitiesIndexed += count
+ this.emit()
+ }
+
+ /**
+ * Set context information
+ */
+ setContext(context: {
+ currentItem?: string
+ currentFile?: string
+ fileNumber?: number
+ totalFiles?: number
+ }): void {
+ if (context.currentItem !== undefined) this.currentItem = context.currentItem
+ if (context.currentFile !== undefined) this.currentFile = context.currentFile
+ if (context.fileNumber !== undefined) this.fileNumber = context.fileNumber
+ if (context.totalFiles !== undefined) this.totalFiles = context.totalFiles
+ this.emit()
+ }
+
+ /**
+ * Set stage message
+ */
+ private setStageMessage(message: string): void {
+ this.currentItem = message
+ }
+
+ /**
+ * Calculate stage progress (0-100 within current stage)
+ */
+ private calculateStageProgress(): number {
+ switch (this.currentStage) {
+ case 'detecting':
+ case 'completing':
+ // These are quick, assume 100% once started
+ return 100
+
+ case 'reading':
+ case 'parsing':
+ // Use bytes as proxy for progress
+ if (this.totalBytes === 0) return 0
+ return Math.min(100, (this.bytesProcessed / this.totalBytes) * 100)
+
+ case 'extracting':
+ // Extraction progress is hard to estimate (AI is unpredictable)
+ // We can't reliably say % complete, so return 0
+ return 0
+
+ case 'indexing':
+ // If we have estimated total entities, use that
+ if (this.entitiesExtracted > 0) {
+ return Math.min(100, (this.entitiesIndexed / this.entitiesExtracted) * 100)
+ }
+ return 0
+
+ default:
+ return 0
+ }
+ }
+
+ /**
+ * Calculate overall progress (0-100 weighted across all stages)
+ */
+ private calculateOverallProgress(): number {
+ // Calculate progress of completed stages
+ let completedWeight = 0
+ for (const stage of this.completedStages) {
+ completedWeight += this.stageWeights[stage]
+ }
+
+ // Calculate progress of current stage
+ const stageProgress = this.calculateStageProgress()
+ const currentStageContribution = this.stageWeights[this.currentStage] * (stageProgress / 100)
+
+ // Overall = completed stages + current stage contribution
+ const overall = (completedWeight + currentStageContribution) * 100
+
+ return Math.min(100, Math.max(0, overall))
+ }
+
+ /**
+ * Calculate bytes per second
+ */
+ private calculateBytesPerSecond(): number | undefined {
+ const now = Date.now()
+ const elapsed = now - this.lastBytesCheckpointTime
+
+ // Need at least 1 second of data
+ if (elapsed < 1000) return undefined
+
+ const bytesDelta = this.bytesProcessed - this.lastBytesCheckpoint
+ const bytesPerSec = (bytesDelta / elapsed) * 1000
+
+ // Update checkpoint
+ this.lastBytesCheckpoint = this.bytesProcessed
+ this.lastBytesCheckpointTime = now
+
+ return bytesPerSec > 0 ? bytesPerSec : undefined
+ }
+
+ /**
+ * Calculate entities per second
+ */
+ private calculateEntitiesPerSecond(): number | undefined {
+ const now = Date.now()
+ const elapsed = now - this.lastEntitiesCheckpointTime
+
+ // Need at least 1 second of data
+ if (elapsed < 1000) return undefined
+
+ // Use appropriate counter based on stage
+ const currentCount = this.currentStage === 'indexing'
+ ? this.entitiesIndexed
+ : this.entitiesExtracted
+
+ const entitiesDelta = currentCount - this.lastEntitiesCheckpoint
+ const entitiesPerSec = (entitiesDelta / elapsed) * 1000
+
+ // Update checkpoint
+ this.lastEntitiesCheckpoint = currentCount
+ this.lastEntitiesCheckpointTime = now
+
+ return entitiesPerSec > 0 ? entitiesPerSec : undefined
+ }
+
+ /**
+ * Estimate total entities
+ */
+ private estimateTotalEntities(): { count: number, confidence: number } | undefined {
+ // Only estimate if we've processed some bytes and extracted some entities
+ if (this.bytesProcessed === 0 || this.entitiesExtracted === 0 || this.totalBytes === 0) {
+ return undefined
+ }
+
+ // Estimate based on entities per byte
+ const bytesPercentage = this.bytesProcessed / this.totalBytes
+ const estimatedTotal = Math.ceil(this.entitiesExtracted / bytesPercentage)
+
+ // Confidence increases with more data
+ const confidence = Math.min(0.95, bytesPercentage)
+
+ return { count: estimatedTotal, confidence }
+ }
+
+ /**
+ * Estimate remaining time
+ */
+ private estimateRemainingTime(): number | undefined {
+ const now = Date.now()
+ const elapsed = now - this.startTime
+
+ // Need at least 5 seconds of data for reasonable estimate
+ if (elapsed < 5000) return undefined
+
+ const overallProgress = this.calculateOverallProgress()
+ if (overallProgress === 0) return undefined
+
+ // Estimate total time based on current progress
+ const estimatedTotalMs = (elapsed / overallProgress) * 100
+ const remainingMs = estimatedTotalMs - elapsed
+
+ return remainingMs > 0 ? remainingMs : undefined
+ }
+
+ /**
+ * Get current memory usage
+ */
+ private getCurrentMemoryMB(): number | undefined {
+ if (typeof process === 'undefined' || !process.memoryUsage) return undefined
+
+ const usage = process.memoryUsage()
+ const currentMB = usage.heapUsed / 1024 / 1024
+
+ // Track peak
+ this.peakMemoryMB = Math.max(this.peakMemoryMB, currentMB)
+
+ return currentMB
+ }
+
+ /**
+ * Build complete progress object
+ */
+ private buildProgress(): ImportProgress {
+ const now = Date.now()
+ const elapsed = now - this.startTime
+
+ const stageProgress = this.calculateStageProgress()
+ const overallProgress = this.calculateOverallProgress()
+ const bytesPerSec = this.calculateBytesPerSecond()
+ const entitiesPerSec = this.calculateEntitiesPerSecond()
+ const entityEstimate = this.estimateTotalEntities()
+ const remainingMs = this.estimateRemainingTime()
+ const currentMemoryMB = this.getCurrentMemoryMB()
+
+ // Determine overall status
+ let overallStatus: ImportStatus
+ if (overallProgress === 0) {
+ overallStatus = 'starting'
+ } else if (overallProgress === 100) {
+ overallStatus = 'done'
+ } else if (this.currentStage === 'completing') {
+ overallStatus = 'completing'
+ } else {
+ overallStatus = 'processing'
+ }
+
+ // Stage message
+ let stageMessage: string
+ if (this.currentItem) {
+ stageMessage = this.currentItem
+ } else {
+ // Default messages
+ switch (this.currentStage) {
+ case 'detecting':
+ stageMessage = 'Detecting file format...'
+ break
+ case 'reading':
+ stageMessage = 'Reading file...'
+ break
+ case 'parsing':
+ stageMessage = 'Parsing file structure...'
+ break
+ case 'extracting':
+ stageMessage = 'Extracting entities using AI...'
+ break
+ case 'indexing':
+ stageMessage = 'Creating graph nodes...'
+ break
+ case 'completing':
+ stageMessage = 'Finalizing import...'
+ break
+ default:
+ stageMessage = 'Processing...'
+ }
+ }
+
+ // Calculate bytes percentage
+ const bytesPercentage = this.totalBytes > 0
+ ? (this.bytesProcessed / this.totalBytes) * 100
+ : 0
+
+ // Build metrics object
+ const metrics: ImportProgress['metrics'] = {
+ parsing_rate_mbps: this.currentStage === 'parsing' && bytesPerSec
+ ? bytesPerSec / 1_000_000
+ : undefined,
+ extraction_rate_entities_per_sec: this.currentStage === 'extracting'
+ ? entitiesPerSec
+ : undefined,
+ indexing_rate_entities_per_sec: this.currentStage === 'indexing'
+ ? entitiesPerSec
+ : undefined,
+ memory_usage_mb: currentMemoryMB,
+ peak_memory_mb: this.peakMemoryMB > 0 ? this.peakMemoryMB : undefined
+ }
+
+ const progress: ImportProgress = {
+ // Overall
+ overall_progress: overallProgress,
+ overall_status: overallStatus,
+
+ // Stage
+ stage: this.currentStage,
+ stage_progress: stageProgress,
+ stage_message: stageMessage,
+
+ // Bytes
+ bytes_processed: this.bytesProcessed,
+ total_bytes: this.totalBytes,
+ bytes_percentage: bytesPercentage,
+ bytes_per_second: bytesPerSec,
+
+ // Entities
+ entities_extracted: this.entitiesExtracted,
+ entities_indexed: this.entitiesIndexed,
+ entities_per_second: entitiesPerSec,
+ estimated_total_entities: entityEstimate?.count,
+ estimation_confidence: entityEstimate?.confidence,
+
+ // Timing
+ elapsed_ms: elapsed,
+ estimated_remaining_ms: remainingMs,
+ estimated_total_ms: remainingMs ? elapsed + remainingMs : undefined,
+
+ // Context
+ current_item: this.currentItem,
+ current_file: this.currentFile,
+ file_number: this.fileNumber,
+ total_files: this.totalFiles,
+
+ // Metrics
+ metrics,
+
+ // Backwards compatibility
+ current: this.entitiesIndexed,
+ total: entityEstimate?.count ?? 0
+ }
+
+ return progress
+ }
+
+ /**
+ * Emit progress (throttled)
+ */
+ private emit(force: boolean = false): void {
+ if (!this.callback) return
+
+ const now = Date.now()
+ const timeSinceLastEmit = now - this.lastEmitTime
+
+ // Throttle unless forced
+ if (!force && timeSinceLastEmit < this.throttleMs) {
+ return
+ }
+
+ const progress = this.buildProgress()
+
+ // Handle both callback types (legacy and new)
+ if (this.callback.length === 2) {
+ // Legacy callback: (current, total) => void
+ ;(this.callback as (current: number, total: number) => void)(
+ progress.current,
+ progress.total
+ )
+ } else {
+ // New callback: (progress: ImportProgress) => void
+ ;(this.callback as (progress: ImportProgress) => void)(progress)
+ }
+
+ this.lastEmitTime = now
+ }
+
+ /**
+ * Force emit (for completion or critical updates)
+ */
+ forceEmit(): void {
+ this.emit(true)
+ }
+
+ /**
+ * Get current progress (without emitting)
+ */
+ getProgress(): ImportProgress {
+ return this.buildProgress()
+ }
+
+ /**
+ * Mark import as complete
+ */
+ complete(): ImportProgress {
+ this.currentStage = 'completing'
+ this.completedStages.add('completing')
+
+ const progress = this.buildProgress()
+ this.forceEmit()
+
+ return progress
+ }
+}