From d5576ffb56c351071dd56be110c02de4c4440dd8 Mon Sep 17 00:00:00 2001 From: David Snelling Date: Fri, 24 Oct 2025 14:45:46 -0700 Subject: [PATCH] feat: comprehensive import progress tracking for all 7 formats MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add real-time progress reporting throughout the entire import pipeline with a standardized API that works across all supported formats. Workshop Team Feature Request: - Eliminates "0% complete" hangs during AI extraction - Shows continuous progress with entities/sec, throughput, ETA - Reports contextual messages ("Processing page 5 of 23") - Standardized progress API for CSV, PDF, Excel, JSON, Markdown, YAML, DOCX Core Changes: - Add FormatHandlerProgressHooks interface for extensible progress - Wire up all 3 binary format handlers (CSV, PDF, Excel) with 7+ progress points - Wire up all 4 text format importers (JSON, Markdown, YAML, DOCX) - Add ImportProgress interface with stage, message, counts, throughput, ETA - ImportCoordinator normalizes all format progress to standard interface CLI Improvements: - Import command now uses brain.import() directly with full progress - Add --include-vfs flag to find command (v4.4.0 compatibility) - Add --confidence and --weight options to add command Documentation: - docs/guides/standard-import-progress.md - Universal API guide - docs/guides/import-progress-implementation.md - Developer guide - docs/guides/import-progress-examples.md - Practical examples - JSDoc on brain.import() with universal handler examples Result: ONE progress handler works for ALL 7 formats with zero format-specific code! ๐Ÿค– Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude --- docs/DEVELOPER_LEARNING_PATH.md | 1239 +++++++++++++++++ docs/guides/import-progress-examples.md | 370 +++++ docs/guides/import-progress-implementation.md | 734 ++++++++++ docs/guides/standard-import-progress.md | 453 ++++++ .../intelligentImport/handlers/csvHandler.ts | 41 +- .../handlers/excelHandler.ts | 65 +- .../intelligentImport/handlers/pdfHandler.ts | 46 + src/augmentations/intelligentImport/types.ts | 40 + src/brainy.ts | 25 +- src/cli/commands/core.ts | 34 +- src/cli/commands/import.ts | 108 +- src/importers/SmartCSVImporter.ts | 36 +- src/importers/SmartDOCXImporter.ts | 14 + src/importers/SmartExcelImporter.ts | 38 +- src/importers/SmartJSONImporter.ts | 21 + src/importers/SmartMarkdownImporter.ts | 30 +- src/importers/SmartPDFImporter.ts | 38 +- src/importers/SmartYAMLImporter.ts | 14 + src/types/brainy.types.ts | 132 ++ src/utils/import-progress-tracker.ts | 541 +++++++ 20 files changed, 3967 insertions(+), 52 deletions(-) create mode 100644 docs/DEVELOPER_LEARNING_PATH.md create mode 100644 docs/guides/import-progress-examples.md create mode 100644 docs/guides/import-progress-implementation.md create mode 100644 docs/guides/standard-import-progress.md create mode 100644 src/utils/import-progress-tracker.ts diff --git a/docs/DEVELOPER_LEARNING_PATH.md b/docs/DEVELOPER_LEARNING_PATH.md new file mode 100644 index 00000000..3436d922 --- /dev/null +++ b/docs/DEVELOPER_LEARNING_PATH.md @@ -0,0 +1,1239 @@ +# ๐ŸŽ“ Brainy Developer Learning Path +**From Zero to Hero in 5 Progressive Levels** + +> This guide takes you from your first Brainy query to production-scale neural database mastery. Follow each level in order for the best learning experience. + +--- + +## ๐Ÿ“‹ Quick Navigation + +- [Level 1: Hello Brainy](#level-1-hello-brainy-15-minutes) - Your first neural database +- [Level 2: Relationships & Batch Operations](#level-2-relationships--batch-operations-30-minutes) - Scale up your data +- [Level 3: Advanced Search & Neural AI](#level-3-advanced-search--neural-ai-45-minutes) - Triple Intelligence +- [Level 4: Virtual Filesystem](#level-4-virtual-filesystem-60-minutes) - Files as intelligent entities +- [Level 5: Production Scale](#level-5-production-scale-90-minutes) - Planet-scale deployment + +--- + +## Level 1: Hello Brainy (15 minutes) + +### What You'll Learn +- Initialize Brainy +- Add your first entity +- Perform semantic search +- Understand basic types + +### Prerequisites +```bash +npm install @soulcraft/brainy +``` + +### Your First Neural Database + +```typescript +import { Brainy, NounType } from '@soulcraft/brainy' + +// Step 1: Create and initialize Brainy +const brain = new Brainy({ + storage: { type: 'memory' } // Start simple - no persistence needed +}) +await brain.init() + +// Step 2: Add some data +const johnId = await brain.add({ + data: 'John Smith is a software engineer at TechCorp', + type: NounType.Person, + metadata: { role: 'Engineer', company: 'TechCorp' } +}) + +const aliceId = await brain.add({ + data: 'Alice Johnson is a product manager at TechCorp', + type: NounType.Person, + metadata: { role: 'Manager', company: 'TechCorp' } +}) + +const projectId = await brain.add({ + data: 'AI-powered customer support system using machine learning', + type: NounType.Project, + metadata: { status: 'active', priority: 'high' } +}) + +// Step 3: Semantic search (this is where magic happens!) +console.log('\n๐Ÿ” Searching for "engineers"...') +const engineers = await brain.find({ query: 'engineers' }) +console.log(`Found ${engineers.length} engineers:`) +for (const result of engineers) { + console.log(` - ${result.entity.data} (score: ${result.score.toFixed(2)})`) +} + +// Step 4: Search with filters +console.log('\n๐Ÿ” Searching for "people at TechCorp"...') +const techcorpPeople = await brain.find({ + query: 'people', + type: NounType.Person, + where: { company: 'TechCorp' }, + limit: 10 +}) +console.log(`Found ${techcorpPeople.length} people at TechCorp`) + +// Step 5: Get entity by ID +const john = await brain.get(johnId) +console.log('\n๐Ÿ‘ค John\'s data:', { + type: john?.type, + data: john?.data, + metadata: john?.metadata +}) + +// Step 6: Clean up +await brain.close() +console.log('\nโœ… Done! You just created your first neural database!') +``` + +### Key Concepts + +#### 1. **NounType** - Entity Classification +Brainy has 31 built-in types including: +- `Person`, `Organization`, `Location` +- `Document`, `File`, `Content` +- `Product`, `Service`, `Event` +- `Project`, `Task`, `Concept` + +**Why it matters**: Proper typing enables intelligent search and organization. + +#### 2. **Semantic Search** - Understanding Meaning +```typescript +// Traditional search: exact keyword matching +// "engineers" would NOT find "software developer" + +// Semantic search: understands meaning +await brain.find({ query: 'engineers' }) +// โœ… Finds: "software engineer", "developer", "programmer", "coder" +``` + +#### 3. **Metadata Filtering** - Precise Control +```typescript +// Combine semantic search with structured filters +await brain.find({ + query: 'machine learning', // Semantic: finds AI, ML, neural networks + where: { company: 'TechCorp' }, // Structured: exact match + type: NounType.Project // Type filter +}) +``` + +### Practice Exercises + +1. Create a small company directory with 5-10 people +2. Search for "managers", "developers", "designers" +3. Add projects and search for "active projects" +4. Experiment with different metadata filters + +### Next Steps +Once you're comfortable with basic operations, move to **Level 2** to learn about relationships and batch operations. + +--- + +## Level 2: Relationships & Batch Operations (30 minutes) + +### What You'll Learn +- Create relationships between entities +- Batch add/update/delete operations +- Query graph relationships +- Understand VerbTypes + +### Building a Knowledge Graph + +```typescript +import { Brainy, NounType, VerbType } from '@soulcraft/brainy' + +const brain = new Brainy({ storage: { type: 'memory' } }) +await brain.init() + +// Batch add multiple entities +console.log('๐Ÿ“ฆ Adding team members...') +const result = await brain.addMany({ + items: [ + { data: 'John Smith - Senior Engineer', type: NounType.Person, metadata: { role: 'Engineer' } }, + { data: 'Alice Johnson - Product Manager', type: NounType.Person, metadata: { role: 'Manager' } }, + { data: 'Bob Wilson - Designer', type: NounType.Person, metadata: { role: 'Designer' } }, + { data: 'TechCorp - Software Company', type: NounType.Organization }, + { data: 'AI Assistant Project', type: NounType.Project, metadata: { status: 'active' } } + ], + parallel: true, + onProgress: (done, total) => console.log(` Progress: ${done}/${total}`) +}) + +console.log(`โœ… Added ${result.successful.length} entities`) +const [johnId, aliceId, bobId, techcorpId, projectId] = result.successful + +// Create relationships (building the graph!) +console.log('\n๐Ÿ”— Creating relationships...') +await brain.relateMany({ + relations: [ + // People work for organization + { from: johnId, to: techcorpId, type: VerbType.WorksWith }, + { from: aliceId, to: techcorpId, type: VerbType.WorksWith }, + { from: bobId, to: techcorpId, type: VerbType.WorksWith }, + + // People work on project + { from: johnId, to: projectId, type: VerbType.WorksOn }, + { from: bobId, to: projectId, type: VerbType.WorksOn }, + + // Alice manages the project + { from: aliceId, to: projectId, type: VerbType.Manages }, + + // Team collaboration + { from: johnId, to: aliceId, type: VerbType.CollaboratesWith, bidirectional: true }, + { from: bobId, to: johnId, type: VerbType.CollaboratesWith, bidirectional: true } + ] +}) + +console.log('โœ… Created relationships') + +// Query relationships +console.log('\n๐Ÿ” Querying relationships...') + +// Who works for TechCorp? +const techcorpEmployees = await brain.getRelations({ + to: techcorpId, + type: VerbType.WorksWith +}) +console.log(`TechCorp has ${techcorpEmployees.length} employees`) + +// Who works on the AI project? +const projectContributors = await brain.getRelations({ + to: projectId, + type: [VerbType.WorksOn, VerbType.Manages] +}) +console.log(`AI Project has ${projectContributors.length} contributors`) + +// Who does John collaborate with? +const johnsCollaborators = await brain.getRelations({ + from: johnId, + type: VerbType.CollaboratesWith +}) +console.log(`John collaborates with ${johnsCollaborators.length} people`) + +// Get graph statistics +const stats = brain.getStats() +console.log('\n๐Ÿ“Š Graph Statistics:', { + entities: stats.entities.total, + relationships: stats.relationships.totalRelationships, + density: stats.density.toFixed(2) +}) + +// Batch update +console.log('\n๐Ÿ“ Updating all team members...') +await brain.updateMany({ + items: [johnId, aliceId, bobId].map(id => ({ + id, + metadata: { team: 'AI Team', updated: new Date().toISOString() }, + merge: true // Merge with existing metadata (don't replace!) + })) +}) + +console.log('โœ… Updated team metadata') + +await brain.close() +``` + +### Key Concepts + +#### 1. **VerbType** - Relationship Types +Brainy has 40 relationship types including: +- Work: `WorksWith`, `WorksOn`, `Manages`, `Supervises` +- Structure: `PartOf`, `Contains`, `BelongsTo` +- Knowledge: `RelatedTo`, `DependsOn`, `Requires` +- Creation: `Creates`, `Modifies`, `Transforms` + +#### 2. **Bidirectional Relationships** +```typescript +await brain.relate({ + from: personA, + to: personB, + type: VerbType.CollaboratesWith, + bidirectional: true // Creates Aโ†’B AND Bโ†’A +}) +``` + +#### 3. **Batch Operations = Performance** +```typescript +// โŒ Slow: 100 individual operations +for (const item of items) { + await brain.add(item) // 100 round trips! +} + +// โœ… Fast: 1 batch operation +await brain.addMany({ items }) // 1 round trip! +``` + +#### 4. **Metadata Merging** +```typescript +// Initial metadata +await brain.add({ + data: 'John', + metadata: { role: 'Engineer', level: 3 } +}) + +// Update with merge: true (default) +await brain.update({ + id: johnId, + metadata: { team: 'AI Team' }, + merge: true // Result: { role: 'Engineer', level: 3, team: 'AI Team' } +}) + +// Update with merge: false +await brain.update({ + id: johnId, + metadata: { team: 'AI Team' }, + merge: false // Result: { team: 'AI Team' } - role and level lost! +}) +``` + +### Practice Exercises + +1. Create an organizational hierarchy (CEO โ†’ Managers โ†’ Engineers) +2. Build a project dependency graph +3. Model a social network with CollaboratesWith relationships +4. Query "Who reports to Alice?" using getRelations() +5. Batch update all projects to add a "year: 2024" field + +### Next Steps +Ready for AI-powered search and clustering? Move to **Level 3**. + +--- + +## Level 3: Advanced Search & Neural AI (45 minutes) + +### What You'll Learn +- Triple Intelligence (Vector + Metadata + Graph) +- Semantic similarity +- Automatic clustering +- Outlier detection +- Natural language queries + +### Triple Intelligence in Action + +```typescript +import { Brainy, NounType, VerbType } from '@soulcraft/brainy' + +const brain = new Brainy({ storage: { type: 'memory' } }) +await brain.init() + +// Create a realistic dataset +console.log('๐Ÿ“ฆ Creating knowledge base...') +const knowledgeBase = await brain.addMany({ + items: [ + // Research papers + { data: 'Deep Learning for Computer Vision using Convolutional Neural Networks', + type: NounType.Document, + metadata: { category: 'AI', year: 2024, citations: 150 } }, + { data: 'Natural Language Processing with Transformer Models', + type: NounType.Document, + metadata: { category: 'AI', year: 2024, citations: 200 } }, + { data: 'Reinforcement Learning for Robotics Applications', + type: NounType.Document, + metadata: { category: 'AI', year: 2023, citations: 80 } }, + + // Different domain + { data: 'Climate Change Impact on Ocean Ecosystems', + type: NounType.Document, + metadata: { category: 'Climate', year: 2024, citations: 120 } }, + { data: 'Renewable Energy Solutions for Urban Planning', + type: NounType.Document, + metadata: { category: 'Energy', year: 2024, citations: 95 } }, + + // Code projects + { data: 'AI-powered code completion tool using GPT', + type: NounType.Project, + metadata: { category: 'Tools', status: 'active' } }, + { data: 'Neural network visualization dashboard', + type: NounType.Project, + metadata: { category: 'Tools', status: 'active' } } + ] +}) + +console.log(`โœ… Created ${knowledgeBase.successful.length} entities\n`) + +// 1. VECTOR INTELLIGENCE: Semantic similarity +console.log('๐Ÿ” 1. VECTOR INTELLIGENCE: Semantic Search') +const aiResults = await brain.find({ + query: 'machine learning and neural networks', // User's natural language + limit: 3 +}) + +console.log('Top 3 semantically similar documents:') +aiResults.forEach((r, i) => { + console.log(` ${i + 1}. [${r.score.toFixed(3)}] ${r.entity.data?.substring(0, 50)}...`) +}) + +// 2. METADATA INTELLIGENCE: Structured filtering +console.log('\n๐Ÿ” 2. METADATA INTELLIGENCE: Precise Filtering') +const recentHighCitations = await brain.find({ + query: 'artificial intelligence', + where: { + year: 2024, + citations: { $gte: 100 } // Brainy Field Operator: greater than or equal + }, + limit: 10 +}) + +console.log(`Found ${recentHighCitations.length} highly-cited AI papers from 2024`) + +// 3. GRAPH INTELLIGENCE: Relationship-aware search +console.log('\n๐Ÿ” 3. GRAPH INTELLIGENCE: Relationship-Aware Search') + +// First, create some relationships +const [paper1, paper2] = knowledgeBase.successful +await brain.relate({ + from: paper1, + to: paper2, + type: VerbType.References +}) + +// Search with graph constraints +const connectedDocs = await brain.find({ + query: 'deep learning', + connected: { + to: paper2, + via: VerbType.References + } +}) + +console.log(`Found ${connectedDocs.length} papers that reference the NLP paper`) + +// 4. FUSION: Combine all three intelligences! +console.log('\n๐Ÿ” 4. TRIPLE INTELLIGENCE FUSION') +const fusionResults = await brain.find({ + query: 'AI research', // Vector: semantic understanding + where: { year: 2024 }, // Metadata: structured filter + type: NounType.Document, // Type constraint + fusion: { + strategy: 'adaptive', // Let Brainy optimize weights + weights: { + vector: 0.5, // 50% semantic similarity + field: 0.3, // 30% metadata match + graph: 0.2 // 20% relationship strength + } + }, + explain: true // See how the score was calculated +}) + +console.log('Fusion search results with score explanations:') +fusionResults.forEach(r => { + console.log(`\n ${r.entity.data?.substring(0, 60)}...`) + console.log(` Total score: ${r.score.toFixed(3)}`) + if (r.explanation) { + console.log(` Vector: ${r.explanation.vector.toFixed(3)}`) + console.log(` Metadata: ${r.explanation.metadata.toFixed(3)}`) + console.log(` Graph: ${r.explanation.graph.toFixed(3)}`) + } +}) + +// 5. NEURAL API: Automatic clustering +console.log('\n\n๐Ÿค– NEURAL API: Automatic Clustering') +const neural = brain.neural() + +const clusters = await neural.clusters({ + maxClusters: 3, + minClusterSize: 1 +}) + +console.log(`Found ${clusters.length} semantic clusters:`) +clusters.forEach((cluster, i) => { + console.log(`\n Cluster ${i + 1}: ${cluster.label || cluster.id}`) + console.log(` Members: ${cluster.members.length}`) + console.log(` Centroid topics: ${cluster.metadata?.topics?.join(', ') || 'N/A'}`) +}) + +// 6. SIMILARITY: Find similar documents +console.log('\n\n๐Ÿ” SIMILARITY: Find Similar Documents') +const similarTo = await brain.similar({ + to: paper1, // Entity ID of first AI paper + limit: 3, + threshold: 0.5, // Minimum similarity score + type: NounType.Document +}) + +console.log(`Documents similar to "${knowledgeBase.successful[0]}":`) +similarTo.forEach(r => { + console.log(` [${r.score.toFixed(3)}] ${r.entity.data?.substring(0, 50)}...`) +}) + +// 7. OUTLIER DETECTION +console.log('\n\n๐Ÿšจ OUTLIER DETECTION') +const outliers = await neural.outliers({ + method: 'statistical', + threshold: 2.0 // 2 standard deviations +}) + +console.log(`Found ${outliers.length} outlier documents:`) +outliers.forEach(o => { + const entity = await brain.get(o.id) + console.log(` [Anomaly score: ${o.score.toFixed(3)}] ${entity?.data?.substring(0, 50)}...`) +}) + +await brain.close() +``` + +### Key Concepts + +#### 1. **Triple Intelligence Explained** + +``` +Traditional Database: WHERE category = 'AI' (exact match only) + โŒ Misses: "artificial intelligence", "machine learning" + +Vector Search: semantic("AI research") (meaning-based) + โœ… Finds: AI, ML, neural networks, deep learning + โŒ No filtering by year, citations, etc. + +Brainy Triple: semantic("AI") + WHERE year=2024 + CONNECTED TO paper123 + โœ… Finds semantically similar + filters + graph aware +``` + +#### 2. **Score Explanations** +```typescript +const results = await brain.find({ + query: 'AI', + explain: true // Get score breakdown +}) + +// result.explanation shows: +// { +// vector: 0.85, // 85% semantic match +// metadata: 0.90, // 90% field match +// graph: 0.70, // 70% graph relevance +// final: 0.82 // Weighted combination +// } +``` + +#### 3. **Fusion Strategies** +```typescript +// 'adaptive' - Brainy automatically adjusts weights based on query +fusion: { strategy: 'adaptive' } + +// 'balanced' - Equal weights to all signals +fusion: { strategy: 'balanced' } + +// 'custom' - You control the weights +fusion: { + strategy: 'custom', + weights: { vector: 0.7, field: 0.2, graph: 0.1 } +} +``` + +#### 4. **Brainy Field Operators (BFO)** +```typescript +where: { + age: { $gte: 18, $lte: 65 }, // Range + role: { $in: ['Engineer', 'Manager'] }, // One of + name: { $contains: 'John' }, // Substring + active: true, // Exact match + tags: { $includes: 'AI' } // Array contains +} +``` + +### Practice Exercises + +1. Create a document collection and find semantically similar items +2. Use fusion search with custom weights +3. Cluster your data and examine the clusters +4. Find outliers in a dataset +5. Compare results with/without explain: true + +### Next Steps +Want to treat files as intelligent entities? Learn the **Virtual Filesystem** in Level 4. + +--- + +## Level 4: Virtual Filesystem (60 minutes) + +### What You'll Learn +- VFS as knowledge operating system +- Files with semantic understanding +- Semantic file search +- Cross-boundary relationships (VFS โ†” Knowledge) +- VFS filtering architecture + +### Files as Intelligent Entities + +```typescript +import { Brainy, NounType, VerbType } from '@soulcraft/brainy' + +const brain = new Brainy({ storage: { type: 'memory' } }) +await brain.init() + +// Initialize VFS +const vfs = brain.vfs() +await vfs.init() + +console.log('๐Ÿ“ Creating semantic filesystem...\n') + +// 1. BASIC FILE OPERATIONS (POSIX-like) +await vfs.mkdir('/projects', { recursive: true }) +await vfs.mkdir('/projects/ai-assistant') +await vfs.mkdir('/docs') + +await vfs.writeFile('/projects/ai-assistant/README.md', ` +# AI Assistant Project + +A neural-powered assistant using transformer models for natural language understanding. + +## Features +- Semantic search +- Context-aware responses +- Multi-turn conversations +`) + +await vfs.writeFile('/projects/ai-assistant/architecture.md', ` +# Architecture + +## Components +- NLP Engine: Transformer-based language model +- Knowledge Graph: Brainy neural database +- API Layer: RESTful endpoints +`) + +await vfs.writeFile('/docs/installation.md', ` +# Installation Guide + +\`\`\`bash +npm install ai-assistant +\`\`\` +`) + +console.log('โœ… Created 3 files\n') + +// 2. VFS-ONLY SEMANTIC SEARCH +console.log('๐Ÿ” Searching VFS for "neural networks"...') +const vfsFiles = await vfs.search('neural networks', { limit: 5 }) +console.log(`Found ${vfsFiles.length} VFS files:`) +vfsFiles.forEach(f => { + console.log(` [${f.score.toFixed(3)}] ${f.path}`) +}) + +// 3. VFS FILTERING IN KNOWLEDGE QUERIES +console.log('\n๐Ÿ” Understanding VFS filtering (v4.4.0)...\n') + +// Create some knowledge entities +const conceptId = await brain.add({ + data: 'Neural networks are computational models inspired by biological neurons', + type: NounType.Concept, + metadata: { topic: 'AI' } +}) + +const projectId = await brain.add({ + data: 'AI Assistant - conversational AI using transformers', + type: NounType.Project, + metadata: { status: 'active' } +}) + +console.log('Created 2 knowledge entities\n') + +// DEFAULT: Knowledge queries exclude VFS (clean separation!) +console.log('๐Ÿ“Š brain.find() - DEFAULT behavior (excludes VFS):') +const knowledgeOnly = await brain.find({ query: 'neural networks' }) +console.log(` Found ${knowledgeOnly.length} entities`) +console.log(` VFS files: ${knowledgeOnly.filter(r => r.metadata?.isVFS).length}`) // 0 +console.log(` Knowledge: ${knowledgeOnly.filter(r => !r.metadata?.isVFS).length}`) + +// OPT-IN: Include VFS when needed +console.log('\n๐Ÿ“Š brain.find() with includeVFS: true:') +const everything = await brain.find({ + query: 'neural networks', + includeVFS: true // Opt-in to include VFS files +}) +console.log(` Found ${everything.length} entities`) +console.log(` VFS files: ${everything.filter(r => r.metadata?.isVFS).length}`) +console.log(` Knowledge: ${everything.filter(r => !r.metadata?.isVFS).length}`) + +// VFS-ONLY: Search only files +console.log('\n๐Ÿ“Š Searching ONLY VFS files:') +const filesOnly = await brain.find({ + where: { vfsType: 'file', extension: '.md' }, + includeVFS: true // Required to find VFS entities +}) +console.log(` Found ${filesOnly.length} markdown files`) + +// 4. CROSS-BOUNDARY RELATIONSHIPS +console.log('\n\n๐Ÿ”— Creating cross-boundary relationships...') + +// Link concept to documentation file +const readmeEntity = await brain.find({ + where: { path: '/projects/ai-assistant/README.md' }, + includeVFS: true, + limit: 1 +}) + +if (readmeEntity.length > 0) { + await brain.relate({ + from: conceptId, + to: readmeEntity[0].id, + type: VerbType.DocumentedBy, + metadata: { section: 'Features' } + }) + console.log('โœ… Linked concept to README.md') +} + +// Query relationships +const conceptDocs = await brain.getRelations({ + from: conceptId, + type: VerbType.DocumentedBy +}) +console.log(`Concept is documented by ${conceptDocs.length} files`) + +// 5. VFS SEMANTIC FEATURES +console.log('\n\n๐Ÿ” VFS Semantic Features:') + +// Find similar files +const similarFiles = await vfs.findSimilar('/projects/ai-assistant/README.md', { + limit: 3, + threshold: 0.5 +}) +console.log(`\nFiles similar to README.md: ${similarFiles.length}`) +similarFiles.forEach(f => { + console.log(` [${f.score.toFixed(3)}] ${f.path}`) +}) + +// Get file stats +const stats = await vfs.stat('/projects/ai-assistant/README.md') +console.log('\nREADME.md stats:', { + size: stats.size, + type: stats.vfsType, + extension: stats.metadata?.extension, + created: new Date(stats.metadata?.createdAt || 0).toLocaleString() +}) + +// Read directory +console.log('\n๐Ÿ“ Directory contents of /projects/ai-assistant:') +const entries = await vfs.readdir('/projects/ai-assistant') +console.log(entries) + +// 6. METADATA & EXTENDED ATTRIBUTES +console.log('\n\n๐Ÿ“ Metadata & Extended Attributes:') + +await vfs.setMetadata('/projects/ai-assistant/README.md', { + author: 'John Smith', + version: '1.0.0', + tags: ['AI', 'documentation', 'project'] +}) + +const metadata = await vfs.getMetadata('/projects/ai-assistant/README.md') +console.log('README metadata:', metadata) + +// Extended attributes (like file properties) +await vfs.setxattr('/projects/ai-assistant/README.md', 'priority', 'high') +await vfs.setxattr('/projects/ai-assistant/README.md', 'reviewStatus', 'approved') + +const xattrs = await vfs.listxattr('/projects/ai-assistant/README.md') +console.log('Extended attributes:', xattrs) + +// 7. FILE OPERATIONS +console.log('\n\n๐Ÿ“‹ Advanced File Operations:') + +// Copy file +await vfs.copy('/docs/installation.md', '/projects/ai-assistant/INSTALL.md') +console.log('โœ… Copied installation.md') + +// Rename +await vfs.rename('/projects/ai-assistant/INSTALL.md', '/projects/ai-assistant/setup.md') +console.log('โœ… Renamed to setup.md') + +// Check existence +const exists = await vfs.exists('/projects/ai-assistant/setup.md') +console.log(`setup.md exists: ${exists}`) + +console.log('\n\nโœ… VFS Tutorial Complete!') +console.log('\n๐Ÿ“š Key Takeaways:') +console.log(' 1. VFS files have semantic understanding (search by meaning)') +console.log(' 2. brain.find() excludes VFS by default (clean knowledge queries)') +console.log(' 3. Use includeVFS: true to include VFS in knowledge queries') +console.log(' 4. vfs.search() ONLY searches VFS files (never knowledge entities)') +console.log(' 5. Cross-boundary relationships link files to concepts') +console.log(' 6. Every file is a full Brainy entity with vector, metadata, and graph') + +await vfs.close() +await brain.close() +``` + +### Key Concepts + +#### 1. **VFS Filtering Architecture (v4.4.0)** + +```typescript +// ๐ŸŽฏ DEFAULT BEHAVIOR: Clean Separation +// +// Knowledge queries stay clean (no VFS pollution) +const concepts = await brain.find({ query: 'AI' }) +// Returns: Only NounType.Concept, NounType.Document, etc. +// Excludes: VFS files (no .path property) + +// VFS queries work with VFS only +const files = await vfs.search('documentation') +// Returns: Only VFS files with .path property +// Excludes: Knowledge entities + +// ๐Ÿ”„ CROSS-BOUNDARY: Opt-in when needed +const everything = await brain.find({ + query: 'machine learning', + includeVFS: true // Include both knowledge AND VFS +}) +// Returns: Knowledge entities + VFS files + +// ๐Ÿ“ VFS-ONLY via brain.find() +const markdownFiles = await brain.find({ + where: { vfsType: 'file', extension: '.md' }, + includeVFS: true // Required to find VFS entities +}) +``` + +#### 2. **Cross-Boundary Relationships** + +```typescript +// Files can relate to knowledge entities +await brain.relate({ + from: conceptId, // Knowledge: NounType.Concept + to: fileId, // VFS: File entity + type: VerbType.DocumentedBy +}) + +// Query across boundaries +const conceptDocs = await brain.getRelations({ + from: conceptId, + type: VerbType.DocumentedBy +}) +// Returns: VFS files that document the concept +``` + +#### 3. **VFS vs Traditional Filesystem** + +| Feature | Traditional FS | Brainy VFS | +|---------|---------------|------------| +| Search | Filename only | Semantic content search | +| Organization | Hierarchy only | Hierarchy + Graph | +| Metadata | Limited (size, dates) | Unlimited custom metadata | +| Relationships | None | Full graph relationships | +| Similarity | None | Find similar files | +| Understanding | None | Vector embeddings | + +#### 4. **When to Use What** + +```typescript +// Use vfs.* methods for file operations +await vfs.writeFile('/path/to/file.txt', content) +await vfs.readFile('/path/to/file.txt') +await vfs.search('semantic query') + +// Use brain.* methods for knowledge operations +await brain.add({ data: 'concept', type: NounType.Concept }) +await brain.find({ query: 'concept' }) // Excludes VFS by default + +// Use includeVFS for cross-boundary queries +await brain.find({ + query: 'documentation', + includeVFS: true // Search both knowledge AND files +}) +``` + +### Practice Exercises + +1. Create a project structure with docs, source code, tests +2. Add semantic tags to files +3. Search for "API documentation" and see VFS filtering in action +4. Create relationships between code files and design documents +5. Find files similar to a specific README +6. Compare results with/without includeVFS + +### Next Steps +Ready for production deployment? Level 5 covers **planet-scale architecture**. + +--- + +## Level 5: Production Scale (90 minutes) + +### What You'll Learn +- Cloud storage (GCS, S3, R2) +- Performance optimization +- Batch imports (CSV, Excel, PDF) +- Metadata query optimization +- Production best practices + +### Production-Ready Deployment + +```typescript +import { Brainy, NounType } from '@soulcraft/brainy' + +// 1. PRODUCTION STORAGE - Google Cloud Storage (Native SDK) +console.log('โ˜๏ธ Initializing production storage...\n') + +const brain = new Brainy({ + storage: { + type: 'gcs-native', // Native GCS SDK (recommended) + gcsNativeStorage: { + bucketName: 'my-brainy-production', + // ADC (Application Default Credentials) - zero config in Cloud Run/GCE! + // Or provide credentials: + // keyFilename: '/path/to/service-account.json' + } + }, + + // Performance tuning + cache: { + maxSize: 10000, // Cache up to 10K entities + ttl: 600000 // 10 minute TTL + }, + + // Monitoring + verbose: process.env.NODE_ENV === 'development' +}) + +await brain.init() +console.log('โœ… Brainy initialized with GCS Native storage\n') + +// 2. BATCH IMPORT - CSV File +console.log('๐Ÿ“Š Importing CSV data...\n') + +const csvResult = await brain.import('./data/customers-1000.csv', { + vfsPath: '/imports/customers.csv', // Store in VFS + createEntities: true, // Create knowledge entities + batchSize: 100, // Process in batches of 100 + onProgress: (done, total) => { + console.log(` Progress: ${done}/${total} (${(done/total*100).toFixed(1)}%)`) + } +}) + +console.log('\n๐Ÿ“Š Import Results:') +console.log(` Entities created: ${csvResult.stats.graphNodesCreated}`) +console.log(` VFS files created: ${csvResult.stats.vfsFilesCreated}`) +console.log(` Duration: ${csvResult.stats.duration}ms`) + +// 3. METADATA QUERY OPTIMIZATION +console.log('\n\n๐Ÿ” Metadata Query Optimization:\n') + +// Discover what fields are available +const fields = await brain.getAvailableFields() +console.log(`Available metadata fields: ${fields.length}`) +console.log(` Top fields: ${fields.slice(0, 10).join(', ')}`) + +// Get field statistics (cardinality, types) +const fieldStats = await brain.getFieldStatistics() +console.log(`\nField statistics:`) +const topFields = Array.from(fieldStats.entries()).slice(0, 5) +topFields.forEach(([field, stats]) => { + console.log(` ${field}: ${stats.cardinality} unique values`) +}) + +// Get optimal query plan +const queryPlan = await brain.getOptimalQueryPlan({ + status: 'active', + year: 2024 +}) +console.log(`\nQuery plan:`) +console.log(` Estimated results: ${queryPlan.estimatedResults}`) +console.log(` Index usage: ${queryPlan.indexUsage.join(', ')}`) +console.log(` Execution time: ~${queryPlan.estimatedMs}ms`) + +// 4. LARGE-SCALE BATCH OPERATIONS +console.log('\n\n๐Ÿ“ฆ Large-Scale Batch Operations:\n') + +// Generate test data +const testItems = Array.from({ length: 1000 }, (_, i) => ({ + data: `Test entity ${i} - Machine learning and artificial intelligence`, + type: NounType.Document, + metadata: { + index: i, + category: i % 5 === 0 ? 'AI' : 'General', + priority: Math.random() > 0.5 ? 'high' : 'normal', + year: 2024 + } +})) + +console.log(`Adding 1000 entities...`) +const startTime = Date.now() + +const batchResult = await brain.addMany({ + items: testItems, + parallel: true, + chunkSize: 100, + onProgress: (done, total) => { + if (done % 200 === 0) console.log(` ${done}/${total}`) + } +}) + +const duration = Date.now() - startTime +console.log(`\nโœ… Batch add complete:`) +console.log(` Success: ${batchResult.successful.length}`) +console.log(` Failed: ${batchResult.failed.length}`) +console.log(` Duration: ${duration}ms`) +console.log(` Throughput: ${(batchResult.successful.length / (duration / 1000)).toFixed(0)} entities/sec`) + +// 5. ADVANCED CLUSTERING (Large Dataset) +console.log('\n\n๐Ÿค– Clustering 1000 entities...\n') + +const neural = brain.neural() + +// Use fast clustering for large datasets +const clusters = await neural.clusterFast({ + maxClusters: 5 +}) + +console.log(`Found ${clusters.length} clusters:`) +clusters.forEach((cluster, i) => { + console.log(`\n Cluster ${i + 1}: ${cluster.label || cluster.id}`) + console.log(` Size: ${cluster.members.length} members`) + console.log(` Density: ${(cluster.density || 0).toFixed(3)}`) + if (cluster.metadata?.keywords) { + console.log(` Keywords: ${cluster.metadata.keywords.slice(0, 5).join(', ')}`) + } +}) + +// 6. PRODUCTION STATISTICS +console.log('\n\n๐Ÿ“Š Production Statistics:\n') + +const stats = brain.getStats() +console.log(`Total Entities: ${stats.entities.total.toLocaleString()}`) +console.log(`Total Relationships: ${stats.relationships.totalRelationships.toLocaleString()}`) +console.log(`Graph Density: ${stats.density.toFixed(4)}`) + +console.log(`\nEntities by Type:`) +Object.entries(stats.entities.byType) + .sort(([, a], [, b]) => (b as number) - (a as number)) + .slice(0, 5) + .forEach(([type, count]) => { + console.log(` ${type}: ${(count as number).toLocaleString()}`) + }) + +// 7. QUERY PERFORMANCE MONITORING +console.log('\n\nโšก Query Performance:\n') + +const perfStart = Date.now() +const searchResults = await brain.find({ + query: 'artificial intelligence machine learning', + where: { category: 'AI' }, + limit: 100, + explain: true +}) +const perfDuration = Date.now() - perfStart + +console.log(`Query completed in ${perfDuration}ms`) +console.log(` Results: ${searchResults.length}`) +console.log(` Avg score: ${(searchResults.reduce((sum, r) => sum + r.score, 0) / searchResults.length).toFixed(3)}`) + +// Show top result explanation +if (searchResults[0]?.explanation) { + console.log(`\n Top result score breakdown:`) + console.log(` Vector: ${searchResults[0].explanation.vector?.toFixed(3) || 'N/A'}`) + console.log(` Metadata: ${searchResults[0].explanation.metadata?.toFixed(3) || 'N/A'}`) + console.log(` Graph: ${searchResults[0].explanation.graph?.toFixed(3) || 'N/A'}`) +} + +// 8. CLEANUP & BEST PRACTICES +console.log('\n\n๐Ÿงน Production Best Practices:\n') + +// Always flush before shutdown +await brain.flush() +console.log('โœ… Flushed all data to storage') + +// Get final stats +const finalStats = brain.getStats() +console.log(`โœ… Final entity count: ${finalStats.entities.total.toLocaleString()}`) + +// Clean shutdown +await brain.close() +console.log('โœ… Brain closed cleanly') + +console.log('\n\n๐ŸŽ“ Production Deployment Complete!') +console.log('\n๐Ÿ“š Key Production Learnings:') +console.log(' 1. Use native cloud storage (GCS, S3, R2) for persistence') +console.log(' 2. Batch operations = 100x faster than individual ops') +console.log(' 3. Metadata query optimization for complex filters') +console.log(' 4. Monitor query performance with explain: true') +console.log(' 5. Always flush() before shutdown') +console.log(' 6. Use getStats() for O(1) counts (no expensive scans)') +console.log(' 7. Stream large imports with progress callbacks') +``` + +### Key Concepts + +#### 1. **Storage Options Comparison** + +| Storage | Use Case | Performance | Cost | Setup | +|---------|----------|-------------|------|-------| +| Memory | Dev/testing | Fastest | Free | Zero config | +| Filesystem | Local prod | Fast | Free | Local path | +| GCS Native | GCP prod | Fast | $$$ | Service account | +| S3 | AWS prod | Fast | $$$ | Access keys | +| R2 | Cloudflare | Fast | $ | Access keys | + +#### 2. **GCS Native vs S3-Compatible** + +```typescript +// โœ… RECOMMENDED: GCS Native SDK +{ + storage: { + type: 'gcs-native', + gcsNativeStorage: { + bucketName: 'my-bucket' + // ADC handles auth automatically in Cloud Run/GCE! + } + } +} + +// โš ๏ธ LEGACY: S3-compatible mode +{ + storage: { + type: 'gcs', + gcsStorage: { + bucketName: 'my-bucket', + accessKeyId: process.env.GCS_ACCESS_KEY, + secretAccessKey: process.env.GCS_SECRET_KEY + } + } +} +``` + +#### 3. **Performance Optimization** + +```typescript +// 1. Use batch operations +await brain.addMany({ items, parallel: true, chunkSize: 100 }) + +// 2. Enable caching +const brain = new Brainy({ + cache: { maxSize: 10000, ttl: 600000 } +}) + +// 3. Use metadata indexes for filtering +await brain.find({ + where: { status: 'active' }, // Uses MetadataIndexManager + limit: 100 +}) + +// 4. Optimize query plans +const plan = await brain.getOptimalQueryPlan(filters) +// Use plan to choose best query strategy + +// 5. Use writeOnly for bulk imports +await brain.add({ + data, + type, + writeOnly: true // Skip validation for speed +}) +``` + +#### 4. **Import Strategies** + +```typescript +// Small files (<10MB) - Direct import +await brain.import('./data.csv') + +// Large files (>10MB) - Stream with progress +await brain.import('./large-data.csv', { + batchSize: 1000, + onProgress: (done, total) => { + console.log(`${(done/total*100).toFixed(1)}%`) + } +}) + +// Very large files (>100MB) - External pipeline +// Use streaming pipeline API for max control +``` + +#### 5. **Monitoring & Observability** + +```typescript +// 1. Track query performance +const start = Date.now() +const results = await brain.find({ query, explain: true }) +const duration = Date.now() - start +console.log(`Query: ${duration}ms, Results: ${results.length}`) + +// 2. Monitor graph statistics +const stats = brain.getStats() +console.log(`Density: ${stats.density}`) // Relationships per entity + +// 3. Track field cardinality +const fieldStats = await brain.getFieldStatistics() +// High cardinality fields = good for filtering + +// 4. Enable verbose logging in dev +const brain = new Brainy({ verbose: true }) +``` + +### Production Checklist + +#### Before Deployment + +- [ ] Choose cloud storage (GCS/S3/R2) +- [ ] Set up authentication (service account/access keys) +- [ ] Configure caching +- [ ] Test batch operations +- [ ] Benchmark query performance +- [ ] Set up monitoring + +#### During Operation + +- [ ] Monitor query latency +- [ ] Track entity/relationship counts +- [ ] Watch for outliers +- [ ] Optimize slow queries +- [ ] Regular backups + +#### Scaling Considerations + +- [ ] Shard data by service/tenant +- [ ] Use read replicas for queries +- [ ] Implement rate limiting +- [ ] Monitor storage costs +- [ ] Plan for growth + +### Practice Exercises + +1. Deploy Brainy with GCS Native storage +2. Import a 10,000 row CSV file +3. Measure query performance for different filters +4. Optimize a slow query using getOptimalQueryPlan() +5. Set up monitoring dashboard +6. Create backup/restore scripts + +--- + +## ๐ŸŽ“ Graduation: You're a Brainy Expert! + +### What You've Mastered + +โœ… **Level 1**: Basic operations (add, find, search) +โœ… **Level 2**: Relationships & batch operations +โœ… **Level 3**: Triple Intelligence & Neural AI +โœ… **Level 4**: Virtual Filesystem +โœ… **Level 5**: Production deployment + +### Next Steps + +#### Advanced Topics + +- **Distributed Systems**: Sharding, replication, coordination +- **Custom Augmentations**: Extend Brainy with plugins +- **Streaming Pipelines**: Real-time data ingestion +- **Security**: Encryption, access control, audit logs +- **Framework Integration**: React, Vue, Next.js, Nuxt + +#### Resources + +- ๐Ÿ“š [API Reference](../API_REFERENCE.md) - Complete API documentation +- ๐Ÿ“ [VFS Guide](../vfs/VFS_API_GUIDE.md) - Virtual Filesystem deep dive +- ๐Ÿค– [Neural API](../guides/neural-api.md) - Advanced neural operations +- ๐ŸŒ [Distributed Guide](../guides/distributed-system.md) - Planet-scale architecture +- ๐Ÿ’ฌ [Discord Community](https://discord.gg/brainy) - Get help, share projects + +#### Share Your Success + +Built something cool with Brainy? Share it with the community! + +- GitHub: https://github.com/soulcraft/brainy +- Twitter: @brainydb +- Discord: https://discord.gg/brainy + +--- + +**Congratulations! You're now a Brainy expert ready to build production neural database applications! ๐ŸŽ‰** diff --git a/docs/guides/import-progress-examples.md b/docs/guides/import-progress-examples.md new file mode 100644 index 00000000..6489d19a --- /dev/null +++ b/docs/guides/import-progress-examples.md @@ -0,0 +1,370 @@ +# Import Progress - Usage Examples + +**How to Use Progress Tracking in Your Applications** + +Brainy v4.5.0+ provides real-time progress tracking for **all 7 supported file formats** (CSV, PDF, Excel, JSON, Markdown, YAML, DOCX). + +> **โš ๏ธ KEY FEATURE:** The progress API is **100% standardized**. Write your progress handler ONCE and it works for ALL formats with zero format-specific code! See [Standard Import Progress API](./standard-import-progress.md) for the complete interface documentation. + +--- + +## ๐Ÿš€ Quick Start + +### Basic Progress Tracking + +```typescript +import { Brainy } from '@soulcraft/brainy' +import * as fs from 'fs' + +const brain = await Brainy.create() + +// Import with progress tracking +const result = await brain.import(fs.readFileSync('large-file.xlsx'), { + onProgress: (progress) => { + console.log(`Progress: ${progress.stage}`) + console.log(` Message: ${progress.message}`) + console.log(` Entities: ${progress.entities || 0}`) + console.log(` Relationships: ${progress.relationships || 0}`) + } +}) + +console.log(`Import complete: ${result.entities.length} entities created`) +``` + +**Expected Output:** +``` +Progress: detecting + Message: Detecting format... + Entities: 0 + Relationships: 0 +Progress: extracting + Message: Loading Excel workbook... + Entities: 0 + Relationships: 0 +Progress: extracting + Message: Reading sheet: Sales (1/3) + Entities: 0 + Relationships: 0 +Progress: extracting + Message: Parsing Excel (33%) + Entities: 0 + Relationships: 0 +Progress: extracting + Message: Reading sheet: Products (2/3) + Entities: 0 + Relationships: 0 +... (more progress updates) +Progress: complete + Message: Import complete + Entities: 1523 + Relationships: 892 +Import complete: 1523 entities created +``` + +--- + +## ๐ŸŽฏ Universal Progress Handler (Works for ALL Formats) + +The examples below show format-specific messages, but **you don't need format-specific code**! The `ImportProgress` interface is the same for all formats: + +```typescript +// ONE HANDLER FOR ALL FORMATS! +function universalProgressHandler(progress) { + console.log(`[${progress.stage}] ${progress.message}`) + + if (progress.processed && progress.total) { + console.log(` Progress: ${progress.processed}/${progress.total}`) + } + + if (progress.entities || progress.relationships) { + console.log(` Extracted: ${progress.entities || 0} entities, ${progress.relationships || 0} relationships`) + } + + if (progress.throughput && progress.eta) { + console.log(` Rate: ${progress.throughput.toFixed(1)}/sec, ETA: ${Math.round(progress.eta/1000)}s`) + } +} + +// Use it for ANY format! +await brain.import(csvBuffer, { onProgress: universalProgressHandler }) +await brain.import(pdfBuffer, { onProgress: universalProgressHandler }) +await brain.import(excelBuffer, { onProgress: universalProgressHandler }) +await brain.import(jsonBuffer, { onProgress: universalProgressHandler }) +await brain.import(markdownString, { onProgress: universalProgressHandler }) +await brain.import(yamlBuffer, { onProgress: universalProgressHandler }) +await brain.import(docxBuffer, { onProgress: universalProgressHandler }) +``` + +--- + +## ๐Ÿ“Š What Different Formats Look Like (Same Handler!) + +The examples below show **what messages look like** for different formats using the **same universal handler** above. + +### CSV Import (Row-by-Row Progress) + +```typescript +await brain.import(csvBuffer, { + format: 'csv', + onProgress: (progress) => { + if (progress.stage === 'extracting') { + // CSV reports: "Parsing CSV (45%)", "Extracted 1000 rows", etc. + console.log(progress.message) + } + } +}) +``` + +**CSV Progress Messages:** +- โœ… "Detecting CSV encoding and delimiter..." +- โœ… "Parsing CSV rows (delimiter: ",")" +- โœ… "Parsed 75%" (via bytes processed) +- โœ… "Extracted 1000 rows" +- โœ… "Converting types: 5000/10000 rows..." +- โœ… "CSV processing complete: 10000 rows" + +--- + +### PDF Import (Page-by-Page Progress) + +```typescript +await brain.import(pdfBuffer, { + format: 'pdf', + onProgress: (progress) => { + // PDF reports exact page numbers + console.log(progress.message) + // Example: "Processing page 5 of 23" + } +}) +``` + +**PDF Progress Messages:** +- โœ… "Loading PDF document..." +- โœ… "Processing 23 pages..." +- โœ… "Processing page 5 of 23" +- โœ… "Parsed 22%" (via bytes processed) +- โœ… "Extracted 156 items from PDF" +- โœ… "PDF complete: 23 pages, 156 items extracted" + +--- + +### Excel Import (Sheet-by-Sheet Progress) + +```typescript +await brain.import(excelBuffer, { + format: 'excel', + onProgress: (progress) => { + // Excel reports sheet names + console.log(progress.message) + // Example: "Reading sheet: Q2 Sales (2/5)" + } +}) +``` + +**Excel Progress Messages:** +- โœ… "Loading Excel workbook..." +- โœ… "Processing 3 sheets..." +- โœ… "Reading sheet: Sales (1/3)" +- โœ… "Parsing Excel (33%)" (via bytes processed) +- โœ… "Extracted 5234 rows from Excel" +- โœ… "Excel complete: 3 sheets, 5234 rows" + +--- + +### JSON Import (Node Traversal) + +```typescript +await brain.import(jsonBuffer, { + format: 'json', + onProgress: (progress) => { + // JSON reports every 10 nodes + console.log(`Processed ${progress.processed} nodes, found ${progress.entities} entities`) + } +}) +``` + +--- + +### Markdown Import (Section-by-Section) + +```typescript +await brain.import(markdownString, { + format: 'markdown', + onProgress: (progress) => { + console.log(`Section ${progress.processed}/${progress.total}`) + } +}) +``` + +--- + +## ๐ŸŽฏ Building Progress UI Components + +### React Progress Bar + +```typescript +function ImportProgress({ file }: { file: File }) { + const [progress, setProgress] = useState({ + stage: 'idle', + message: '', + percent: 0, + entities: 0, + relationships: 0 + }) + + const handleImport = async () => { + const buffer = await file.arrayBuffer() + + await brain.import(Buffer.from(buffer), { + onProgress: (p) => { + setProgress({ + stage: p.stage, + message: p.message, + // Estimate percentage from stage + percent: { + detecting: 10, + extracting: 50, + 'storing-vfs': 80, + 'storing-graph': 90, + complete: 100 + }[p.stage] || 0, + entities: p.entities || 0, + relationships: p.relationships || 0 + }) + } + }) + } + + return ( +
+ +

{progress.message}

+

Entities: {progress.entities} | Relationships: {progress.relationships}

+
+ ) +} +``` + +--- + +### CLI Progress Spinner + +```typescript +import ora from 'ora' + +const spinner = ora('Starting import...').start() + +await brain.import(buffer, { + onProgress: (progress) => { + spinner.text = progress.message + + if (progress.stage === 'complete') { + spinner.succeed(`Import complete: ${progress.entities} entities`) + } + } +}) +``` + +**CLI Output:** +``` +โ ‹ Detecting format... +โ ™ Loading Excel workbook... +โ น Reading sheet: Sales (1/3) +โ ธ Parsing Excel (33%) +โ ผ Reading sheet: Products (2/3) +... +โœ” Import complete: 1523 entities +``` + +--- + +### Progress Dashboard with ETA + +```typescript +let startTime = Date.now() +let lastUpdate = startTime + +await brain.import(buffer, { + onProgress: (progress) => { + const elapsed = Date.now() - startTime + const rate = progress.entities / (elapsed / 1000) // entities/sec + + console.clear() + console.log('Import Progress Dashboard') + console.log('========================') + console.log(`Stage: ${progress.stage}`) + console.log(`Status: ${progress.message}`) + console.log(`Entities: ${progress.entities}`) + console.log(`Relationships: ${progress.relationships}`) + console.log(`Rate: ${rate.toFixed(1)} entities/sec`) + console.log(`Elapsed: ${(elapsed / 1000).toFixed(1)}s`) + } +}) +``` + +--- + +## ๐Ÿ”ง Advanced: Format-Specific Optimization + +### Detecting Format to Show Appropriate Progress + +```typescript +const formatMessages = { + csv: (p) => `CSV: ${p.message}`, + pdf: (p) => `PDF: ${p.message}`, + excel: (p) => `Excel: ${p.message}`, + json: (p) => `JSON: ${p.processed} nodes, ${p.entities} entities`, + markdown: (p) => `Markdown: Section ${p.processed}/${p.total}`, + yaml: (p) => `YAML: ${p.processed} nodes`, + docx: (p) => `DOCX: ${p.processed} paragraphs` +} + +await brain.import(buffer, { + onProgress: (progress) => { + // Format is available in progress.stage metadata + const message = formatMessages[detectedFormat]?.(progress) || progress.message + console.log(message) + } +}) +``` + +--- + +## โšก Performance Tips + +### Throttle UI Updates + +```typescript +let lastUIUpdate = 0 +const THROTTLE_MS = 100 // Update UI max once per 100ms + +await brain.import(buffer, { + onProgress: (progress) => { + const now = Date.now() + if (now - lastUIUpdate < THROTTLE_MS && progress.stage !== 'complete') { + return // Skip this update + } + + lastUIUpdate = now + updateUI(progress) // Only update every 100ms + } +}) +``` + +**Note:** Brainy already throttles progress callbacks internally, but additional UI throttling can help with heavy rendering. + +--- + +## ๐Ÿ“ Summary + +โœ… **All 7 formats** have consistent progress reporting +โœ… **Real-time updates** during long imports (no more "0%" hangs) +โœ… **Contextual messages** show exactly what's happening +โœ… **Build reliable tools** with standardized progress callbacks +โœ… **Workshop team problem SOLVED** - users see progress throughout import + +**Files Modified:** +- 3 handlers: `csvHandler.ts`, `pdfHandler.ts`, `excelHandler.ts` +- 7 importers: `SmartCSVImporter.ts`, `SmartPDFImporter.ts`, `SmartExcelImporter.ts`, `SmartJSONImporter.ts`, `SmartMarkdownImporter.ts`, `SmartYAMLImporter.ts`, `SmartDOCXImporter.ts` + +**Result:** Comprehensive, consistent progress tracking across ALL import formats! diff --git a/docs/guides/import-progress-implementation.md b/docs/guides/import-progress-implementation.md new file mode 100644 index 00000000..2b6ac5b7 --- /dev/null +++ b/docs/guides/import-progress-implementation.md @@ -0,0 +1,734 @@ +# Import Progress Implementation Guide +**For Developers: How to Add Progress Tracking to ANY File Handler** + +> This guide shows the **standard pattern** for implementing rich progress tracking in Brainy import handlers. Follow this template for **all 7 supported formats** (CSV, PDF, Excel, JSON, Markdown, YAML, DOCX) or any future file format. + +--- + +## ๐Ÿ“Š Supported Formats & Consistent Progress Reporting + +> **โš ๏ธ IMPORTANT FOR DEVELOPERS:** The public API (`ImportProgress`) is 100% standardized across all formats. You can build ONE progress handler that works for CSV, PDF, Excel, JSON, Markdown, YAML, and DOCX with **zero format-specific code**. See [Standard Import Progress API](./standard-import-progress.md) for details. + +**ALL 7 formats now have consistent, standardized progress reporting** for building reliable import tools: + +| Format | Category | Progress Points | File Location | Status | +|--------|----------|-----------------|---------------|--------| +| **CSV** | Tabular | Parsing โ†’ Row extraction โ†’ Type conversion โ†’ Complete | `handlers/csvHandler.ts` + `SmartCSVImporter.ts` | โœ… Complete | +| **PDF** | Document | Loading โ†’ Page-by-page โ†’ Item extraction โ†’ Complete | `handlers/pdfHandler.ts` + `SmartPDFImporter.ts` | โœ… Complete | +| **Excel** | Tabular | Loading โ†’ Sheet-by-sheet โ†’ Row extraction โ†’ Type conversion โ†’ Complete | `handlers/excelHandler.ts` + `SmartExcelImporter.ts` | โœ… Complete | +| **JSON** | Structured | Parsing โ†’ Node traversal (every 10 nodes) โ†’ Complete | `SmartJSONImporter.ts` | โœ… Complete | +| **Markdown** | Document | Parsing โ†’ Section-by-section โ†’ Complete | `SmartMarkdownImporter.ts` | โœ… Complete | +| **YAML** | Structured | Parsing โ†’ Node traversal (every 10 nodes) โ†’ Complete | `SmartYAMLImporter.ts` | โœ… Complete | +| **DOCX** | Document | Parsing โ†’ Paragraph-by-paragraph (every 10) โ†’ Complete | `SmartDOCXImporter.ts` | โœ… Complete | + +### The Standard Public API + +**Developers calling `brain.import()` see ONE standardized interface** regardless of format: + +```typescript +// THE PUBLIC API - Same for ALL 7 formats! +brain.import(buffer, { + onProgress: (progress: ImportProgress) => { + // These fields work for CSV, PDF, Excel, JSON, Markdown, YAML, DOCX + progress.stage // 'detecting' | 'extracting' | 'storing-vfs' | 'storing-graph' | 'complete' + progress.message // Human-readable status (varies by format, always readable) + progress.processed // Items processed (optional) + progress.total // Total items (optional) + progress.entities // Entities extracted (optional) + progress.relationships // Relationships inferred (optional) + progress.throughput // Items/sec (optional, during extraction) + progress.eta // Time remaining in ms (optional) + } +}) +``` + +**Internal Implementation** (for developers adding new format handlers): + +The table below shows how formats implement progress *internally*. Normal developers don't need to know this - they just use the standard `ImportProgress` interface above! + +```typescript +// Internal: Binary formats use handler hooks (you added these!) +interface FormatHandlerProgressHooks { + onBytesProcessed?: (bytes: number) => void + onCurrentItem?: (message: string) => void + onDataExtracted?: (count: number, total?: number) => void +} + +// Internal: Text formats use importer callbacks +interface ImporterProgressCallback { + onProgress?: (stats: { processed, total, entities, relationships }) => void +} + +// Both are converted to ImportProgress by ImportCoordinator! +``` + +### Developer Benefits + +โœ… **Consistent API** - Same pattern across all 7 formats +โœ… **Throttled Updates** - Progress reported every 10-1000 items (no spam) +โœ… **Contextual Messages** - "Processing page 5 of 23", "Reading sheet: Sales (2/5)" +โœ… **Real-time Estimates** - Users see progress during long imports +โœ… **Build Monitoring Tools** - Reliable progress data for UIs, dashboards, CLI tools + +--- + +## ๐ŸŽฏ Overview + +As of v4.5.0, Brainy supports comprehensive, multi-dimensional progress tracking for imports: +- **Bytes processed** (always available, most deterministic) +- **Entities extracted** (AI extraction phase) +- **Stage-specific metrics** (parsing: MB/s, extraction: entities/s) +- **Time estimates** (remaining time, total time) +- **Context information** ("Processing page 5 of 23") + +All handlers follow a simple, consistent pattern using **progress hooks**. + +--- + +## ๐Ÿ“‹ The Progress Hooks Pattern + +### 1. Progress Hooks Interface + +```typescript +export interface FormatHandlerProgressHooks { + /** + * Report bytes processed + * Call this as you read/parse the file + */ + onBytesProcessed?: (bytes: number) => void + + /** + * Set current processing context + * Examples: "Processing page 5", "Reading sheet: Q2 Sales" + */ + onCurrentItem?: (item: string) => void + + /** + * Report structured data extraction progress + * Examples: "Extracted 100 rows", "Parsed 50 paragraphs" + */ + onDataExtracted?: (count: number, total?: number) => void +} +``` + +### 2. Handler Options (Automatic) + +Progress hooks are automatically passed to your handler via `FormatHandlerOptions`: + +```typescript +export interface FormatHandlerOptions { + // ... existing options ... + + /** + * Progress hooks (v4.5.0) + * Handlers call these to report progress during processing + */ + progressHooks?: FormatHandlerProgressHooks + + /** + * Total file size in bytes (v4.5.0) + * Used for progress percentage calculation + */ + totalBytes?: number +} +``` + +**You don't need to modify FormatHandlerOptions** - it's already done! + +### 3. Standard Implementation Pattern + +Every handler follows these 5 steps: + +```typescript +async process(data: Buffer | string, options: FormatHandlerOptions): Promise { + const progressHooks = options.progressHooks // Step 1: Get hooks + + // Step 2: Report initial progress + if (progressHooks?.onCurrentItem) { + progressHooks.onCurrentItem('Starting import...') + } + + // Step 3: Report bytes as you process + const buffer = Buffer.isBuffer(data) ? data : Buffer.from(data) + if (progressHooks?.onBytesProcessed) { + progressHooks.onBytesProcessed(0) // Start + } + + // ... do parsing ... + + if (progressHooks?.onBytesProcessed) { + progressHooks.onBytesProcessed(buffer.length) // Complete + } + + // Step 4: Report data extraction + if (progressHooks?.onDataExtracted) { + progressHooks.onDataExtracted(data.length, data.length) + } + + // Step 5: Report completion + if (progressHooks?.onCurrentItem) { + progressHooks.onCurrentItem(`Complete: ${data.length} items processed`) + } + + return { format, data, metadata } +} +``` + +--- + +## ๐Ÿ“š Complete Example: CSV Handler + +Here's the **ACTUAL implementation** from CSV handler showing all the key progress points: + +```typescript +async process(data: Buffer | string, options: FormatHandlerOptions): Promise { + const startTime = Date.now() + const progressHooks = options.progressHooks // โœ… Step 1 + + // Convert to buffer if string + const buffer = Buffer.isBuffer(data) ? data : Buffer.from(data, 'utf-8') + const totalBytes = buffer.length + + // โœ… Step 2: Report start + if (progressHooks?.onBytesProcessed) { + progressHooks.onBytesProcessed(0) + } + if (progressHooks?.onCurrentItem) { + progressHooks.onCurrentItem('Detecting CSV encoding and delimiter...') + } + + // Detect encoding + const detectedEncoding = options.encoding || this.detectEncodingSafe(buffer) + const text = buffer.toString(detectedEncoding as BufferEncoding) + + // Detect delimiter + const delimiter = options.csvDelimiter || this.detectDelimiter(text) + + // โœ… Progress update: Parsing phase + if (progressHooks?.onCurrentItem) { + progressHooks.onCurrentItem(`Parsing CSV rows (delimiter: "${delimiter}")...`) + } + + // Parse CSV + const records = parse(text, { /* options */ }) + + // โœ… Step 3: Report bytes processed (entire file parsed) + if (progressHooks?.onBytesProcessed) { + progressHooks.onBytesProcessed(totalBytes) + } + + const data = Array.isArray(records) ? records : [records] + + // โœ… Step 4: Report data extraction + if (progressHooks?.onDataExtracted) { + progressHooks.onDataExtracted(data.length, data.length) + } + if (progressHooks?.onCurrentItem) { + progressHooks.onCurrentItem(`Extracted ${data.length} rows, inferring types...`) + } + + // Type inference and conversion + const fields = data.length > 0 ? Object.keys(data[0]) : [] + const types = this.inferFieldTypes(data) + + const convertedData = data.map((row, index) => { + const converted = this.convertRow(row, types) + + // โœ… Progress update every 1000 rows (avoid spam) + if (progressHooks?.onCurrentItem && index > 0 && index % 1000 === 0) { + progressHooks.onCurrentItem(`Converting types: ${index}/${data.length} rows...`) + } + + return converted + }) + + // โœ… Step 5: Report completion + if (progressHooks?.onCurrentItem) { + progressHooks.onCurrentItem(`CSV processing complete: ${convertedData.length} rows`) + } + + return { + format: this.format, + data: convertedData, + metadata: { /* ... */ } + } +} +``` + +### Key Progress Points in CSV Handler + +| Progress Point | Hook Used | Message Example | +|----------------|-----------|-----------------| +| **Start** | `onCurrentItem` | "Detecting CSV encoding and delimiter..." | +| **Start bytes** | `onBytesProcessed(0)` | 0 bytes | +| **Parsing** | `onCurrentItem` | "Parsing CSV rows (delimiter: \",\")..." | +| **Bytes complete** | `onBytesProcessed(totalBytes)` | All bytes read | +| **Data extracted** | `onDataExtracted(count, total)` | Number of rows extracted | +| **Type conversion** | `onCurrentItem` (every 1000 rows) | "Converting types: 5000/10000 rows..." | +| **Complete** | `onCurrentItem` | "CSV processing complete: 10000 rows" | + +--- + +## ๐Ÿ“– Implementation Guide by File Type + +### Supported Formats + +Brainy supports **7 file formats** with full progress tracking: + +**Binary Formats** (use handlers): +1. **CSV** - Row-by-row parsing with type inference +2. **PDF** - Page-by-page extraction with table detection +3. **Excel** - Sheet-by-sheet processing with formula evaluation + +**Text/Structured Formats** (parse inline): +4. **JSON** - Recursive traversal of nested structures +5. **Markdown** - Section-by-section with heading extraction +6. **YAML** - Hierarchical traversal with relationship inference +7. **DOCX** - Paragraph-by-paragraph with structure analysis + +--- + +### PDF Handler (Multi-Page) + +```typescript +async process(data: Buffer, options: FormatHandlerOptions): Promise { + const progressHooks = options.progressHooks + const totalBytes = data.length + + // Report start + if (progressHooks?.onCurrentItem) { + progressHooks.onCurrentItem('Loading PDF document...') + } + + const pdfDoc = await loadPDF(data) + const totalPages = pdfDoc.numPages + + const extractedData: any[] = [] + let bytesProcessed = 0 + + for (let pageNum = 1; pageNum <= totalPages; pageNum++) { + // โœ… Report current page + if (progressHooks?.onCurrentItem) { + progressHooks.onCurrentItem(`Processing page ${pageNum} of ${totalPages}`) + } + + const page = await pdfDoc.getPage(pageNum) + const text = await page.getTextContent() + extractedData.push(this.processPageText(text)) + + // โœ… Estimate bytes processed (pages are sequential) + bytesProcessed = Math.floor((pageNum / totalPages) * totalBytes) + if (progressHooks?.onBytesProcessed) { + progressHooks.onBytesProcessed(bytesProcessed) + } + + // โœ… Report extraction progress + if (progressHooks?.onDataExtracted) { + progressHooks.onDataExtracted(pageNum, totalPages) + } + } + + // Final progress + if (progressHooks?.onBytesProcessed) { + progressHooks.onBytesProcessed(totalBytes) + } + if (progressHooks?.onCurrentItem) { + progressHooks.onCurrentItem(`PDF complete: ${totalPages} pages processed`) + } + + return { format: 'pdf', data: extractedData, metadata: { /* ... */ } } +} +``` + +### Excel Handler (Multi-Sheet) + +```typescript +async process(data: Buffer, options: FormatHandlerOptions): Promise { + const progressHooks = options.progressHooks + const totalBytes = data.length + + // Load workbook + if (progressHooks?.onCurrentItem) { + progressHooks.onCurrentItem('Loading Excel workbook...') + } + + const workbook = XLSX.read(data) + const sheetNames = options.excelSheets === 'all' + ? workbook.SheetNames + : (options.excelSheets || [workbook.SheetNames[0]]) + + const allData: any[] = [] + let bytesProcessed = 0 + + for (let i = 0; i < sheetNames.length; i++) { + const sheetName = sheetNames[i] + + // โœ… Report current sheet + if (progressHooks?.onCurrentItem) { + progressHooks.onCurrentItem(`Reading sheet: ${sheetName} (${i + 1}/${sheetNames.length})`) + } + + const sheet = workbook.Sheets[sheetName] + const sheetData = XLSX.utils.sheet_to_json(sheet) + allData.push(...sheetData) + + // โœ… Estimate bytes processed (sheets processed sequentially) + bytesProcessed = Math.floor(((i + 1) / sheetNames.length) * totalBytes) + if (progressHooks?.onBytesProcessed) { + progressHooks.onBytesProcessed(bytesProcessed) + } + + // โœ… Report data extraction + if (progressHooks?.onDataExtracted) { + progressHooks.onDataExtracted(allData.length, undefined) // Total unknown until done + } + } + + // Final progress + if (progressHooks?.onBytesProcessed) { + progressHooks.onBytesProcessed(totalBytes) + } + if (progressHooks?.onCurrentItem) { + progressHooks.onCurrentItem(`Excel complete: ${sheetNames.length} sheets, ${allData.length} rows`) + } + + return { format: 'xlsx', data: allData, metadata: { /* ... */ } } +} +``` + +### JSON Importer (Recursive Traversal) + +```typescript +async extract(data: any, options: SmartJSONOptions = {}): Promise { + // โœ… Report parsing start + options.onProgress?.({ processed: 0, entities: 0, relationships: 0 }) + + // Parse JSON if string + let jsonData = typeof data === 'string' ? JSON.parse(data) : data + + // โœ… Report parsing complete + options.onProgress?.({ processed: 0, entities: 0, relationships: 0 }) + + // Traverse and extract (reports progress every 10 nodes) + const entities: ExtractedJSONEntity[] = [] + const relationships: ExtractedJSONRelationship[] = [] + let nodesProcessed = 0 + + await this.traverseJSON( + jsonData, + entities, + relationships, + () => { + nodesProcessed++ + if (nodesProcessed % 10 === 0) { + options.onProgress?.({ + processed: nodesProcessed, + entities: entities.length, + relationships: relationships.length + }) + } + } + ) + + // โœ… Report completion + options.onProgress?.({ + processed: nodesProcessed, + entities: entities.length, + relationships: relationships.length + }) + + return { nodesProcessed, entitiesExtracted: entities.length, ... } +} +``` + +### Markdown Importer (Section-Based) + +```typescript +async extract(markdown: string, options: SmartMarkdownOptions = {}): Promise { + // โœ… Report parsing start + options.onProgress?.({ processed: 0, total: 0, entities: 0, relationships: 0 }) + + // Parse markdown into sections + const parsedSections = this.parseMarkdown(markdown, options) + + // โœ… Report parsing complete + options.onProgress?.({ processed: 0, total: parsedSections.length, entities: 0, relationships: 0 }) + + // Process each section (reports progress after each section) + const sections: MarkdownSection[] = [] + for (let i = 0; i < parsedSections.length; i++) { + const section = await this.processSection(parsedSections[i], options) + sections.push(section) + + options.onProgress?.({ + processed: i + 1, + total: parsedSections.length, + entities: sections.reduce((sum, s) => sum + s.entities.length, 0), + relationships: sections.reduce((sum, s) => sum + s.relationships.length, 0) + }) + } + + // โœ… Report completion + options.onProgress?.({ + processed: sections.length, + total: sections.length, + entities: sections.reduce((sum, s) => sum + s.entities.length, 0), + relationships: sections.reduce((sum, s) => sum + s.relationships.length, 0) + }) + + return { sectionsProcessed: sections.length, ... } +} +``` + +### YAML Importer (Hierarchical) + +```typescript +async extract(yamlContent: string | Buffer, options: SmartYAMLOptions = {}): Promise { + // โœ… Report parsing start + options.onProgress?.({ processed: 0, entities: 0, relationships: 0 }) + + // Parse YAML + const yamlString = typeof yamlContent === 'string' ? yamlContent : yamlContent.toString('utf-8') + const data = yaml.load(yamlString) + + // โœ… Report parsing complete + options.onProgress?.({ processed: 0, entities: 0, relationships: 0 }) + + // Traverse YAML structure (reports progress every 10 nodes) + // ... similar to JSON traversal ... + + // โœ… Report completion (already implemented) + options.onProgress?.({ + processed: nodesProcessed, + entities: entities.length, + relationships: relationships.length + }) + + return { nodesProcessed, entitiesExtracted: entities.length, ... } +} +``` + +### DOCX Importer (Paragraph-Based) + +```typescript +async extract(buffer: Buffer, options: SmartDOCXOptions = {}): Promise { + // โœ… Report parsing start + options.onProgress?.({ processed: 0, entities: 0, relationships: 0 }) + + // Extract text and HTML using Mammoth + const textResult = await mammoth.extractRawText({ buffer }) + const htmlResult = await mammoth.convertToHtml({ buffer }) + + // โœ… Report parsing complete + options.onProgress?.({ processed: 0, entities: 0, relationships: 0 }) + + // Process paragraphs (reports progress every 10 paragraphs) + const paragraphs = textResult.value.split(/\n\n+/).filter(p => p.trim().length >= minLength) + + for (let i = 0; i < paragraphs.length; i++) { + await this.processParagraph(paragraphs[i]) + + if (i % 10 === 0) { + options.onProgress?.({ + processed: i + 1, + entities: entities.length, + relationships: relationships.length + }) + } + } + + // โœ… Report completion (already implemented) + options.onProgress?.({ + processed: paragraphs.length, + entities: entities.length, + relationships: relationships.length + }) + + return { paragraphsProcessed: paragraphs.length, ... } +} +``` + +--- + +## ๐ŸŽฏ Best Practices + +### 1. Always Check if Hooks Exist + +Progress hooks are **optional**. Always check before calling: + +```typescript +// โœ… Good - safe +if (progressHooks?.onBytesProcessed) { + progressHooks.onBytesProcessed(bytes) +} + +// โŒ Bad - will crash if hooks undefined +progressHooks.onBytesProcessed(bytes) // TypeError! +``` + +### 2. Report Bytes at Start and End + +```typescript +// โœ… Good - clear start and end +progressHooks?.onBytesProcessed(0) // Start +// ... processing ... +progressHooks?.onBytesProcessed(totalBytes) // End + +// โŒ Bad - no clear boundaries +// ... just start processing without reporting start +``` + +### 3. Throttle Frequent Updates + +```typescript +// โœ… Good - report every 1000 items +for (let i = 0; i < items.length; i++) { + processItem(items[i]) + + if (i > 0 && i % 1000 === 0) { + progressHooks?.onCurrentItem(`Processing: ${i}/${items.length}`) + } +} + +// โŒ Bad - report EVERY item (spam!) +for (let i = 0; i < items.length; i++) { + processItem(items[i]) + progressHooks?.onCurrentItem(`Processing: ${i}/${items.length}`) // 1M callbacks! +} +``` + +### 4. Provide Contextual Messages + +```typescript +// โœ… Good - specific and helpful +progressHooks?.onCurrentItem('Parsing CSV rows (delimiter: ",")') +progressHooks?.onCurrentItem('Processing page 5 of 23') +progressHooks?.onCurrentItem('Reading sheet: Q2 Sales Data') + +// โŒ Bad - vague +progressHooks?.onCurrentItem('Processing...') +progressHooks?.onCurrentItem('Working...') +``` + +### 5. Report Data Extraction with Totals (if known) + +```typescript +// โœ… Good - total known +progressHooks?.onDataExtracted(100, 1000) // 100 of 1000 rows + +// โœ… Also good - total unknown (streaming) +progressHooks?.onDataExtracted(100, undefined) // 100 rows so far + +// โœ… Also good - complete +progressHooks?.onDataExtracted(1000, 1000) // All 1000 rows +``` + +--- + +## ๐Ÿ”ง Testing Your Handler + +### Manual Test + +```typescript +import { CSVHandler } from './csvHandler.js' +import * as fs from 'fs' + +const handler = new CSVHandler() +const data = fs.readFileSync('./test.csv') + +const result = await handler.process(data, { + filename: 'test.csv', + progressHooks: { + onBytesProcessed: (bytes) => { + console.log(`Bytes: ${bytes}`) + }, + onCurrentItem: (item) => { + console.log(`Status: ${item}`) + }, + onDataExtracted: (count, total) => { + console.log(`Extracted: ${count}${total ? `/${total}` : ''}`) + } + } +}) + +console.log(`Complete: ${result.data.length} rows`) +``` + +### Expected Output + +``` +Status: Detecting CSV encoding and delimiter... +Bytes: 0 +Status: Parsing CSV rows (delimiter: ",")... +Bytes: 52438 +Extracted: 1000/1000 +Status: Extracted 1000 rows, inferring types... +Status: CSV processing complete: 1000 rows +Complete: 1000 rows +``` + +--- + +## ๐Ÿ“Š Progress Flow Diagram + +``` +User Imports File + โ†“ +ImportManager + โ†“ +Creates ProgressTracker + โ†“ +Calls Handler.process() with progressHooks + โ†“ +Handler Reports Progress: + โ”œโ”€ onBytesProcessed(0) โ†’ ProgressTracker โ†’ overall_progress calculated + โ”œโ”€ onCurrentItem("Parsing...") โ†’ ProgressTracker โ†’ stage_message updated + โ”œโ”€ onBytesProcessed(bytes) โ†’ ProgressTracker โ†’ bytes_per_second calculated + โ”œโ”€ onDataExtracted(count) โ†’ ProgressTracker โ†’ entities_extracted updated + โ””โ”€ onCurrentItem("Complete") โ†’ ProgressTracker โ†’ final progress + โ†“ +ProgressTracker emits to callback (throttled 100ms) + โ†“ +User sees: + "Overall: 45% | PARSING | 12.5 MB/s | Parsing CSV rows..." +``` + +--- + +## โœ… Checklist for New Handlers + +When implementing a new file format handler: + +- [ ] Get `progressHooks` from `options` +- [ ] Get `totalBytes` (if available) +- [ ] Report `onBytesProcessed(0)` at start +- [ ] Report `onCurrentItem()` for key stages +- [ ] Report `onBytesProcessed()` as you process +- [ ] Report `onDataExtracted()` when you extract data +- [ ] Throttle frequent updates (every 1000 items max) +- [ ] Report `onBytesProcessed(totalBytes)` at end +- [ ] Report final `onCurrentItem()` with summary +- [ ] Test with progress callback to verify output + +--- + +## ๐ŸŽ“ Summary + +**The Pattern (5 Steps)**: +1. Get `progressHooks` from options +2. Report start (`onBytesProcessed(0)`, `onCurrentItem("Starting...")`) +3. Report progress as you process (`onBytesProcessed(bytes)`, `onCurrentItem("Page 5...")`) +4. Report data extraction (`onDataExtracted(count, total)`) +5. Report completion (`onBytesProcessed(totalBytes)`, `onCurrentItem("Complete")`) + +**Always Check**: `progressHooks?.method()` + +**Throttle**: Report every N items, not every single item + +**Context**: Provide specific, helpful messages + +**Testing**: Use manual test with console.log callbacks + +--- + +**This pattern makes it trivial to add progress tracking to ANY file format. Copy this template and adapt for your handler!** diff --git a/docs/guides/standard-import-progress.md b/docs/guides/standard-import-progress.md new file mode 100644 index 00000000..d57f8803 --- /dev/null +++ b/docs/guides/standard-import-progress.md @@ -0,0 +1,453 @@ +# Standard Import Progress API + +## โœ… Build Once, Works for ALL Formats + +**Brainy provides a 100% standardized progress API** - write your UI/tool once, and it works for all 7 supported formats (CSV, PDF, Excel, JSON, Markdown, YAML, DOCX) with **zero format-specific code**. + +--- + +## ๐ŸŽฏ The Standard Interface + +### One Interface for Everything + +```typescript +import { Brainy } from '@soulcraft/brainy' + +const brain = await Brainy.create() + +// THIS CODE WORKS FOR ALL 7 FORMATS - NO FORMAT-SPECIFIC LOGIC NEEDED! +await brain.import(anyBuffer, { + onProgress: (progress) => { + // Standard fields - ALWAYS available regardless of format + console.log(progress.stage) // Current stage + console.log(progress.message) // Human-readable status + + // Optional fields - available when relevant + console.log(progress.processed) // Items processed so far + console.log(progress.total) // Total items (if known) + console.log(progress.entities) // Entities extracted + console.log(progress.relationships) // Relationships inferred + console.log(progress.throughput) // Items/sec (during extraction) + console.log(progress.eta) // Time remaining in ms + } +}) +``` + +### The Complete Interface + +```typescript +interface ImportProgress { + // === ALWAYS PRESENT === + + /** High-level stage (5 stages for all formats) */ + stage: 'detecting' | 'extracting' | 'storing-vfs' | 'storing-graph' | 'complete' + + /** Human-readable status message */ + message: string + + // === AVAILABLE WHEN RELEVANT === + + /** Items processed (rows, pages, nodes, etc.) */ + processed?: number + + /** Total items to process (if known ahead of time) */ + total?: number + + /** Entities extracted so far */ + entities?: number + + /** Relationships inferred so far */ + relationships?: number + + /** Processing rate (items per second) */ + throughput?: number + + /** Estimated time remaining (milliseconds) */ + eta?: number + + /** Whether data is queryable at this point (v4.2.0+) */ + queryable?: boolean +} +``` + +--- + +## ๐ŸŽจ Generic UI Components + +### React Progress Component (Works for ALL Formats) + +```typescript +import { useState } from 'react' +import { Brainy } from '@soulcraft/brainy' + +function UniversalImportProgress({ file }: { file: File }) { + const [progress, setProgress] = useState({ + stage: 'idle', + message: 'Ready to import', + percent: 0, + entities: 0, + relationships: 0 + }) + + const handleImport = async () => { + const buffer = await file.arrayBuffer() + const brain = await Brainy.create() + + await brain.import(Buffer.from(buffer), { + // THIS WORKS FOR CSV, PDF, EXCEL, JSON, MARKDOWN, YAML, DOCX! + onProgress: (p) => { + setProgress({ + stage: p.stage, + message: p.message, + + // Calculate percentage from stage + processed/total + percent: calculatePercent(p), + + entities: p.entities || 0, + relationships: p.relationships || 0 + }) + } + }) + } + + // Helper: Calculate percentage from progress + function calculatePercent(p: ImportProgress): number { + // Use processed/total if available + if (p.processed && p.total) { + return Math.round((p.processed / p.total) * 100) + } + + // Otherwise estimate from stage + const stagePercents = { + detecting: 5, + extracting: 50, + 'storing-vfs': 80, + 'storing-graph': 90, + complete: 100 + } + return stagePercents[p.stage] || 0 + } + + return ( +
+ {/* Stage Indicator */} +
+ {['detecting', 'extracting', 'storing-vfs', 'storing-graph', 'complete'].map(s => ( + + {s} + + ))} +
+ + {/* Progress Bar */} +
+
+
+ + {/* Status Message (format-specific but always readable) */} +

{progress.message}

+ + {/* Counts */} +
+ Entities: {progress.entities} + Relationships: {progress.relationships} +
+
+ ) +} +``` + +**This component works perfectly for:** +- โœ… CSV files with 10,000 rows +- โœ… PDF documents with 200 pages +- โœ… Excel workbooks with 5 sheets +- โœ… JSON files with nested structures +- โœ… Markdown documents with sections +- โœ… YAML configuration files +- โœ… DOCX documents with paragraphs + +**No format detection needed. No format-specific rendering. Just works.** + +--- + +### CLI Progress Indicator (Works for ALL Formats) + +```typescript +import ora from 'ora' +import { Brainy } from '@soulcraft/brainy' + +async function importWithProgress(filePath: string) { + const spinner = ora('Starting import...').start() + const brain = await Brainy.create() + + try { + await brain.import(filePath, { + // THIS WORKS FOR ALL 7 FORMATS! + onProgress: (p) => { + // Update spinner text with current message + spinner.text = p.message + + // Add counts if available + if (p.entities || p.relationships) { + spinner.text += ` (${p.entities || 0} entities, ${p.relationships || 0} relationships)` + } + + // Add throughput/ETA if available (during extraction) + if (p.throughput && p.eta) { + const etaSec = Math.round(p.eta / 1000) + spinner.text += ` [${p.throughput.toFixed(1)}/sec, ETA: ${etaSec}s]` + } + + // Change spinner when complete + if (p.stage === 'complete') { + spinner.succeed(p.message) + } + } + }) + } catch (error) { + spinner.fail(`Import failed: ${error.message}`) + } +} + +// Works for ANY format! +await importWithProgress('data.csv') +await importWithProgress('document.pdf') +await importWithProgress('workbook.xlsx') +await importWithProgress('config.yaml') +``` + +**CLI Output (same code, different formats):** + +```bash +# CSV Import +โ ‹ Detecting format... +โ ™ Parsing CSV rows (delimiter: ",") +โ น Extracting entities from csv (45 rows/sec, ETA: 120s) (150 entities, 45 relationships) +โ ธ Extracting entities from csv (45 rows/sec, ETA: 60s) (750 entities, 223 relationships) +โœ” Import complete (1350 entities, 401 relationships) + +# PDF Import +โ ‹ Detecting format... +โ ™ Loading PDF document... +โ น Processing page 5 of 23 +โ ธ Extracting entities from pdf (2.5 pages/sec, ETA: 30s) (45 entities, 12 relationships) +โœ” Import complete (156 entities, 89 relationships) + +# Excel Import +โ ‹ Detecting format... +โ ™ Loading Excel workbook... +โ น Reading sheet: Sales (2/5) +โ ธ Extracting entities from excel (120 rows/sec, ETA: 45s) (500 entities, 234 relationships) +โœ” Import complete (2340 entities, 892 relationships) +``` + +**Same code. Different formats. Perfect progress for all.** + +--- + +### Dashboard with Real-Time Stats (Works for ALL Formats) + +```typescript +function ImportDashboard() { + const [stats, setStats] = useState({ + stage: '', + message: '', + elapsed: 0, + entities: 0, + relationships: 0, + throughput: 0, + eta: 0 + }) + + const startTime = Date.now() + + const handleImport = async (file: File) => { + await brain.import(await file.arrayBuffer(), { + // UNIVERSAL PROGRESS HANDLER - WORKS FOR ALL FORMATS! + onProgress: (p) => { + setStats({ + stage: p.stage, + message: p.message, + elapsed: Date.now() - startTime, + entities: p.entities || 0, + relationships: p.relationships || 0, + throughput: p.throughput || 0, + eta: p.eta || 0 + }) + } + }) + } + + return ( +
+

Import Progress

+ +
+ + {stats.stage} +
+ +
+ + {stats.message} +
+ +
+ + {(stats.elapsed / 1000).toFixed(1)}s +
+ +
+ + {stats.entities.toLocaleString()} +
+ +
+ + {stats.relationships.toLocaleString()} +
+ + {stats.throughput > 0 && ( +
+ + {stats.throughput.toFixed(1)} items/sec +
+ )} + + {stats.eta > 0 && ( +
+ + {(stats.eta / 1000).toFixed(0)}s +
+ )} +
+ ) +} +``` + +**This dashboard shows live stats for ANY format** - CSV, PDF, Excel, JSON, Markdown, YAML, DOCX. + +--- + +## ๐Ÿ“Š What Messages Look Like (Format-Specific Text, Standard Fields) + +While the **fields are standardized**, the **message text** varies by format to be most helpful: + +```typescript +// CSV Import Messages +"Detecting format..." +"Parsing CSV rows (delimiter: ",")" +"Extracted 1000 rows, inferring types..." +"Extracting entities from csv (45 rows/sec, ETA: 120s)..." +"Creating VFS structure..." +"Import complete" + +// PDF Import Messages +"Detecting format..." +"Loading PDF document..." +"Processing page 5 of 23" +"Extracting entities from pdf (2.5 pages/sec, ETA: 30s)..." +"Creating VFS structure..." +"Import complete" + +// Excel Import Messages +"Detecting format..." +"Loading Excel workbook..." +"Reading sheet: Sales (2/5)" +"Extracting entities from excel (120 rows/sec, ETA: 45s)..." +"Creating VFS structure..." +"Import complete" +``` + +**Key Point:** You can display `progress.message` directly in your UI **without parsing it**. It's always human-readable and contextually appropriate. + +--- + +## โœ… The 5 Standard Stages (Same for ALL Formats) + +Every import goes through these 5 stages in order: + +| Stage | Duration | Description | Fields Available | +|-------|----------|-------------|------------------| +| **detecting** | ~1% | Format detection | `stage`, `message` | +| **extracting** | ~70% | Parse file + AI extraction | `stage`, `message`, `processed`, `total`, `entities`, `relationships`, `throughput`, `eta` | +| **storing-vfs** | ~5% | Create file structure | `stage`, `message` | +| **storing-graph** | ~20% | Create graph nodes | `stage`, `message`, `entities`, `relationships` | +| **complete** | ~1% | Finalize | `stage`, `message`, `entities`, `relationships` | + +**These 5 stages are the same whether you're importing:** +- A 10MB CSV file with 50,000 rows +- A 200-page PDF document +- A 5-sheet Excel workbook +- A nested JSON structure +- A Markdown document +- A YAML configuration +- A DOCX document + +--- + +## ๐ŸŽฏ Why This Matters + +### Build Tools That Work for Everything + +```typescript +// ONE progress handler for your entire application +function universalProgressHandler(progress: ImportProgress) { + // Update UI (works for all formats) + updateProgressBar(progress) + updateStatusText(progress.message) + updateCounts(progress.entities, progress.relationships) + + // Log to analytics (works for all formats) + analytics.track('import_progress', { + stage: progress.stage, + processed: progress.processed, + total: progress.total + }) + + // Send to monitoring (works for all formats) + monitoring.gauge('import.entities', progress.entities) + monitoring.gauge('import.throughput', progress.throughput) +} + +// Use it everywhere +await brain.import(csvFile, { onProgress: universalProgressHandler }) +await brain.import(pdfFile, { onProgress: universalProgressHandler }) +await brain.import(excelFile, { onProgress: universalProgressHandler }) +await brain.import(jsonFile, { onProgress: universalProgressHandler }) +``` + +### No Format Detection Needed + +```typescript +// โŒ DON'T DO THIS (format-specific handling) +if (format === 'csv') { + // CSV-specific progress code +} else if (format === 'pdf') { + // PDF-specific progress code +} else if (format === 'excel') { + // Excel-specific progress code +} + +// โœ… DO THIS (universal handling) +onProgress: (p) => { + // Works for ALL formats! + updateUI(p.stage, p.message, p.entities, p.relationships) +} +``` + +--- + +## ๐Ÿ“ Summary + +โœ… **100% Standardized** - Same `ImportProgress` interface for all 7 formats +โœ… **Build Once** - Your progress UI works for CSV, PDF, Excel, JSON, Markdown, YAML, DOCX +โœ… **No Format Detection** - No need to check file type in your progress handler +โœ… **Human-Readable Messages** - Display `progress.message` directly, no parsing needed +โœ… **Standard Fields** - `stage`, `processed`, `total`, `entities`, `relationships` work everywhere +โœ… **Optional Enhancements** - `throughput`, `eta` available during extraction (all formats) + +**The Workshop team (and any developer) can now build monitoring tools, dashboards, CLIs, and UIs that work perfectly for all import formats with zero format-specific code!** diff --git a/src/augmentations/intelligentImport/handlers/csvHandler.ts b/src/augmentations/intelligentImport/handlers/csvHandler.ts index 40a70d2d..87951ad4 100644 --- a/src/augmentations/intelligentImport/handlers/csvHandler.ts +++ b/src/augmentations/intelligentImport/handlers/csvHandler.ts @@ -34,9 +34,19 @@ export class CSVHandler extends BaseFormatHandler { async process(data: Buffer | string, options: FormatHandlerOptions): Promise { const startTime = Date.now() + const progressHooks = options.progressHooks // Convert to buffer if string const buffer = Buffer.isBuffer(data) ? data : Buffer.from(data, 'utf-8') + const totalBytes = buffer.length + + // v4.5.0: Report total bytes for progress tracking + if (progressHooks?.onBytesProcessed) { + progressHooks.onBytesProcessed(0) + } + if (progressHooks?.onCurrentItem) { + progressHooks.onCurrentItem('Detecting CSV encoding and delimiter...') + } // Detect encoding const detectedEncoding = options.encoding || this.detectEncodingSafe(buffer) @@ -45,6 +55,11 @@ export class CSVHandler extends BaseFormatHandler { // Detect delimiter if not specified const delimiter = options.csvDelimiter || this.detectDelimiter(text) + // v4.5.0: Report progress - parsing started + if (progressHooks?.onCurrentItem) { + progressHooks.onCurrentItem(`Parsing CSV rows (delimiter: "${delimiter}")...`) + } + // Parse CSV const hasHeaders = options.csvHeaders !== false const maxRows = options.maxRows @@ -60,23 +75,47 @@ export class CSVHandler extends BaseFormatHandler { cast: false // We'll do type inference ourselves }) + // v4.5.0: Report bytes processed (entire file parsed) + if (progressHooks?.onBytesProcessed) { + progressHooks.onBytesProcessed(totalBytes) + } + // Convert to array of objects const data = Array.isArray(records) ? records : [records] + // v4.5.0: Report data extraction progress + if (progressHooks?.onDataExtracted) { + progressHooks.onDataExtracted(data.length, data.length) + } + if (progressHooks?.onCurrentItem) { + progressHooks.onCurrentItem(`Extracted ${data.length} rows, inferring types...`) + } + // Infer types and convert values const fields = data.length > 0 ? Object.keys(data[0]) : [] const types = this.inferFieldTypes(data) - const convertedData = data.map(row => { + const convertedData = data.map((row, index) => { const converted: Record = {} for (const [key, value] of Object.entries(row)) { converted[key] = this.convertValue(value, types[key] || 'string') } + + // v4.5.0: Report progress every 1000 rows + if (progressHooks?.onCurrentItem && index > 0 && index % 1000 === 0) { + progressHooks.onCurrentItem(`Converting types: ${index}/${data.length} rows...`) + } + return converted }) const processingTime = Date.now() - startTime + // v4.5.0: Final progress update + if (progressHooks?.onCurrentItem) { + progressHooks.onCurrentItem(`CSV processing complete: ${convertedData.length} rows`) + } + return { format: this.format, data: convertedData, diff --git a/src/augmentations/intelligentImport/handlers/excelHandler.ts b/src/augmentations/intelligentImport/handlers/excelHandler.ts index 0b32609d..f5f7e314 100644 --- a/src/augmentations/intelligentImport/handlers/excelHandler.ts +++ b/src/augmentations/intelligentImport/handlers/excelHandler.ts @@ -21,9 +21,19 @@ export class ExcelHandler extends BaseFormatHandler { async process(data: Buffer | string, options: FormatHandlerOptions): Promise { const startTime = Date.now() + const progressHooks = options.progressHooks // Convert to buffer if string (though Excel should always be binary) const buffer = Buffer.isBuffer(data) ? data : Buffer.from(data, 'binary') + const totalBytes = buffer.length + + // v4.5.0: Report start + if (progressHooks?.onBytesProcessed) { + progressHooks.onBytesProcessed(0) + } + if (progressHooks?.onCurrentItem) { + progressHooks.onCurrentItem('Loading Excel workbook...') + } try { // Read workbook @@ -37,11 +47,25 @@ export class ExcelHandler extends BaseFormatHandler { // Determine which sheets to process const sheetsToProcess = this.getSheetsToProcess(workbook, options) + // v4.5.0: Report workbook loaded + if (progressHooks?.onCurrentItem) { + progressHooks.onCurrentItem(`Processing ${sheetsToProcess.length} sheets...`) + } + // Extract data from sheets const allData: Array> = [] const sheetMetadata: Record = {} - for (const sheetName of sheetsToProcess) { + for (let sheetIndex = 0; sheetIndex < sheetsToProcess.length; sheetIndex++) { + const sheetName = sheetsToProcess[sheetIndex] + + // v4.5.0: Report current sheet + if (progressHooks?.onCurrentItem) { + progressHooks.onCurrentItem( + `Reading sheet: ${sheetName} (${sheetIndex + 1}/${sheetsToProcess.length})` + ) + } + const sheet = workbook.Sheets[sheetName] if (!sheet) continue @@ -92,6 +116,25 @@ export class ExcelHandler extends BaseFormatHandler { columnCount: headers.length, headers } + + // v4.5.0: Estimate bytes processed (sheets are sequential) + const bytesProcessed = Math.floor(((sheetIndex + 1) / sheetsToProcess.length) * totalBytes) + if (progressHooks?.onBytesProcessed) { + progressHooks.onBytesProcessed(bytesProcessed) + } + + // v4.5.0: Report extraction progress + if (progressHooks?.onDataExtracted) { + progressHooks.onDataExtracted(allData.length, undefined) // Total unknown until complete + } + } + + // v4.5.0: Report data extraction complete + if (progressHooks?.onCurrentItem) { + progressHooks.onCurrentItem(`Extracted ${allData.length} rows, inferring types...`) + } + if (progressHooks?.onDataExtracted) { + progressHooks.onDataExtracted(allData.length, allData.length) } // Infer types (excluding _sheet field) @@ -99,7 +142,7 @@ export class ExcelHandler extends BaseFormatHandler { const types = this.inferFieldTypes(allData) // Convert values to appropriate types - const convertedData = allData.map(row => { + const convertedData = allData.map((row, index) => { const converted: Record = {} for (const [key, value] of Object.entries(row)) { if (key === '_sheet') { @@ -108,11 +151,29 @@ export class ExcelHandler extends BaseFormatHandler { converted[key] = this.convertValue(value, types[key] || 'string') } } + + // v4.5.0: Report progress every 1000 rows (avoid spam) + if (progressHooks?.onCurrentItem && index > 0 && index % 1000 === 0) { + progressHooks.onCurrentItem(`Converting types: ${index}/${allData.length} rows...`) + } + return converted }) + // v4.5.0: Final progress - all bytes processed + if (progressHooks?.onBytesProcessed) { + progressHooks.onBytesProcessed(totalBytes) + } + const processingTime = Date.now() - startTime + // v4.5.0: Report completion + if (progressHooks?.onCurrentItem) { + progressHooks.onCurrentItem( + `Excel complete: ${sheetsToProcess.length} sheets, ${convertedData.length} rows` + ) + } + return { format: this.format, data: convertedData, diff --git a/src/augmentations/intelligentImport/handlers/pdfHandler.ts b/src/augmentations/intelligentImport/handlers/pdfHandler.ts index ed2863a8..f8bc9838 100644 --- a/src/augmentations/intelligentImport/handlers/pdfHandler.ts +++ b/src/augmentations/intelligentImport/handlers/pdfHandler.ts @@ -46,9 +46,19 @@ export class PDFHandler extends BaseFormatHandler { async process(data: Buffer | string, options: FormatHandlerOptions): Promise { const startTime = Date.now() + const progressHooks = options.progressHooks // Convert to buffer const buffer = Buffer.isBuffer(data) ? data : Buffer.from(data, 'binary') + const totalBytes = buffer.length + + // v4.5.0: Report start + if (progressHooks?.onBytesProcessed) { + progressHooks.onBytesProcessed(0) + } + if (progressHooks?.onCurrentItem) { + progressHooks.onCurrentItem('Loading PDF document...') + } try { // Load PDF document @@ -64,12 +74,22 @@ export class PDFHandler extends BaseFormatHandler { const metadata = await pdfDoc.getMetadata() const numPages = pdfDoc.numPages + // v4.5.0: Report document loaded + if (progressHooks?.onCurrentItem) { + progressHooks.onCurrentItem(`Processing ${numPages} pages...`) + } + // Extract text and structure from all pages const allData: Array> = [] let totalTextLength = 0 let detectedTables = 0 for (let pageNum = 1; pageNum <= numPages; pageNum++) { + // v4.5.0: Report current page + if (progressHooks?.onCurrentItem) { + progressHooks.onCurrentItem(`Processing page ${pageNum} of ${numPages}`) + } + const page = await pdfDoc.getPage(pageNum) const textContent = await page.getTextContent() @@ -110,10 +130,36 @@ export class PDFHandler extends BaseFormatHandler { }) } } + + // v4.5.0: Estimate bytes processed (pages are sequential) + const bytesProcessed = Math.floor((pageNum / numPages) * totalBytes) + if (progressHooks?.onBytesProcessed) { + progressHooks.onBytesProcessed(bytesProcessed) + } + + // v4.5.0: Report extraction progress + if (progressHooks?.onDataExtracted) { + progressHooks.onDataExtracted(allData.length, undefined) // Total unknown until complete + } + } + + // v4.5.0: Final progress - all bytes processed + if (progressHooks?.onBytesProcessed) { + progressHooks.onBytesProcessed(totalBytes) + } + if (progressHooks?.onDataExtracted) { + progressHooks.onDataExtracted(allData.length, allData.length) } const processingTime = Date.now() - startTime + // v4.5.0: Report completion + if (progressHooks?.onCurrentItem) { + progressHooks.onCurrentItem( + `PDF complete: ${numPages} pages, ${allData.length} items extracted` + ) + } + // Get all unique fields (excluding metadata fields) const fields = allData.length > 0 ? Object.keys(allData[0]).filter(k => !k.startsWith('_')) diff --git a/src/augmentations/intelligentImport/types.ts b/src/augmentations/intelligentImport/types.ts index 054174b3..2a08ec24 100644 --- a/src/augmentations/intelligentImport/types.ts +++ b/src/augmentations/intelligentImport/types.ts @@ -3,6 +3,34 @@ * Handles Excel, PDF, and CSV import with intelligent extraction */ +import { ImportProgressTracker } from '../../utils/import-progress-tracker.js' + +/** + * Progress hooks for format handlers + * + * Handlers call these hooks to report progress during processing. + * This enables real-time progress tracking for any file format. + */ +export interface FormatHandlerProgressHooks { + /** + * Report bytes processed + * Call this as you read/parse the file + */ + onBytesProcessed?: (bytes: number) => void + + /** + * Set current processing context + * Examples: "Processing page 5", "Reading sheet: Q2 Sales" + */ + onCurrentItem?: (item: string) => void + + /** + * Report structured data extraction progress + * Examples: "Extracted 100 rows", "Parsed 50 paragraphs" + */ + onDataExtracted?: (count: number, total?: number) => void +} + export interface FormatHandler { /** * Format name (e.g., 'csv', 'xlsx', 'pdf') @@ -58,6 +86,18 @@ export interface FormatHandlerOptions { /** Whether to stream large files */ streaming?: boolean + + /** + * Progress hooks (v4.5.0) + * Handlers call these to report progress during processing + */ + progressHooks?: FormatHandlerProgressHooks + + /** + * Total file size in bytes (v4.5.0) + * Used for progress percentage calculation + */ + totalBytes?: number } export interface ProcessedData { diff --git a/src/brainy.ts b/src/brainy.ts index aa101335..ef2de0ae 100644 --- a/src/brainy.ts +++ b/src/brainy.ts @@ -2039,9 +2039,27 @@ export class Brainy implements BrainyInterface { * groupBy: 'type', // Organize by entity type * preserveSource: true, // Keep original file * - * // Progress tracking - * onProgress: (p) => console.log(p.message) + * // Progress tracking (v4.5.0 - STANDARDIZED FOR ALL 7 FORMATS!) + * onProgress: (p) => { + * console.log(`[${p.stage}] ${p.message}`) + * console.log(`Entities: ${p.entities || 0}, Rels: ${p.relationships || 0}`) + * if (p.throughput) console.log(`Rate: ${p.throughput.toFixed(1)}/sec`) + * } * }) + * // THIS SAME HANDLER WORKS FOR CSV, PDF, Excel, JSON, Markdown, YAML, DOCX! + * ``` + * + * @example Universal Progress Handler (v4.5.0) + * ```typescript + * // ONE handler for ALL 7 formats - no format-specific code needed! + * const universalProgress = (p) => { + * updateUI(p.stage, p.message, p.entities, p.relationships) + * } + * + * await brain.import(csvBuffer, { onProgress: universalProgress }) + * await brain.import(pdfBuffer, { onProgress: universalProgress }) + * await brain.import(excelBuffer, { onProgress: universalProgress }) + * // Works for JSON, Markdown, YAML, DOCX too! * ``` * * @example Performance Tuning (Large Files) @@ -2066,6 +2084,7 @@ export class Brainy implements BrainyInterface { * * @see {@link https://brainy.dev/docs/api/import API Documentation} * @see {@link https://brainy.dev/docs/guides/migrating-to-v4 Migration Guide} + * @see {@link https://brainy.dev/docs/guides/standard-import-progress Standard Progress API (v4.5.0)} * * @remarks * **โš ๏ธ Breaking Changes from v3.x:** @@ -2098,7 +2117,7 @@ export class Brainy implements BrainyInterface { async import( source: Buffer | string | object, options?: { - format?: 'excel' | 'pdf' | 'csv' | 'json' | 'markdown' + format?: 'excel' | 'pdf' | 'csv' | 'json' | 'markdown' | 'yaml' | 'docx' vfsPath?: string groupBy?: 'type' | 'sheet' | 'flat' | 'custom' customGrouping?: (entity: any) => string diff --git a/src/cli/commands/core.ts b/src/cli/commands/core.ts index 66c6f4c8..40ebb119 100644 --- a/src/cli/commands/core.ts +++ b/src/cli/commands/core.ts @@ -21,6 +21,8 @@ interface AddOptions extends CoreOptions { id?: string metadata?: string type?: string + confidence?: string + weight?: string } interface SearchOptions extends CoreOptions { @@ -35,6 +37,7 @@ interface SearchOptions extends CoreOptions { via?: string explain?: boolean includeRelations?: boolean + includeVfs?: boolean fusion?: string vectorWeight?: string graphWeight?: string @@ -163,24 +166,40 @@ export const coreCommands = { } // Add with explicit type - const result = await brain.add({ + const addParams: any = { data: text, type: nounType, metadata - }) - + } + + // v4.3.x: Add confidence and weight if provided + if (options.confidence) { + addParams.confidence = parseFloat(options.confidence) + } + if (options.weight) { + addParams.weight = parseFloat(options.weight) + } + + const result = await brain.add(addParams) + spinner.succeed('Added successfully') - + if (!options.json) { console.log(chalk.green(`โœ“ Added with ID: ${result}`)) if (options.type) { console.log(chalk.dim(` Type: ${options.type}`)) } + if (options.confidence) { + console.log(chalk.dim(` Confidence: ${options.confidence}`)) + } + if (options.weight) { + console.log(chalk.dim(` Weight: ${options.weight}`)) + } if (Object.keys(metadata).length > 0) { console.log(chalk.dim(` Metadata: ${JSON.stringify(metadata)}`)) } } else { - formatOutput({ id: result, metadata }, options) + formatOutput({ id: result, metadata, confidence: addParams.confidence, weight: addParams.weight }, options) } } catch (error: any) { if (spinner) spinner.fail('Failed to add data') @@ -328,6 +347,11 @@ export const coreCommands = { searchParams.includeRelations = true } + // Include VFS files (v4.4.0 - find excludes VFS by default) + if (options.includeVfs) { + searchParams.includeVFS = true + } + // Triple Intelligence Fusion - custom weighting if (options.fusion || options.vectorWeight || options.graphWeight || options.fieldWeight) { searchParams.fusion = { diff --git a/src/cli/commands/import.ts b/src/cli/commands/import.ts index 7240725a..7d257025 100644 --- a/src/cli/commands/import.ts +++ b/src/cli/commands/import.ts @@ -149,23 +149,28 @@ export const importCommands = { options.recursive = answer.recursive } - spinner = ora('Initializing neural import...').start() + spinner = ora('Initializing import...').start() const brain = getBrainy() - // Load UniversalImportAPI - const { UniversalImportAPI } = await import('../../api/UniversalImportAPI.js') - const universalImport = new UniversalImportAPI(brain) - await universalImport.init() - - spinner.text = 'Processing import...' - // Handle different source types let result: any if (isURL) { - // URL import + // URL import - fetch first spinner.text = `Fetching from ${source}...` - result = await universalImport.importFromURL(source) + const response = await fetch(source) + const buffer = Buffer.from(await response.arrayBuffer()) + + spinner.text = 'Importing...' + result = await brain.import(buffer, { + enableNeuralExtraction: options.extractEntities !== false, + enableRelationshipInference: options.detectRelationships !== false, + enableConceptExtraction: options.extractConcepts || false, + confidenceThreshold: options.confidence ? parseFloat(options.confidence) : 0.6, + onProgress: options.progress ? (p) => { + spinner.text = `${p.message}${p.entities ? ` (${p.entities} entities)` : ''}` + } : undefined + }) } else if (isDirectory) { // Directory import - process each file spinner.text = `Scanning directory: ${source}...` @@ -202,34 +207,45 @@ export const importCommands = { spinner.succeed(`Found ${files.length} files`) - // Process files in batches - const batchSize = options.batchSize ? parseInt(options.batchSize) : 100 + // Process files with progress let totalEntities = 0 let totalRelationships = 0 let filesProcessed = 0 - for (let i = 0; i < files.length; i += batchSize) { - const batch = files.slice(i, i + batchSize) + for (const file of files) { + try { + if (options.progress) { + spinner = ora(`[${filesProcessed + 1}/${files.length}] Importing ${file}...`).start() + } - if (options.progress) { - spinner = ora(`Processing batch ${Math.floor(i / batchSize) + 1}/${Math.ceil(files.length / batchSize)} (${filesProcessed}/${files.length} files)...`).start() - } + const fileResult = await brain.import(file, { + enableNeuralExtraction: options.extractEntities !== false, + enableRelationshipInference: options.detectRelationships !== false, + enableConceptExtraction: options.extractConcepts || false, + confidenceThreshold: options.confidence ? parseFloat(options.confidence) : 0.6, + onProgress: options.progress ? (p) => { + spinner.text = `[${filesProcessed + 1}/${files.length}] ${p.message}` + } : undefined + }) - for (const file of batch) { - try { - const fileResult = await universalImport.importFromFile(file) - totalEntities += fileResult.stats.entitiesCreated - totalRelationships += fileResult.stats.relationshipsCreated - filesProcessed++ - } catch (error: any) { - if (options.verbose) { - console.log(chalk.yellow(`โš ๏ธ Failed to import ${file}: ${error.message}`)) - } + totalEntities += fileResult.entities.length + totalRelationships += fileResult.relationships.length + filesProcessed++ + + if (options.progress) { + spinner.succeed(`[${filesProcessed}/${files.length}] ${file}`) + } + } catch (error: any) { + if (options.verbose) { + if (spinner) spinner.fail(`Failed: ${file}`) + console.log(chalk.yellow(`โš ๏ธ ${error.message}`)) } } } result = { + entities: [], + relationships: [], stats: { filesProcessed, entitiesCreated: totalEntities, @@ -238,10 +254,22 @@ export const importCommands = { } } - spinner.succeed('Directory import complete') + spinner = ora().succeed(`Directory import complete: ${filesProcessed} files`) } else { - // File import - result = await universalImport.importFromFile(source) + // File import with progress + result = await brain.import(source, { + format: options.format as any, + enableNeuralExtraction: options.extractEntities !== false, + enableRelationshipInference: options.detectRelationships !== false, + enableConceptExtraction: options.extractConcepts || false, + confidenceThreshold: options.confidence ? parseFloat(options.confidence) : 0.6, + onProgress: options.progress ? (p) => { + spinner.text = `${p.message}${p.entities ? ` (${p.entities} entities, ${p.relationships || 0} relationships)` : ''}` + if (p.throughput && p.eta) { + spinner.text += ` - ${p.throughput.toFixed(1)}/sec, ETA: ${Math.round(p.eta / 1000)}s` + } + } : undefined + }) } spinner.succeed('Import complete') @@ -323,18 +351,26 @@ export const importCommands = { console.log(chalk.cyan('\n๐Ÿ“Š Import Results:\n')) console.log(chalk.bold('Statistics:')) - console.log(` Entities created: ${chalk.green(result.stats.entitiesCreated)}`) + const entitiesCount = result.stats?.entitiesCreated || result.entities?.length || 0 + const relationshipsCount = result.stats?.relationshipsCreated || result.relationships?.length || 0 - if (result.stats.relationshipsCreated > 0) { - console.log(` Relationships created: ${chalk.green(result.stats.relationshipsCreated)}`) + console.log(` Entities created: ${chalk.green(entitiesCount)}`) + + if (relationshipsCount > 0) { + console.log(` Relationships created: ${chalk.green(relationshipsCount)}`) } - if (result.stats.filesProcessed) { + if (result.stats?.filesProcessed) { console.log(` Files processed: ${chalk.green(result.stats.filesProcessed)}`) } - console.log(` Average confidence: ${chalk.yellow((result.stats.averageConfidence * 100).toFixed(1))}%`) - console.log(` Processing time: ${chalk.dim(result.stats.processingTimeMs)}ms`) + if (result.stats?.averageConfidence) { + console.log(` Average confidence: ${chalk.yellow((result.stats.averageConfidence * 100).toFixed(1))}%`) + } + + if (result.stats?.processingTimeMs) { + console.log(` Processing time: ${chalk.dim(result.stats.processingTimeMs)}ms`) + } if (options.verbose && result.entities && result.entities.length > 0) { console.log(chalk.bold('\n๐Ÿ“ฆ Imported Entities (first 10):')) diff --git a/src/importers/SmartCSVImporter.ts b/src/importers/SmartCSVImporter.ts index 6adf9e16..576bb979 100644 --- a/src/importers/SmartCSVImporter.ts +++ b/src/importers/SmartCSVImporter.ts @@ -169,10 +169,44 @@ export class SmartCSVImporter { } // Parse CSV using existing handler + // v4.5.0: Pass progress hooks to handler for file parsing progress const processedData = await this.csvHandler.process(buffer, { ...options, csvDelimiter: opts.csvDelimiter, - csvHeaders: opts.csvHeaders + csvHeaders: opts.csvHeaders, + totalBytes: buffer.length, + progressHooks: { + onBytesProcessed: (bytes) => { + // Handler reports bytes processed during parsing + opts.onProgress?.({ + processed: 0, + total: 0, + entities: 0, + relationships: 0, + phase: `Parsing CSV (${Math.round((bytes / buffer.length) * 100)}%)` + }) + }, + onCurrentItem: (message) => { + // Handler reports current processing step + opts.onProgress?.({ + processed: 0, + total: 0, + entities: 0, + relationships: 0, + phase: message + }) + }, + onDataExtracted: (count, total) => { + // Handler reports rows extracted + opts.onProgress?.({ + processed: 0, + total: total || count, + entities: 0, + relationships: 0, + phase: `Extracted ${count} rows` + }) + } + } }) const rows = processedData.data diff --git a/src/importers/SmartDOCXImporter.ts b/src/importers/SmartDOCXImporter.ts index 0999093b..02a017c9 100644 --- a/src/importers/SmartDOCXImporter.ts +++ b/src/importers/SmartDOCXImporter.ts @@ -187,12 +187,26 @@ export class SmartDOCXImporter { await this.init() } + // v4.5.0: Report parsing start + options.onProgress?.({ + processed: 0, + entities: 0, + relationships: 0 + }) + // Extract raw text for entity extraction const textResult = await mammoth.extractRawText({ buffer }) // Extract HTML for structure analysis (headings, tables) const htmlResult = await mammoth.convertToHtml({ buffer }) + // v4.5.0: Report parsing complete + options.onProgress?.({ + processed: 0, + entities: 0, + relationships: 0 + }) + // Process the document const result = await this.extractFromContent( textResult.value, diff --git a/src/importers/SmartExcelImporter.ts b/src/importers/SmartExcelImporter.ts index b996772f..08f5591d 100644 --- a/src/importers/SmartExcelImporter.ts +++ b/src/importers/SmartExcelImporter.ts @@ -184,7 +184,43 @@ export class SmartExcelImporter { } // Parse Excel using existing handler - const processedData = await this.excelHandler.process(buffer, options) + // v4.5.0: Pass progress hooks to handler for file parsing progress + const processedData = await this.excelHandler.process(buffer, { + ...options, + totalBytes: buffer.length, + progressHooks: { + onBytesProcessed: (bytes) => { + // Handler reports bytes processed during parsing + opts.onProgress?.({ + processed: 0, + total: 0, + entities: 0, + relationships: 0, + phase: `Parsing Excel (${Math.round((bytes / buffer.length) * 100)}%)` + }) + }, + onCurrentItem: (message) => { + // Handler reports current processing step (e.g., "Reading sheet: Sales (1/3)") + opts.onProgress?.({ + processed: 0, + total: 0, + entities: 0, + relationships: 0, + phase: message + }) + }, + onDataExtracted: (count, total) => { + // Handler reports rows extracted + opts.onProgress?.({ + processed: 0, + total: total || count, + entities: 0, + relationships: 0, + phase: `Extracted ${count} rows from Excel` + }) + } + } + }) const rows = processedData.data if (rows.length === 0) { diff --git a/src/importers/SmartJSONImporter.ts b/src/importers/SmartJSONImporter.ts index bad98c6b..fd5a79e8 100644 --- a/src/importers/SmartJSONImporter.ts +++ b/src/importers/SmartJSONImporter.ts @@ -167,6 +167,13 @@ export class SmartJSONImporter { ...options } + // v4.5.0: Report parsing start + opts.onProgress({ + processed: 0, + entities: 0, + relationships: 0 + }) + // Parse JSON if string let jsonData: any if (typeof data === 'string') { @@ -179,6 +186,13 @@ export class SmartJSONImporter { jsonData = data } + // v4.5.0: Report parsing complete, starting traversal + opts.onProgress({ + processed: 0, + entities: 0, + relationships: 0 + }) + // Traverse and extract const entities: ExtractedJSONEntity[] = [] const relationships: ExtractedJSONRelationship[] = [] @@ -214,6 +228,13 @@ export class SmartJSONImporter { } ) + // v4.5.0: Report completion + opts.onProgress({ + processed: nodesProcessed, + entities: entities.length, + relationships: relationships.length + }) + return { nodesProcessed, entitiesExtracted: entities.length, diff --git a/src/importers/SmartMarkdownImporter.ts b/src/importers/SmartMarkdownImporter.ts index 6a85213f..44471f65 100644 --- a/src/importers/SmartMarkdownImporter.ts +++ b/src/importers/SmartMarkdownImporter.ts @@ -174,9 +174,25 @@ export class SmartMarkdownImporter { ...options } + // v4.5.0: Report parsing start + opts.onProgress({ + processed: 0, + total: 0, + entities: 0, + relationships: 0 + }) + // Parse markdown into sections const parsedSections = this.parseMarkdown(markdown, opts) + // v4.5.0: Report parsing complete + opts.onProgress({ + processed: 0, + total: parsedSections.length, + entities: 0, + relationships: 0 + }) + // Process each section const sections: MarkdownSection[] = [] const entityMap = new Map() @@ -202,10 +218,20 @@ export class SmartMarkdownImporter { }) } + // v4.5.0: Report completion + const totalEntities = sections.reduce((sum, s) => sum + s.entities.length, 0) + const totalRelationships = sections.reduce((sum, s) => sum + s.relationships.length, 0) + opts.onProgress({ + processed: sections.length, + total: sections.length, + entities: totalEntities, + relationships: totalRelationships + }) + return { sectionsProcessed: sections.length, - entitiesExtracted: sections.reduce((sum, s) => sum + s.entities.length, 0), - relationshipsInferred: sections.reduce((sum, s) => sum + s.relationships.length, 0), + entitiesExtracted: totalEntities, + relationshipsInferred: totalRelationships, sections, entityMap, processingTime: Date.now() - startTime, diff --git a/src/importers/SmartPDFImporter.ts b/src/importers/SmartPDFImporter.ts index 7ddc3a7e..18dc716c 100644 --- a/src/importers/SmartPDFImporter.ts +++ b/src/importers/SmartPDFImporter.ts @@ -181,7 +181,43 @@ export class SmartPDFImporter { } // Parse PDF using existing handler - const processedData = await this.pdfHandler.process(buffer, options) + // v4.5.0: Pass progress hooks to handler for file parsing progress + const processedData = await this.pdfHandler.process(buffer, { + ...options, + totalBytes: buffer.length, + progressHooks: { + onBytesProcessed: (bytes) => { + // Handler reports bytes processed during parsing + opts.onProgress?.({ + processed: 0, + total: 0, + entities: 0, + relationships: 0, + phase: `Parsing PDF (${Math.round((bytes / buffer.length) * 100)}%)` + }) + }, + onCurrentItem: (message) => { + // Handler reports current processing step (e.g., "Processing page 5 of 23") + opts.onProgress?.({ + processed: 0, + total: 0, + entities: 0, + relationships: 0, + phase: message + }) + }, + onDataExtracted: (count, total) => { + // Handler reports items extracted (paragraphs + tables) + opts.onProgress?.({ + processed: 0, + total: total || count, + entities: 0, + relationships: 0, + phase: `Extracted ${count} items from PDF` + }) + } + } + }) const data = processedData.data const pdfMetadata = processedData.metadata.additionalInfo?.pdfMetadata || {} diff --git a/src/importers/SmartYAMLImporter.ts b/src/importers/SmartYAMLImporter.ts index 0811a39a..73d6fc7d 100644 --- a/src/importers/SmartYAMLImporter.ts +++ b/src/importers/SmartYAMLImporter.ts @@ -159,6 +159,13 @@ export class SmartYAMLImporter { ): Promise { const startTime = Date.now() + // v4.5.0: Report parsing start + options.onProgress?.({ + processed: 0, + entities: 0, + relationships: 0 + }) + // Parse YAML to JavaScript object const yamlString = typeof yamlContent === 'string' ? yamlContent @@ -171,6 +178,13 @@ export class SmartYAMLImporter { throw new Error(`Failed to parse YAML: ${error.message}`) } + // v4.5.0: Report parsing complete + options.onProgress?.({ + processed: 0, + entities: 0, + relationships: 0 + }) + // Process as JSON-like structure const result = await this.extractFromData(data, options) result.processingTime = Date.now() - startTime diff --git a/src/types/brainy.types.ts b/src/types/brainy.types.ts index 1f54622a..8bea452f 100644 --- a/src/types/brainy.types.ts +++ b/src/types/brainy.types.ts @@ -385,6 +385,138 @@ export interface BatchResult { duration: number // Time taken in ms } +// ============= Import Progress (v4.5.0) ============= + +/** + * Import stage enumeration + */ +export type ImportStage = + | 'detecting' // Detecting file format + | 'reading' // Reading file from disk/network + | 'parsing' // Parsing file structure (CSV rows, PDF pages, Excel sheets) + | 'extracting' // Extracting entities using AI + | 'indexing' // Creating graph nodes and relationships + | 'completing' // Final cleanup and stats + +/** + * Overall import status + */ +export type ImportStatus = + | 'starting' // Initializing import + | 'processing' // Actively importing + | 'completing' // Finalizing + | 'done' // Complete + +/** + * Comprehensive import progress information + * + * Provides multi-dimensional progress tracking: + * - Bytes processed (always deterministic) + * - Entities extracted and indexed + * - Stage-specific progress + * - Time estimates + * - Performance metrics + * + * @since v4.5.0 + */ +export interface ImportProgress { + // Overall Progress + overall_progress: number // 0-100 weighted estimate across all stages + overall_status: ImportStatus // High-level status + + // Current Stage + stage: ImportStage // What's happening now + stage_progress: number // 0-100 within current stage (0 if unknown) + stage_message: string // Human-readable: "Extracting entities from PDF..." + + // Bytes (Always Available - most deterministic metric) + bytes_processed: number // Bytes read/processed so far + total_bytes: number // Total file size (0 if streaming/unknown) + bytes_percentage: number // Convenience: bytes_processed / total_bytes * 100 + bytes_per_second?: number // Processing rate (relevant during parsing) + + // Entities (Available when extraction starts) + entities_extracted: number // Entities found during AI extraction + entities_indexed: number // Entities added to Brainy graph + entities_per_second?: number // Entities/sec (relevant during extraction/indexing) + estimated_total_entities?: number // Estimated final count + estimation_confidence?: number // 0-1 confidence in estimation + + // Timing + elapsed_ms: number // Time since import started + estimated_remaining_ms?: number // Estimated time remaining + estimated_total_ms?: number // Estimated total time + + // Context (helps users understand what's happening) + current_item?: string // "Processing page 5 of 23" + current_file?: string // "Sheet: Q2 Sales Data" + file_number?: number // 3 (when importing multiple files) + total_files?: number // 10 + + // Performance Metrics (for debugging/optimization) + metrics?: { + parsing_rate_mbps?: number // MB/s during parsing + extraction_rate_entities_per_sec?: number // Entities/s during extraction + indexing_rate_entities_per_sec?: number // Entities/s during indexing + memory_usage_mb?: number // Current memory usage + peak_memory_mb?: number // Peak memory usage + } + + // Backwards Compatibility (for legacy code) + current: number // Alias for entities_indexed + total: number // Alias for estimated_total_entities or 0 +} + +/** + * Import progress callback - backwards compatible + * + * Supports both legacy (current, total) and new (ImportProgress object) signatures + */ +export type ImportProgressCallback = + | ((progress: ImportProgress) => void) + | ((current: number, total: number) => void) + +/** + * Stage weight configuration for overall progress calculation + * + * These weights reflect the typical time distribution across stages. + * Extraction is typically the slowest stage (60% of time). + */ +export interface StageWeights { + detecting: number // Default: 0.01 (1%) + reading: number // Default: 0.05 (5%) + parsing: number // Default: 0.10 (10%) + extracting: number // Default: 0.60 (60% - slowest!) + indexing: number // Default: 0.20 (20%) + completing: number // Default: 0.04 (4%) +} + +/** + * Import result statistics + */ +export interface ImportStats { + graphNodesCreated: number // Entities added to graph + graphEdgesCreated: number // Relationships created + vfsFilesCreated: number // VFS files created + duration: number // Total time in ms + bytesProcessed: number // Total bytes read + averageRate: number // Average entities/sec + peakMemoryMB?: number // Peak memory usage +} + +/** + * Import operation result + */ +export interface ImportResult { + success: boolean + stats: ImportStats + errors?: Array<{ + stage: ImportStage + message: string + error?: any + }> +} + // ============= Advanced Operations ============= /** diff --git a/src/utils/import-progress-tracker.ts b/src/utils/import-progress-tracker.ts new file mode 100644 index 00000000..7090eee1 --- /dev/null +++ b/src/utils/import-progress-tracker.ts @@ -0,0 +1,541 @@ +/** + * Import Progress Tracker (v4.5.0) + * + * Comprehensive progress tracking for imports with: + * - Multi-dimensional progress (bytes, entities, stages, timing) + * - Smart estimation (entity count, time remaining) + * - Stage-specific metrics (bytes/sec vs entities/sec) + * - Throttled callbacks (avoid spam) + * - Weighted overall progress + * + * @since v4.5.0 + */ + +import { + ImportProgress, + ImportStage, + ImportStatus, + StageWeights, + ImportProgressCallback +} from '../types/brainy.types.js' + +/** + * Default stage weights (reflect typical time distribution) + */ +const DEFAULT_STAGE_WEIGHTS: StageWeights = { + detecting: 0.01, // 1% - very fast + reading: 0.05, // 5% - reading file + parsing: 0.10, // 10% - parsing structure + extracting: 0.60, // 60% - AI extraction (slowest!) + indexing: 0.20, // 20% - creating graph + completing: 0.04 // 4% - cleanup +} + +/** + * Stage ordering for progress calculation + */ +const STAGE_ORDER: ImportStage[] = [ + 'detecting', + 'reading', + 'parsing', + 'extracting', + 'indexing', + 'completing' +] + +/** + * Progress tracker for imports + */ +export class ImportProgressTracker { + // Configuration + private readonly stageWeights: StageWeights + private readonly throttleMs: number + private readonly callback?: ImportProgressCallback + + // Tracking state + private startTime: number + private lastEmitTime: number = 0 + private currentStage: ImportStage = 'detecting' + private completedStages: Set = new Set() + + // Metrics + private totalBytes: number = 0 + private bytesProcessed: number = 0 + private entitiesExtracted: number = 0 + private entitiesIndexed: number = 0 + private parseStartTime?: number + private extractStartTime?: number + private indexStartTime?: number + + // Estimation + private lastBytesCheckpoint: number = 0 + private lastBytesCheckpointTime: number = 0 + private lastEntitiesCheckpoint: number = 0 + private lastEntitiesCheckpointTime: number = 0 + + // Context + private currentItem?: string + private currentFile?: string + private fileNumber?: number + private totalFiles?: number + + // Memory tracking + private peakMemoryMB: number = 0 + + constructor(options: { + totalBytes?: number + stageWeights?: Partial + throttleMs?: number + callback?: ImportProgressCallback + } = {}) { + this.stageWeights = { ...DEFAULT_STAGE_WEIGHTS, ...options.stageWeights } + this.throttleMs = options.throttleMs ?? 100 // 100ms default + this.callback = options.callback + this.totalBytes = options.totalBytes ?? 0 + this.startTime = Date.now() + this.lastBytesCheckpointTime = this.startTime + this.lastEntitiesCheckpointTime = this.startTime + } + + /** + * Set total file size (if known later) + */ + setTotalBytes(bytes: number): void { + this.totalBytes = bytes + } + + /** + * Update current stage + */ + setStage(stage: ImportStage, message?: string): void { + // Mark previous stage as complete + if (this.currentStage !== stage) { + this.completedStages.add(this.currentStage) + } + + this.currentStage = stage + if (message) { + this.setStageMessage(message) + } + + // Track stage start times + const now = Date.now() + switch (stage) { + case 'parsing': + this.parseStartTime = now + break + case 'extracting': + this.extractStartTime = now + break + case 'indexing': + this.indexStartTime = now + break + } + + // Force emit on stage change + this.emit(true) + } + + /** + * Update bytes processed + */ + updateBytes(bytes: number): void { + this.bytesProcessed = bytes + this.emit() + } + + /** + * Increment bytes processed + */ + addBytes(bytes: number): void { + this.bytesProcessed += bytes + this.emit() + } + + /** + * Update entities extracted + */ + updateEntitiesExtracted(count: number): void { + this.entitiesExtracted = count + this.emit() + } + + /** + * Increment entities extracted + */ + addEntitiesExtracted(count: number): void { + this.entitiesExtracted += count + this.emit() + } + + /** + * Update entities indexed + */ + updateEntitiesIndexed(count: number): void { + this.entitiesIndexed = count + this.emit() + } + + /** + * Increment entities indexed + */ + addEntitiesIndexed(count: number): void { + this.entitiesIndexed += count + this.emit() + } + + /** + * Set context information + */ + setContext(context: { + currentItem?: string + currentFile?: string + fileNumber?: number + totalFiles?: number + }): void { + if (context.currentItem !== undefined) this.currentItem = context.currentItem + if (context.currentFile !== undefined) this.currentFile = context.currentFile + if (context.fileNumber !== undefined) this.fileNumber = context.fileNumber + if (context.totalFiles !== undefined) this.totalFiles = context.totalFiles + this.emit() + } + + /** + * Set stage message + */ + private setStageMessage(message: string): void { + this.currentItem = message + } + + /** + * Calculate stage progress (0-100 within current stage) + */ + private calculateStageProgress(): number { + switch (this.currentStage) { + case 'detecting': + case 'completing': + // These are quick, assume 100% once started + return 100 + + case 'reading': + case 'parsing': + // Use bytes as proxy for progress + if (this.totalBytes === 0) return 0 + return Math.min(100, (this.bytesProcessed / this.totalBytes) * 100) + + case 'extracting': + // Extraction progress is hard to estimate (AI is unpredictable) + // We can't reliably say % complete, so return 0 + return 0 + + case 'indexing': + // If we have estimated total entities, use that + if (this.entitiesExtracted > 0) { + return Math.min(100, (this.entitiesIndexed / this.entitiesExtracted) * 100) + } + return 0 + + default: + return 0 + } + } + + /** + * Calculate overall progress (0-100 weighted across all stages) + */ + private calculateOverallProgress(): number { + // Calculate progress of completed stages + let completedWeight = 0 + for (const stage of this.completedStages) { + completedWeight += this.stageWeights[stage] + } + + // Calculate progress of current stage + const stageProgress = this.calculateStageProgress() + const currentStageContribution = this.stageWeights[this.currentStage] * (stageProgress / 100) + + // Overall = completed stages + current stage contribution + const overall = (completedWeight + currentStageContribution) * 100 + + return Math.min(100, Math.max(0, overall)) + } + + /** + * Calculate bytes per second + */ + private calculateBytesPerSecond(): number | undefined { + const now = Date.now() + const elapsed = now - this.lastBytesCheckpointTime + + // Need at least 1 second of data + if (elapsed < 1000) return undefined + + const bytesDelta = this.bytesProcessed - this.lastBytesCheckpoint + const bytesPerSec = (bytesDelta / elapsed) * 1000 + + // Update checkpoint + this.lastBytesCheckpoint = this.bytesProcessed + this.lastBytesCheckpointTime = now + + return bytesPerSec > 0 ? bytesPerSec : undefined + } + + /** + * Calculate entities per second + */ + private calculateEntitiesPerSecond(): number | undefined { + const now = Date.now() + const elapsed = now - this.lastEntitiesCheckpointTime + + // Need at least 1 second of data + if (elapsed < 1000) return undefined + + // Use appropriate counter based on stage + const currentCount = this.currentStage === 'indexing' + ? this.entitiesIndexed + : this.entitiesExtracted + + const entitiesDelta = currentCount - this.lastEntitiesCheckpoint + const entitiesPerSec = (entitiesDelta / elapsed) * 1000 + + // Update checkpoint + this.lastEntitiesCheckpoint = currentCount + this.lastEntitiesCheckpointTime = now + + return entitiesPerSec > 0 ? entitiesPerSec : undefined + } + + /** + * Estimate total entities + */ + private estimateTotalEntities(): { count: number, confidence: number } | undefined { + // Only estimate if we've processed some bytes and extracted some entities + if (this.bytesProcessed === 0 || this.entitiesExtracted === 0 || this.totalBytes === 0) { + return undefined + } + + // Estimate based on entities per byte + const bytesPercentage = this.bytesProcessed / this.totalBytes + const estimatedTotal = Math.ceil(this.entitiesExtracted / bytesPercentage) + + // Confidence increases with more data + const confidence = Math.min(0.95, bytesPercentage) + + return { count: estimatedTotal, confidence } + } + + /** + * Estimate remaining time + */ + private estimateRemainingTime(): number | undefined { + const now = Date.now() + const elapsed = now - this.startTime + + // Need at least 5 seconds of data for reasonable estimate + if (elapsed < 5000) return undefined + + const overallProgress = this.calculateOverallProgress() + if (overallProgress === 0) return undefined + + // Estimate total time based on current progress + const estimatedTotalMs = (elapsed / overallProgress) * 100 + const remainingMs = estimatedTotalMs - elapsed + + return remainingMs > 0 ? remainingMs : undefined + } + + /** + * Get current memory usage + */ + private getCurrentMemoryMB(): number | undefined { + if (typeof process === 'undefined' || !process.memoryUsage) return undefined + + const usage = process.memoryUsage() + const currentMB = usage.heapUsed / 1024 / 1024 + + // Track peak + this.peakMemoryMB = Math.max(this.peakMemoryMB, currentMB) + + return currentMB + } + + /** + * Build complete progress object + */ + private buildProgress(): ImportProgress { + const now = Date.now() + const elapsed = now - this.startTime + + const stageProgress = this.calculateStageProgress() + const overallProgress = this.calculateOverallProgress() + const bytesPerSec = this.calculateBytesPerSecond() + const entitiesPerSec = this.calculateEntitiesPerSecond() + const entityEstimate = this.estimateTotalEntities() + const remainingMs = this.estimateRemainingTime() + const currentMemoryMB = this.getCurrentMemoryMB() + + // Determine overall status + let overallStatus: ImportStatus + if (overallProgress === 0) { + overallStatus = 'starting' + } else if (overallProgress === 100) { + overallStatus = 'done' + } else if (this.currentStage === 'completing') { + overallStatus = 'completing' + } else { + overallStatus = 'processing' + } + + // Stage message + let stageMessage: string + if (this.currentItem) { + stageMessage = this.currentItem + } else { + // Default messages + switch (this.currentStage) { + case 'detecting': + stageMessage = 'Detecting file format...' + break + case 'reading': + stageMessage = 'Reading file...' + break + case 'parsing': + stageMessage = 'Parsing file structure...' + break + case 'extracting': + stageMessage = 'Extracting entities using AI...' + break + case 'indexing': + stageMessage = 'Creating graph nodes...' + break + case 'completing': + stageMessage = 'Finalizing import...' + break + default: + stageMessage = 'Processing...' + } + } + + // Calculate bytes percentage + const bytesPercentage = this.totalBytes > 0 + ? (this.bytesProcessed / this.totalBytes) * 100 + : 0 + + // Build metrics object + const metrics: ImportProgress['metrics'] = { + parsing_rate_mbps: this.currentStage === 'parsing' && bytesPerSec + ? bytesPerSec / 1_000_000 + : undefined, + extraction_rate_entities_per_sec: this.currentStage === 'extracting' + ? entitiesPerSec + : undefined, + indexing_rate_entities_per_sec: this.currentStage === 'indexing' + ? entitiesPerSec + : undefined, + memory_usage_mb: currentMemoryMB, + peak_memory_mb: this.peakMemoryMB > 0 ? this.peakMemoryMB : undefined + } + + const progress: ImportProgress = { + // Overall + overall_progress: overallProgress, + overall_status: overallStatus, + + // Stage + stage: this.currentStage, + stage_progress: stageProgress, + stage_message: stageMessage, + + // Bytes + bytes_processed: this.bytesProcessed, + total_bytes: this.totalBytes, + bytes_percentage: bytesPercentage, + bytes_per_second: bytesPerSec, + + // Entities + entities_extracted: this.entitiesExtracted, + entities_indexed: this.entitiesIndexed, + entities_per_second: entitiesPerSec, + estimated_total_entities: entityEstimate?.count, + estimation_confidence: entityEstimate?.confidence, + + // Timing + elapsed_ms: elapsed, + estimated_remaining_ms: remainingMs, + estimated_total_ms: remainingMs ? elapsed + remainingMs : undefined, + + // Context + current_item: this.currentItem, + current_file: this.currentFile, + file_number: this.fileNumber, + total_files: this.totalFiles, + + // Metrics + metrics, + + // Backwards compatibility + current: this.entitiesIndexed, + total: entityEstimate?.count ?? 0 + } + + return progress + } + + /** + * Emit progress (throttled) + */ + private emit(force: boolean = false): void { + if (!this.callback) return + + const now = Date.now() + const timeSinceLastEmit = now - this.lastEmitTime + + // Throttle unless forced + if (!force && timeSinceLastEmit < this.throttleMs) { + return + } + + const progress = this.buildProgress() + + // Handle both callback types (legacy and new) + if (this.callback.length === 2) { + // Legacy callback: (current, total) => void + ;(this.callback as (current: number, total: number) => void)( + progress.current, + progress.total + ) + } else { + // New callback: (progress: ImportProgress) => void + ;(this.callback as (progress: ImportProgress) => void)(progress) + } + + this.lastEmitTime = now + } + + /** + * Force emit (for completion or critical updates) + */ + forceEmit(): void { + this.emit(true) + } + + /** + * Get current progress (without emitting) + */ + getProgress(): ImportProgress { + return this.buildProgress() + } + + /** + * Mark import as complete + */ + complete(): ImportProgress { + this.currentStage = 'completing' + this.completedStages.add('completing') + + const progress = this.buildProgress() + this.forceEmit() + + return progress + } +}