feat: comprehensive import progress tracking for all 7 formats
Add real-time progress reporting throughout the entire import pipeline
with a standardized API that works across all supported formats.
Workshop Team Feature Request:
- Eliminates "0% complete" hangs during AI extraction
- Shows continuous progress with entities/sec, throughput, ETA
- Reports contextual messages ("Processing page 5 of 23")
- Standardized progress API for CSV, PDF, Excel, JSON, Markdown, YAML, DOCX
Core Changes:
- Add FormatHandlerProgressHooks interface for extensible progress
- Wire up all 3 binary format handlers (CSV, PDF, Excel) with 7+ progress points
- Wire up all 4 text format importers (JSON, Markdown, YAML, DOCX)
- Add ImportProgress interface with stage, message, counts, throughput, ETA
- ImportCoordinator normalizes all format progress to standard interface
CLI Improvements:
- Import command now uses brain.import() directly with full progress
- Add --include-vfs flag to find command (v4.4.0 compatibility)
- Add --confidence and --weight options to add command
Documentation:
- docs/guides/standard-import-progress.md - Universal API guide
- docs/guides/import-progress-implementation.md - Developer guide
- docs/guides/import-progress-examples.md - Practical examples
- JSDoc on brain.import() with universal handler examples
Result: ONE progress handler works for ALL 7 formats with zero format-specific code!
🤖 Generated with [Claude Code](https://claude.com/claude-code)
Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
parent
e7ea9c4e4b
commit
d5576ffb56
20 changed files with 3967 additions and 52 deletions
|
|
@ -385,6 +385,138 @@ export interface BatchResult<T = any> {
|
|||
duration: number // Time taken in ms
|
||||
}
|
||||
|
||||
// ============= Import Progress (v4.5.0) =============
|
||||
|
||||
/**
|
||||
* Import stage enumeration
|
||||
*/
|
||||
export type ImportStage =
|
||||
| 'detecting' // Detecting file format
|
||||
| 'reading' // Reading file from disk/network
|
||||
| 'parsing' // Parsing file structure (CSV rows, PDF pages, Excel sheets)
|
||||
| 'extracting' // Extracting entities using AI
|
||||
| 'indexing' // Creating graph nodes and relationships
|
||||
| 'completing' // Final cleanup and stats
|
||||
|
||||
/**
|
||||
* Overall import status
|
||||
*/
|
||||
export type ImportStatus =
|
||||
| 'starting' // Initializing import
|
||||
| 'processing' // Actively importing
|
||||
| 'completing' // Finalizing
|
||||
| 'done' // Complete
|
||||
|
||||
/**
|
||||
* Comprehensive import progress information
|
||||
*
|
||||
* Provides multi-dimensional progress tracking:
|
||||
* - Bytes processed (always deterministic)
|
||||
* - Entities extracted and indexed
|
||||
* - Stage-specific progress
|
||||
* - Time estimates
|
||||
* - Performance metrics
|
||||
*
|
||||
* @since v4.5.0
|
||||
*/
|
||||
export interface ImportProgress {
|
||||
// Overall Progress
|
||||
overall_progress: number // 0-100 weighted estimate across all stages
|
||||
overall_status: ImportStatus // High-level status
|
||||
|
||||
// Current Stage
|
||||
stage: ImportStage // What's happening now
|
||||
stage_progress: number // 0-100 within current stage (0 if unknown)
|
||||
stage_message: string // Human-readable: "Extracting entities from PDF..."
|
||||
|
||||
// Bytes (Always Available - most deterministic metric)
|
||||
bytes_processed: number // Bytes read/processed so far
|
||||
total_bytes: number // Total file size (0 if streaming/unknown)
|
||||
bytes_percentage: number // Convenience: bytes_processed / total_bytes * 100
|
||||
bytes_per_second?: number // Processing rate (relevant during parsing)
|
||||
|
||||
// Entities (Available when extraction starts)
|
||||
entities_extracted: number // Entities found during AI extraction
|
||||
entities_indexed: number // Entities added to Brainy graph
|
||||
entities_per_second?: number // Entities/sec (relevant during extraction/indexing)
|
||||
estimated_total_entities?: number // Estimated final count
|
||||
estimation_confidence?: number // 0-1 confidence in estimation
|
||||
|
||||
// Timing
|
||||
elapsed_ms: number // Time since import started
|
||||
estimated_remaining_ms?: number // Estimated time remaining
|
||||
estimated_total_ms?: number // Estimated total time
|
||||
|
||||
// Context (helps users understand what's happening)
|
||||
current_item?: string // "Processing page 5 of 23"
|
||||
current_file?: string // "Sheet: Q2 Sales Data"
|
||||
file_number?: number // 3 (when importing multiple files)
|
||||
total_files?: number // 10
|
||||
|
||||
// Performance Metrics (for debugging/optimization)
|
||||
metrics?: {
|
||||
parsing_rate_mbps?: number // MB/s during parsing
|
||||
extraction_rate_entities_per_sec?: number // Entities/s during extraction
|
||||
indexing_rate_entities_per_sec?: number // Entities/s during indexing
|
||||
memory_usage_mb?: number // Current memory usage
|
||||
peak_memory_mb?: number // Peak memory usage
|
||||
}
|
||||
|
||||
// Backwards Compatibility (for legacy code)
|
||||
current: number // Alias for entities_indexed
|
||||
total: number // Alias for estimated_total_entities or 0
|
||||
}
|
||||
|
||||
/**
|
||||
* Import progress callback - backwards compatible
|
||||
*
|
||||
* Supports both legacy (current, total) and new (ImportProgress object) signatures
|
||||
*/
|
||||
export type ImportProgressCallback =
|
||||
| ((progress: ImportProgress) => void)
|
||||
| ((current: number, total: number) => void)
|
||||
|
||||
/**
|
||||
* Stage weight configuration for overall progress calculation
|
||||
*
|
||||
* These weights reflect the typical time distribution across stages.
|
||||
* Extraction is typically the slowest stage (60% of time).
|
||||
*/
|
||||
export interface StageWeights {
|
||||
detecting: number // Default: 0.01 (1%)
|
||||
reading: number // Default: 0.05 (5%)
|
||||
parsing: number // Default: 0.10 (10%)
|
||||
extracting: number // Default: 0.60 (60% - slowest!)
|
||||
indexing: number // Default: 0.20 (20%)
|
||||
completing: number // Default: 0.04 (4%)
|
||||
}
|
||||
|
||||
/**
|
||||
* Import result statistics
|
||||
*/
|
||||
export interface ImportStats {
|
||||
graphNodesCreated: number // Entities added to graph
|
||||
graphEdgesCreated: number // Relationships created
|
||||
vfsFilesCreated: number // VFS files created
|
||||
duration: number // Total time in ms
|
||||
bytesProcessed: number // Total bytes read
|
||||
averageRate: number // Average entities/sec
|
||||
peakMemoryMB?: number // Peak memory usage
|
||||
}
|
||||
|
||||
/**
|
||||
* Import operation result
|
||||
*/
|
||||
export interface ImportResult {
|
||||
success: boolean
|
||||
stats: ImportStats
|
||||
errors?: Array<{
|
||||
stage: ImportStage
|
||||
message: string
|
||||
error?: any
|
||||
}>
|
||||
}
|
||||
|
||||
// ============= Advanced Operations =============
|
||||
|
||||
/**
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue