feat: comprehensive import progress tracking for all 7 formats

Add real-time progress reporting throughout the entire import pipeline
with a standardized API that works across all supported formats.

Workshop Team Feature Request:
- Eliminates "0% complete" hangs during AI extraction
- Shows continuous progress with entities/sec, throughput, ETA
- Reports contextual messages ("Processing page 5 of 23")
- Standardized progress API for CSV, PDF, Excel, JSON, Markdown, YAML, DOCX

Core Changes:
- Add FormatHandlerProgressHooks interface for extensible progress
- Wire up all 3 binary format handlers (CSV, PDF, Excel) with 7+ progress points
- Wire up all 4 text format importers (JSON, Markdown, YAML, DOCX)
- Add ImportProgress interface with stage, message, counts, throughput, ETA
- ImportCoordinator normalizes all format progress to standard interface

CLI Improvements:
- Import command now uses brain.import() directly with full progress
- Add --include-vfs flag to find command (v4.4.0 compatibility)
- Add --confidence and --weight options to add command

Documentation:
- docs/guides/standard-import-progress.md - Universal API guide
- docs/guides/import-progress-implementation.md - Developer guide
- docs/guides/import-progress-examples.md - Practical examples
- JSDoc on brain.import() with universal handler examples

Result: ONE progress handler works for ALL 7 formats with zero format-specific code!

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
David Snelling 2025-10-24 14:45:46 -07:00
parent e7ea9c4e4b
commit d5576ffb56
20 changed files with 3967 additions and 52 deletions

View file

@ -34,9 +34,19 @@ export class CSVHandler extends BaseFormatHandler {
async process(data: Buffer | string, options: FormatHandlerOptions): Promise<ProcessedData> {
const startTime = Date.now()
const progressHooks = options.progressHooks
// Convert to buffer if string
const buffer = Buffer.isBuffer(data) ? data : Buffer.from(data, 'utf-8')
const totalBytes = buffer.length
// v4.5.0: Report total bytes for progress tracking
if (progressHooks?.onBytesProcessed) {
progressHooks.onBytesProcessed(0)
}
if (progressHooks?.onCurrentItem) {
progressHooks.onCurrentItem('Detecting CSV encoding and delimiter...')
}
// Detect encoding
const detectedEncoding = options.encoding || this.detectEncodingSafe(buffer)
@ -45,6 +55,11 @@ export class CSVHandler extends BaseFormatHandler {
// Detect delimiter if not specified
const delimiter = options.csvDelimiter || this.detectDelimiter(text)
// v4.5.0: Report progress - parsing started
if (progressHooks?.onCurrentItem) {
progressHooks.onCurrentItem(`Parsing CSV rows (delimiter: "${delimiter}")...`)
}
// Parse CSV
const hasHeaders = options.csvHeaders !== false
const maxRows = options.maxRows
@ -60,23 +75,47 @@ export class CSVHandler extends BaseFormatHandler {
cast: false // We'll do type inference ourselves
})
// v4.5.0: Report bytes processed (entire file parsed)
if (progressHooks?.onBytesProcessed) {
progressHooks.onBytesProcessed(totalBytes)
}
// Convert to array of objects
const data = Array.isArray(records) ? records : [records]
// v4.5.0: Report data extraction progress
if (progressHooks?.onDataExtracted) {
progressHooks.onDataExtracted(data.length, data.length)
}
if (progressHooks?.onCurrentItem) {
progressHooks.onCurrentItem(`Extracted ${data.length} rows, inferring types...`)
}
// Infer types and convert values
const fields = data.length > 0 ? Object.keys(data[0]) : []
const types = this.inferFieldTypes(data)
const convertedData = data.map(row => {
const convertedData = data.map((row, index) => {
const converted: Record<string, any> = {}
for (const [key, value] of Object.entries(row)) {
converted[key] = this.convertValue(value, types[key] || 'string')
}
// v4.5.0: Report progress every 1000 rows
if (progressHooks?.onCurrentItem && index > 0 && index % 1000 === 0) {
progressHooks.onCurrentItem(`Converting types: ${index}/${data.length} rows...`)
}
return converted
})
const processingTime = Date.now() - startTime
// v4.5.0: Final progress update
if (progressHooks?.onCurrentItem) {
progressHooks.onCurrentItem(`CSV processing complete: ${convertedData.length} rows`)
}
return {
format: this.format,
data: convertedData,