feat: comprehensive import progress tracking for all 7 formats
Add real-time progress reporting throughout the entire import pipeline
with a standardized API that works across all supported formats.
Workshop Team Feature Request:
- Eliminates "0% complete" hangs during AI extraction
- Shows continuous progress with entities/sec, throughput, ETA
- Reports contextual messages ("Processing page 5 of 23")
- Standardized progress API for CSV, PDF, Excel, JSON, Markdown, YAML, DOCX
Core Changes:
- Add FormatHandlerProgressHooks interface for extensible progress
- Wire up all 3 binary format handlers (CSV, PDF, Excel) with 7+ progress points
- Wire up all 4 text format importers (JSON, Markdown, YAML, DOCX)
- Add ImportProgress interface with stage, message, counts, throughput, ETA
- ImportCoordinator normalizes all format progress to standard interface
CLI Improvements:
- Import command now uses brain.import() directly with full progress
- Add --include-vfs flag to find command (v4.4.0 compatibility)
- Add --confidence and --weight options to add command
Documentation:
- docs/guides/standard-import-progress.md - Universal API guide
- docs/guides/import-progress-implementation.md - Developer guide
- docs/guides/import-progress-examples.md - Practical examples
- JSDoc on brain.import() with universal handler examples
Result: ONE progress handler works for ALL 7 formats with zero format-specific code!
🤖 Generated with [Claude Code](https://claude.com/claude-code)
Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
parent
e7ea9c4e4b
commit
d5576ffb56
20 changed files with 3967 additions and 52 deletions
|
|
@ -34,9 +34,19 @@ export class CSVHandler extends BaseFormatHandler {
|
|||
|
||||
async process(data: Buffer | string, options: FormatHandlerOptions): Promise<ProcessedData> {
|
||||
const startTime = Date.now()
|
||||
const progressHooks = options.progressHooks
|
||||
|
||||
// Convert to buffer if string
|
||||
const buffer = Buffer.isBuffer(data) ? data : Buffer.from(data, 'utf-8')
|
||||
const totalBytes = buffer.length
|
||||
|
||||
// v4.5.0: Report total bytes for progress tracking
|
||||
if (progressHooks?.onBytesProcessed) {
|
||||
progressHooks.onBytesProcessed(0)
|
||||
}
|
||||
if (progressHooks?.onCurrentItem) {
|
||||
progressHooks.onCurrentItem('Detecting CSV encoding and delimiter...')
|
||||
}
|
||||
|
||||
// Detect encoding
|
||||
const detectedEncoding = options.encoding || this.detectEncodingSafe(buffer)
|
||||
|
|
@ -45,6 +55,11 @@ export class CSVHandler extends BaseFormatHandler {
|
|||
// Detect delimiter if not specified
|
||||
const delimiter = options.csvDelimiter || this.detectDelimiter(text)
|
||||
|
||||
// v4.5.0: Report progress - parsing started
|
||||
if (progressHooks?.onCurrentItem) {
|
||||
progressHooks.onCurrentItem(`Parsing CSV rows (delimiter: "${delimiter}")...`)
|
||||
}
|
||||
|
||||
// Parse CSV
|
||||
const hasHeaders = options.csvHeaders !== false
|
||||
const maxRows = options.maxRows
|
||||
|
|
@ -60,23 +75,47 @@ export class CSVHandler extends BaseFormatHandler {
|
|||
cast: false // We'll do type inference ourselves
|
||||
})
|
||||
|
||||
// v4.5.0: Report bytes processed (entire file parsed)
|
||||
if (progressHooks?.onBytesProcessed) {
|
||||
progressHooks.onBytesProcessed(totalBytes)
|
||||
}
|
||||
|
||||
// Convert to array of objects
|
||||
const data = Array.isArray(records) ? records : [records]
|
||||
|
||||
// v4.5.0: Report data extraction progress
|
||||
if (progressHooks?.onDataExtracted) {
|
||||
progressHooks.onDataExtracted(data.length, data.length)
|
||||
}
|
||||
if (progressHooks?.onCurrentItem) {
|
||||
progressHooks.onCurrentItem(`Extracted ${data.length} rows, inferring types...`)
|
||||
}
|
||||
|
||||
// Infer types and convert values
|
||||
const fields = data.length > 0 ? Object.keys(data[0]) : []
|
||||
const types = this.inferFieldTypes(data)
|
||||
|
||||
const convertedData = data.map(row => {
|
||||
const convertedData = data.map((row, index) => {
|
||||
const converted: Record<string, any> = {}
|
||||
for (const [key, value] of Object.entries(row)) {
|
||||
converted[key] = this.convertValue(value, types[key] || 'string')
|
||||
}
|
||||
|
||||
// v4.5.0: Report progress every 1000 rows
|
||||
if (progressHooks?.onCurrentItem && index > 0 && index % 1000 === 0) {
|
||||
progressHooks.onCurrentItem(`Converting types: ${index}/${data.length} rows...`)
|
||||
}
|
||||
|
||||
return converted
|
||||
})
|
||||
|
||||
const processingTime = Date.now() - startTime
|
||||
|
||||
// v4.5.0: Final progress update
|
||||
if (progressHooks?.onCurrentItem) {
|
||||
progressHooks.onCurrentItem(`CSV processing complete: ${convertedData.length} rows`)
|
||||
}
|
||||
|
||||
return {
|
||||
format: this.format,
|
||||
data: convertedData,
|
||||
|
|
|
|||
Reference in a new issue