feat: comprehensive import progress tracking for all 7 formats
Add real-time progress reporting throughout the entire import pipeline
with a standardized API that works across all supported formats.
Workshop Team Feature Request:
- Eliminates "0% complete" hangs during AI extraction
- Shows continuous progress with entities/sec, throughput, ETA
- Reports contextual messages ("Processing page 5 of 23")
- Standardized progress API for CSV, PDF, Excel, JSON, Markdown, YAML, DOCX
Core Changes:
- Add FormatHandlerProgressHooks interface for extensible progress
- Wire up all 3 binary format handlers (CSV, PDF, Excel) with 7+ progress points
- Wire up all 4 text format importers (JSON, Markdown, YAML, DOCX)
- Add ImportProgress interface with stage, message, counts, throughput, ETA
- ImportCoordinator normalizes all format progress to standard interface
CLI Improvements:
- Import command now uses brain.import() directly with full progress
- Add --include-vfs flag to find command (v4.4.0 compatibility)
- Add --confidence and --weight options to add command
Documentation:
- docs/guides/standard-import-progress.md - Universal API guide
- docs/guides/import-progress-implementation.md - Developer guide
- docs/guides/import-progress-examples.md - Practical examples
- JSDoc on brain.import() with universal handler examples
Result: ONE progress handler works for ALL 7 formats with zero format-specific code!
🤖 Generated with [Claude Code](https://claude.com/claude-code)
Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
parent
e7ea9c4e4b
commit
d5576ffb56
20 changed files with 3967 additions and 52 deletions
|
|
@ -46,9 +46,19 @@ export class PDFHandler extends BaseFormatHandler {
|
|||
|
||||
async process(data: Buffer | string, options: FormatHandlerOptions): Promise<ProcessedData> {
|
||||
const startTime = Date.now()
|
||||
const progressHooks = options.progressHooks
|
||||
|
||||
// Convert to buffer
|
||||
const buffer = Buffer.isBuffer(data) ? data : Buffer.from(data, 'binary')
|
||||
const totalBytes = buffer.length
|
||||
|
||||
// v4.5.0: Report start
|
||||
if (progressHooks?.onBytesProcessed) {
|
||||
progressHooks.onBytesProcessed(0)
|
||||
}
|
||||
if (progressHooks?.onCurrentItem) {
|
||||
progressHooks.onCurrentItem('Loading PDF document...')
|
||||
}
|
||||
|
||||
try {
|
||||
// Load PDF document
|
||||
|
|
@ -64,12 +74,22 @@ export class PDFHandler extends BaseFormatHandler {
|
|||
const metadata = await pdfDoc.getMetadata()
|
||||
const numPages = pdfDoc.numPages
|
||||
|
||||
// v4.5.0: Report document loaded
|
||||
if (progressHooks?.onCurrentItem) {
|
||||
progressHooks.onCurrentItem(`Processing ${numPages} pages...`)
|
||||
}
|
||||
|
||||
// Extract text and structure from all pages
|
||||
const allData: Array<Record<string, any>> = []
|
||||
let totalTextLength = 0
|
||||
let detectedTables = 0
|
||||
|
||||
for (let pageNum = 1; pageNum <= numPages; pageNum++) {
|
||||
// v4.5.0: Report current page
|
||||
if (progressHooks?.onCurrentItem) {
|
||||
progressHooks.onCurrentItem(`Processing page ${pageNum} of ${numPages}`)
|
||||
}
|
||||
|
||||
const page = await pdfDoc.getPage(pageNum)
|
||||
const textContent = await page.getTextContent()
|
||||
|
||||
|
|
@ -110,10 +130,36 @@ export class PDFHandler extends BaseFormatHandler {
|
|||
})
|
||||
}
|
||||
}
|
||||
|
||||
// v4.5.0: Estimate bytes processed (pages are sequential)
|
||||
const bytesProcessed = Math.floor((pageNum / numPages) * totalBytes)
|
||||
if (progressHooks?.onBytesProcessed) {
|
||||
progressHooks.onBytesProcessed(bytesProcessed)
|
||||
}
|
||||
|
||||
// v4.5.0: Report extraction progress
|
||||
if (progressHooks?.onDataExtracted) {
|
||||
progressHooks.onDataExtracted(allData.length, undefined) // Total unknown until complete
|
||||
}
|
||||
}
|
||||
|
||||
// v4.5.0: Final progress - all bytes processed
|
||||
if (progressHooks?.onBytesProcessed) {
|
||||
progressHooks.onBytesProcessed(totalBytes)
|
||||
}
|
||||
if (progressHooks?.onDataExtracted) {
|
||||
progressHooks.onDataExtracted(allData.length, allData.length)
|
||||
}
|
||||
|
||||
const processingTime = Date.now() - startTime
|
||||
|
||||
// v4.5.0: Report completion
|
||||
if (progressHooks?.onCurrentItem) {
|
||||
progressHooks.onCurrentItem(
|
||||
`PDF complete: ${numPages} pages, ${allData.length} items extracted`
|
||||
)
|
||||
}
|
||||
|
||||
// Get all unique fields (excluding metadata fields)
|
||||
const fields = allData.length > 0
|
||||
? Object.keys(allData[0]).filter(k => !k.startsWith('_'))
|
||||
|
|
|
|||
Reference in a new issue