feat: comprehensive import progress tracking for all 7 formats
Add real-time progress reporting throughout the entire import pipeline
with a standardized API that works across all supported formats.
Workshop Team Feature Request:
- Eliminates "0% complete" hangs during AI extraction
- Shows continuous progress with entities/sec, throughput, ETA
- Reports contextual messages ("Processing page 5 of 23")
- Standardized progress API for CSV, PDF, Excel, JSON, Markdown, YAML, DOCX
Core Changes:
- Add FormatHandlerProgressHooks interface for extensible progress
- Wire up all 3 binary format handlers (CSV, PDF, Excel) with 7+ progress points
- Wire up all 4 text format importers (JSON, Markdown, YAML, DOCX)
- Add ImportProgress interface with stage, message, counts, throughput, ETA
- ImportCoordinator normalizes all format progress to standard interface
CLI Improvements:
- Import command now uses brain.import() directly with full progress
- Add --include-vfs flag to find command (v4.4.0 compatibility)
- Add --confidence and --weight options to add command
Documentation:
- docs/guides/standard-import-progress.md - Universal API guide
- docs/guides/import-progress-implementation.md - Developer guide
- docs/guides/import-progress-examples.md - Practical examples
- JSDoc on brain.import() with universal handler examples
Result: ONE progress handler works for ALL 7 formats with zero format-specific code!
🤖 Generated with [Claude Code](https://claude.com/claude-code)
Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
parent
e7ea9c4e4b
commit
d5576ffb56
20 changed files with 3967 additions and 52 deletions
|
|
@ -169,10 +169,44 @@ export class SmartCSVImporter {
|
|||
}
|
||||
|
||||
// Parse CSV using existing handler
|
||||
// v4.5.0: Pass progress hooks to handler for file parsing progress
|
||||
const processedData = await this.csvHandler.process(buffer, {
|
||||
...options,
|
||||
csvDelimiter: opts.csvDelimiter,
|
||||
csvHeaders: opts.csvHeaders
|
||||
csvHeaders: opts.csvHeaders,
|
||||
totalBytes: buffer.length,
|
||||
progressHooks: {
|
||||
onBytesProcessed: (bytes) => {
|
||||
// Handler reports bytes processed during parsing
|
||||
opts.onProgress?.({
|
||||
processed: 0,
|
||||
total: 0,
|
||||
entities: 0,
|
||||
relationships: 0,
|
||||
phase: `Parsing CSV (${Math.round((bytes / buffer.length) * 100)}%)`
|
||||
})
|
||||
},
|
||||
onCurrentItem: (message) => {
|
||||
// Handler reports current processing step
|
||||
opts.onProgress?.({
|
||||
processed: 0,
|
||||
total: 0,
|
||||
entities: 0,
|
||||
relationships: 0,
|
||||
phase: message
|
||||
})
|
||||
},
|
||||
onDataExtracted: (count, total) => {
|
||||
// Handler reports rows extracted
|
||||
opts.onProgress?.({
|
||||
processed: 0,
|
||||
total: total || count,
|
||||
entities: 0,
|
||||
relationships: 0,
|
||||
phase: `Extracted ${count} rows`
|
||||
})
|
||||
}
|
||||
}
|
||||
})
|
||||
const rows = processedData.data
|
||||
|
||||
|
|
|
|||
|
|
@ -187,12 +187,26 @@ export class SmartDOCXImporter {
|
|||
await this.init()
|
||||
}
|
||||
|
||||
// v4.5.0: Report parsing start
|
||||
options.onProgress?.({
|
||||
processed: 0,
|
||||
entities: 0,
|
||||
relationships: 0
|
||||
})
|
||||
|
||||
// Extract raw text for entity extraction
|
||||
const textResult = await mammoth.extractRawText({ buffer })
|
||||
|
||||
// Extract HTML for structure analysis (headings, tables)
|
||||
const htmlResult = await mammoth.convertToHtml({ buffer })
|
||||
|
||||
// v4.5.0: Report parsing complete
|
||||
options.onProgress?.({
|
||||
processed: 0,
|
||||
entities: 0,
|
||||
relationships: 0
|
||||
})
|
||||
|
||||
// Process the document
|
||||
const result = await this.extractFromContent(
|
||||
textResult.value,
|
||||
|
|
|
|||
|
|
@ -184,7 +184,43 @@ export class SmartExcelImporter {
|
|||
}
|
||||
|
||||
// Parse Excel using existing handler
|
||||
const processedData = await this.excelHandler.process(buffer, options)
|
||||
// v4.5.0: Pass progress hooks to handler for file parsing progress
|
||||
const processedData = await this.excelHandler.process(buffer, {
|
||||
...options,
|
||||
totalBytes: buffer.length,
|
||||
progressHooks: {
|
||||
onBytesProcessed: (bytes) => {
|
||||
// Handler reports bytes processed during parsing
|
||||
opts.onProgress?.({
|
||||
processed: 0,
|
||||
total: 0,
|
||||
entities: 0,
|
||||
relationships: 0,
|
||||
phase: `Parsing Excel (${Math.round((bytes / buffer.length) * 100)}%)`
|
||||
})
|
||||
},
|
||||
onCurrentItem: (message) => {
|
||||
// Handler reports current processing step (e.g., "Reading sheet: Sales (1/3)")
|
||||
opts.onProgress?.({
|
||||
processed: 0,
|
||||
total: 0,
|
||||
entities: 0,
|
||||
relationships: 0,
|
||||
phase: message
|
||||
})
|
||||
},
|
||||
onDataExtracted: (count, total) => {
|
||||
// Handler reports rows extracted
|
||||
opts.onProgress?.({
|
||||
processed: 0,
|
||||
total: total || count,
|
||||
entities: 0,
|
||||
relationships: 0,
|
||||
phase: `Extracted ${count} rows from Excel`
|
||||
})
|
||||
}
|
||||
}
|
||||
})
|
||||
const rows = processedData.data
|
||||
|
||||
if (rows.length === 0) {
|
||||
|
|
|
|||
|
|
@ -167,6 +167,13 @@ export class SmartJSONImporter {
|
|||
...options
|
||||
}
|
||||
|
||||
// v4.5.0: Report parsing start
|
||||
opts.onProgress({
|
||||
processed: 0,
|
||||
entities: 0,
|
||||
relationships: 0
|
||||
})
|
||||
|
||||
// Parse JSON if string
|
||||
let jsonData: any
|
||||
if (typeof data === 'string') {
|
||||
|
|
@ -179,6 +186,13 @@ export class SmartJSONImporter {
|
|||
jsonData = data
|
||||
}
|
||||
|
||||
// v4.5.0: Report parsing complete, starting traversal
|
||||
opts.onProgress({
|
||||
processed: 0,
|
||||
entities: 0,
|
||||
relationships: 0
|
||||
})
|
||||
|
||||
// Traverse and extract
|
||||
const entities: ExtractedJSONEntity[] = []
|
||||
const relationships: ExtractedJSONRelationship[] = []
|
||||
|
|
@ -214,6 +228,13 @@ export class SmartJSONImporter {
|
|||
}
|
||||
)
|
||||
|
||||
// v4.5.0: Report completion
|
||||
opts.onProgress({
|
||||
processed: nodesProcessed,
|
||||
entities: entities.length,
|
||||
relationships: relationships.length
|
||||
})
|
||||
|
||||
return {
|
||||
nodesProcessed,
|
||||
entitiesExtracted: entities.length,
|
||||
|
|
|
|||
|
|
@ -174,9 +174,25 @@ export class SmartMarkdownImporter {
|
|||
...options
|
||||
}
|
||||
|
||||
// v4.5.0: Report parsing start
|
||||
opts.onProgress({
|
||||
processed: 0,
|
||||
total: 0,
|
||||
entities: 0,
|
||||
relationships: 0
|
||||
})
|
||||
|
||||
// Parse markdown into sections
|
||||
const parsedSections = this.parseMarkdown(markdown, opts)
|
||||
|
||||
// v4.5.0: Report parsing complete
|
||||
opts.onProgress({
|
||||
processed: 0,
|
||||
total: parsedSections.length,
|
||||
entities: 0,
|
||||
relationships: 0
|
||||
})
|
||||
|
||||
// Process each section
|
||||
const sections: MarkdownSection[] = []
|
||||
const entityMap = new Map<string, string>()
|
||||
|
|
@ -202,10 +218,20 @@ export class SmartMarkdownImporter {
|
|||
})
|
||||
}
|
||||
|
||||
// v4.5.0: Report completion
|
||||
const totalEntities = sections.reduce((sum, s) => sum + s.entities.length, 0)
|
||||
const totalRelationships = sections.reduce((sum, s) => sum + s.relationships.length, 0)
|
||||
opts.onProgress({
|
||||
processed: sections.length,
|
||||
total: sections.length,
|
||||
entities: totalEntities,
|
||||
relationships: totalRelationships
|
||||
})
|
||||
|
||||
return {
|
||||
sectionsProcessed: sections.length,
|
||||
entitiesExtracted: sections.reduce((sum, s) => sum + s.entities.length, 0),
|
||||
relationshipsInferred: sections.reduce((sum, s) => sum + s.relationships.length, 0),
|
||||
entitiesExtracted: totalEntities,
|
||||
relationshipsInferred: totalRelationships,
|
||||
sections,
|
||||
entityMap,
|
||||
processingTime: Date.now() - startTime,
|
||||
|
|
|
|||
|
|
@ -181,7 +181,43 @@ export class SmartPDFImporter {
|
|||
}
|
||||
|
||||
// Parse PDF using existing handler
|
||||
const processedData = await this.pdfHandler.process(buffer, options)
|
||||
// v4.5.0: Pass progress hooks to handler for file parsing progress
|
||||
const processedData = await this.pdfHandler.process(buffer, {
|
||||
...options,
|
||||
totalBytes: buffer.length,
|
||||
progressHooks: {
|
||||
onBytesProcessed: (bytes) => {
|
||||
// Handler reports bytes processed during parsing
|
||||
opts.onProgress?.({
|
||||
processed: 0,
|
||||
total: 0,
|
||||
entities: 0,
|
||||
relationships: 0,
|
||||
phase: `Parsing PDF (${Math.round((bytes / buffer.length) * 100)}%)`
|
||||
})
|
||||
},
|
||||
onCurrentItem: (message) => {
|
||||
// Handler reports current processing step (e.g., "Processing page 5 of 23")
|
||||
opts.onProgress?.({
|
||||
processed: 0,
|
||||
total: 0,
|
||||
entities: 0,
|
||||
relationships: 0,
|
||||
phase: message
|
||||
})
|
||||
},
|
||||
onDataExtracted: (count, total) => {
|
||||
// Handler reports items extracted (paragraphs + tables)
|
||||
opts.onProgress?.({
|
||||
processed: 0,
|
||||
total: total || count,
|
||||
entities: 0,
|
||||
relationships: 0,
|
||||
phase: `Extracted ${count} items from PDF`
|
||||
})
|
||||
}
|
||||
}
|
||||
})
|
||||
const data = processedData.data
|
||||
const pdfMetadata = processedData.metadata.additionalInfo?.pdfMetadata || {}
|
||||
|
||||
|
|
|
|||
|
|
@ -159,6 +159,13 @@ export class SmartYAMLImporter {
|
|||
): Promise<SmartYAMLResult> {
|
||||
const startTime = Date.now()
|
||||
|
||||
// v4.5.0: Report parsing start
|
||||
options.onProgress?.({
|
||||
processed: 0,
|
||||
entities: 0,
|
||||
relationships: 0
|
||||
})
|
||||
|
||||
// Parse YAML to JavaScript object
|
||||
const yamlString = typeof yamlContent === 'string'
|
||||
? yamlContent
|
||||
|
|
@ -171,6 +178,13 @@ export class SmartYAMLImporter {
|
|||
throw new Error(`Failed to parse YAML: ${error.message}`)
|
||||
}
|
||||
|
||||
// v4.5.0: Report parsing complete
|
||||
options.onProgress?.({
|
||||
processed: 0,
|
||||
entities: 0,
|
||||
relationships: 0
|
||||
})
|
||||
|
||||
// Process as JSON-like structure
|
||||
const result = await this.extractFromData(data, options)
|
||||
result.processingTime = Date.now() - startTime
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue