feat: comprehensive import progress tracking for all 7 formats

Add real-time progress reporting throughout the entire import pipeline
with a standardized API that works across all supported formats.

Workshop Team Feature Request:
- Eliminates "0% complete" hangs during AI extraction
- Shows continuous progress with entities/sec, throughput, ETA
- Reports contextual messages ("Processing page 5 of 23")
- Standardized progress API for CSV, PDF, Excel, JSON, Markdown, YAML, DOCX

Core Changes:
- Add FormatHandlerProgressHooks interface for extensible progress
- Wire up all 3 binary format handlers (CSV, PDF, Excel) with 7+ progress points
- Wire up all 4 text format importers (JSON, Markdown, YAML, DOCX)
- Add ImportProgress interface with stage, message, counts, throughput, ETA
- ImportCoordinator normalizes all format progress to standard interface

CLI Improvements:
- Import command now uses brain.import() directly with full progress
- Add --include-vfs flag to find command (v4.4.0 compatibility)
- Add --confidence and --weight options to add command

Documentation:
- docs/guides/standard-import-progress.md - Universal API guide
- docs/guides/import-progress-implementation.md - Developer guide
- docs/guides/import-progress-examples.md - Practical examples
- JSDoc on brain.import() with universal handler examples

Result: ONE progress handler works for ALL 7 formats with zero format-specific code!

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude <noreply@anthropic.com>

src/augmentations/intelligentImport/handlers/csvHandler.ts

@@ -34,9 +34,19 @@ export class CSVHandler extends BaseFormatHandler {
 
   async process(data: Buffer | string, options: FormatHandlerOptions): Promise<ProcessedData> {
     const startTime = Date.now()
+    const progressHooks = options.progressHooks
 
     // Convert to buffer if string
     const buffer = Buffer.isBuffer(data) ? data : Buffer.from(data, 'utf-8')
+    const totalBytes = buffer.length
+
+    // v4.5.0: Report total bytes for progress tracking
+    if (progressHooks?.onBytesProcessed) {
+      progressHooks.onBytesProcessed(0)
+    }
+    if (progressHooks?.onCurrentItem) {
+      progressHooks.onCurrentItem('Detecting CSV encoding and delimiter...')
+    }
 
     // Detect encoding
     const detectedEncoding = options.encoding || this.detectEncodingSafe(buffer)
@@ -45,6 +55,11 @@ export class CSVHandler extends BaseFormatHandler {
     // Detect delimiter if not specified
     const delimiter = options.csvDelimiter || this.detectDelimiter(text)
 
+    // v4.5.0: Report progress - parsing started
+    if (progressHooks?.onCurrentItem) {
+      progressHooks.onCurrentItem(`Parsing CSV rows (delimiter: "${delimiter}")...`)
+    }
+
     // Parse CSV
     const hasHeaders = options.csvHeaders !== false
     const maxRows = options.maxRows
@@ -60,23 +75,47 @@ export class CSVHandler extends BaseFormatHandler {
         cast: false // We'll do type inference ourselves
       })
 
+      // v4.5.0: Report bytes processed (entire file parsed)
+      if (progressHooks?.onBytesProcessed) {
+        progressHooks.onBytesProcessed(totalBytes)
+      }
+
       // Convert to array of objects
       const data = Array.isArray(records) ? records : [records]
 
+      // v4.5.0: Report data extraction progress
+      if (progressHooks?.onDataExtracted) {
+        progressHooks.onDataExtracted(data.length, data.length)
+      }
+      if (progressHooks?.onCurrentItem) {
+        progressHooks.onCurrentItem(`Extracted ${data.length} rows, inferring types...`)
+      }
+
       // Infer types and convert values
       const fields = data.length > 0 ? Object.keys(data[0]) : []
       const types = this.inferFieldTypes(data)
 
-      const convertedData = data.map(row => {
+      const convertedData = data.map((row, index) => {
         const converted: Record<string, any> = {}
         for (const [key, value] of Object.entries(row)) {
           converted[key] = this.convertValue(value, types[key] || 'string')
         }
+
+        // v4.5.0: Report progress every 1000 rows
+        if (progressHooks?.onCurrentItem && index > 0 && index % 1000 === 0) {
+          progressHooks.onCurrentItem(`Converting types: ${index}/${data.length} rows...`)
+        }
+
         return converted
       })
 
       const processingTime = Date.now() - startTime
 
+      // v4.5.0: Final progress update
+      if (progressHooks?.onCurrentItem) {
+        progressHooks.onCurrentItem(`CSV processing complete: ${convertedData.length} rows`)
+      }
+
       return {
         format: this.format,
         data: convertedData,

src/augmentations/intelligentImport/handlers/excelHandler.ts

@@ -21,9 +21,19 @@ export class ExcelHandler extends BaseFormatHandler {
 
   async process(data: Buffer | string, options: FormatHandlerOptions): Promise<ProcessedData> {
     const startTime = Date.now()
+    const progressHooks = options.progressHooks
 
     // Convert to buffer if string (though Excel should always be binary)
     const buffer = Buffer.isBuffer(data) ? data : Buffer.from(data, 'binary')
+    const totalBytes = buffer.length
+
+    // v4.5.0: Report start
+    if (progressHooks?.onBytesProcessed) {
+      progressHooks.onBytesProcessed(0)
+    }
+    if (progressHooks?.onCurrentItem) {
+      progressHooks.onCurrentItem('Loading Excel workbook...')
+    }
 
     try {
       // Read workbook
@@ -37,11 +47,25 @@ export class ExcelHandler extends BaseFormatHandler {
       // Determine which sheets to process
       const sheetsToProcess = this.getSheetsToProcess(workbook, options)
 
+      // v4.5.0: Report workbook loaded
+      if (progressHooks?.onCurrentItem) {
+        progressHooks.onCurrentItem(`Processing ${sheetsToProcess.length} sheets...`)
+      }
+
       // Extract data from sheets
       const allData: Array<Record<string, any>> = []
       const sheetMetadata: Record<string, any> = {}
 
-      for (const sheetName of sheetsToProcess) {
+      for (let sheetIndex = 0; sheetIndex < sheetsToProcess.length; sheetIndex++) {
+        const sheetName = sheetsToProcess[sheetIndex]
+
+        // v4.5.0: Report current sheet
+        if (progressHooks?.onCurrentItem) {
+          progressHooks.onCurrentItem(
+            `Reading sheet: ${sheetName} (${sheetIndex + 1}/${sheetsToProcess.length})`
+          )
+        }
+
         const sheet = workbook.Sheets[sheetName]
         if (!sheet) continue
 
@@ -92,6 +116,25 @@ export class ExcelHandler extends BaseFormatHandler {
           columnCount: headers.length,
           headers
         }
+
+        // v4.5.0: Estimate bytes processed (sheets are sequential)
+        const bytesProcessed = Math.floor(((sheetIndex + 1) / sheetsToProcess.length) * totalBytes)
+        if (progressHooks?.onBytesProcessed) {
+          progressHooks.onBytesProcessed(bytesProcessed)
+        }
+
+        // v4.5.0: Report extraction progress
+        if (progressHooks?.onDataExtracted) {
+          progressHooks.onDataExtracted(allData.length, undefined) // Total unknown until complete
+        }
+      }
+
+      // v4.5.0: Report data extraction complete
+      if (progressHooks?.onCurrentItem) {
+        progressHooks.onCurrentItem(`Extracted ${allData.length} rows, inferring types...`)
+      }
+      if (progressHooks?.onDataExtracted) {
+        progressHooks.onDataExtracted(allData.length, allData.length)
       }
 
       // Infer types (excluding _sheet field)
@@ -99,7 +142,7 @@ export class ExcelHandler extends BaseFormatHandler {
       const types = this.inferFieldTypes(allData)
 
       // Convert values to appropriate types
-      const convertedData = allData.map(row => {
+      const convertedData = allData.map((row, index) => {
         const converted: Record<string, any> = {}
         for (const [key, value] of Object.entries(row)) {
           if (key === '_sheet') {
@@ -108,11 +151,29 @@ export class ExcelHandler extends BaseFormatHandler {
             converted[key] = this.convertValue(value, types[key] || 'string')
           }
         }
+
+        // v4.5.0: Report progress every 1000 rows (avoid spam)
+        if (progressHooks?.onCurrentItem && index > 0 && index % 1000 === 0) {
+          progressHooks.onCurrentItem(`Converting types: ${index}/${allData.length} rows...`)
+        }
+
         return converted
       })
 
+      // v4.5.0: Final progress - all bytes processed
+      if (progressHooks?.onBytesProcessed) {
+        progressHooks.onBytesProcessed(totalBytes)
+      }
+
       const processingTime = Date.now() - startTime
 
+      // v4.5.0: Report completion
+      if (progressHooks?.onCurrentItem) {
+        progressHooks.onCurrentItem(
+          `Excel complete: ${sheetsToProcess.length} sheets, ${convertedData.length} rows`
+        )
+      }
+
       return {
         format: this.format,
         data: convertedData,

src/augmentations/intelligentImport/handlers/pdfHandler.ts

@@ -46,9 +46,19 @@ export class PDFHandler extends BaseFormatHandler {
 
   async process(data: Buffer | string, options: FormatHandlerOptions): Promise<ProcessedData> {
     const startTime = Date.now()
+    const progressHooks = options.progressHooks
 
     // Convert to buffer
     const buffer = Buffer.isBuffer(data) ? data : Buffer.from(data, 'binary')
+    const totalBytes = buffer.length
+
+    // v4.5.0: Report start
+    if (progressHooks?.onBytesProcessed) {
+      progressHooks.onBytesProcessed(0)
+    }
+    if (progressHooks?.onCurrentItem) {
+      progressHooks.onCurrentItem('Loading PDF document...')
+    }
 
     try {
       // Load PDF document
@@ -64,12 +74,22 @@ export class PDFHandler extends BaseFormatHandler {
       const metadata = await pdfDoc.getMetadata()
       const numPages = pdfDoc.numPages
 
+      // v4.5.0: Report document loaded
+      if (progressHooks?.onCurrentItem) {
+        progressHooks.onCurrentItem(`Processing ${numPages} pages...`)
+      }
+
       // Extract text and structure from all pages
       const allData: Array<Record<string, any>> = []
       let totalTextLength = 0
       let detectedTables = 0
 
       for (let pageNum = 1; pageNum <= numPages; pageNum++) {
+        // v4.5.0: Report current page
+        if (progressHooks?.onCurrentItem) {
+          progressHooks.onCurrentItem(`Processing page ${pageNum} of ${numPages}`)
+        }
+
         const page = await pdfDoc.getPage(pageNum)
         const textContent = await page.getTextContent()
 
@@ -110,10 +130,36 @@ export class PDFHandler extends BaseFormatHandler {
             })
           }
         }
+
+        // v4.5.0: Estimate bytes processed (pages are sequential)
+        const bytesProcessed = Math.floor((pageNum / numPages) * totalBytes)
+        if (progressHooks?.onBytesProcessed) {
+          progressHooks.onBytesProcessed(bytesProcessed)
+        }
+
+        // v4.5.0: Report extraction progress
+        if (progressHooks?.onDataExtracted) {
+          progressHooks.onDataExtracted(allData.length, undefined) // Total unknown until complete
+        }
+      }
+
+      // v4.5.0: Final progress - all bytes processed
+      if (progressHooks?.onBytesProcessed) {
+        progressHooks.onBytesProcessed(totalBytes)
+      }
+      if (progressHooks?.onDataExtracted) {
+        progressHooks.onDataExtracted(allData.length, allData.length)
       }
 
       const processingTime = Date.now() - startTime
 
+      // v4.5.0: Report completion
+      if (progressHooks?.onCurrentItem) {
+        progressHooks.onCurrentItem(
+          `PDF complete: ${numPages} pages, ${allData.length} items extracted`
+        )
+      }
+
       // Get all unique fields (excluding metadata fields)
       const fields = allData.length > 0
         ? Object.keys(allData[0]).filter(k => !k.startsWith('_'))

src/augmentations/intelligentImport/types.ts

@@ -3,6 +3,34 @@
  * Handles Excel, PDF, and CSV import with intelligent extraction
  */
 
+import { ImportProgressTracker } from '../../utils/import-progress-tracker.js'
+
+/**
+ * Progress hooks for format handlers
+ *
+ * Handlers call these hooks to report progress during processing.
+ * This enables real-time progress tracking for any file format.
+ */
+export interface FormatHandlerProgressHooks {
+  /**
+   * Report bytes processed
+   * Call this as you read/parse the file
+   */
+  onBytesProcessed?: (bytes: number) => void
+
+  /**
+   * Set current processing context
+   * Examples: "Processing page 5", "Reading sheet: Q2 Sales"
+   */
+  onCurrentItem?: (item: string) => void
+
+  /**
+   * Report structured data extraction progress
+   * Examples: "Extracted 100 rows", "Parsed 50 paragraphs"
+   */
+  onDataExtracted?: (count: number, total?: number) => void
+}
+
 export interface FormatHandler {
   /**
    * Format name (e.g., 'csv', 'xlsx', 'pdf')
@@ -58,6 +86,18 @@ export interface FormatHandlerOptions {
 
   /** Whether to stream large files */
   streaming?: boolean
+
+  /**
+   * Progress hooks (v4.5.0)
+   * Handlers call these to report progress during processing
+   */
+  progressHooks?: FormatHandlerProgressHooks
+
+  /**
+   * Total file size in bytes (v4.5.0)
+   * Used for progress percentage calculation
+   */
+  totalBytes?: number
 }
 
 export interface ProcessedData {