feat: comprehensive import progress tracking for all 7 formats
Add real-time progress reporting throughout the entire import pipeline
with a standardized API that works across all supported formats.
Workshop Team Feature Request:
- Eliminates "0% complete" hangs during AI extraction
- Shows continuous progress with entities/sec, throughput, ETA
- Reports contextual messages ("Processing page 5 of 23")
- Standardized progress API for CSV, PDF, Excel, JSON, Markdown, YAML, DOCX
Core Changes:
- Add FormatHandlerProgressHooks interface for extensible progress
- Wire up all 3 binary format handlers (CSV, PDF, Excel) with 7+ progress points
- Wire up all 4 text format importers (JSON, Markdown, YAML, DOCX)
- Add ImportProgress interface with stage, message, counts, throughput, ETA
- ImportCoordinator normalizes all format progress to standard interface
CLI Improvements:
- Import command now uses brain.import() directly with full progress
- Add --include-vfs flag to find command (v4.4.0 compatibility)
- Add --confidence and --weight options to add command
Documentation:
- docs/guides/standard-import-progress.md - Universal API guide
- docs/guides/import-progress-implementation.md - Developer guide
- docs/guides/import-progress-examples.md - Practical examples
- JSDoc on brain.import() with universal handler examples
Result: ONE progress handler works for ALL 7 formats with zero format-specific code!
🤖 Generated with [Claude Code](https://claude.com/claude-code)
Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
parent
e7ea9c4e4b
commit
d5576ffb56
20 changed files with 3967 additions and 52 deletions
|
|
@ -21,6 +21,8 @@ interface AddOptions extends CoreOptions {
|
|||
id?: string
|
||||
metadata?: string
|
||||
type?: string
|
||||
confidence?: string
|
||||
weight?: string
|
||||
}
|
||||
|
||||
interface SearchOptions extends CoreOptions {
|
||||
|
|
@ -35,6 +37,7 @@ interface SearchOptions extends CoreOptions {
|
|||
via?: string
|
||||
explain?: boolean
|
||||
includeRelations?: boolean
|
||||
includeVfs?: boolean
|
||||
fusion?: string
|
||||
vectorWeight?: string
|
||||
graphWeight?: string
|
||||
|
|
@ -163,24 +166,40 @@ export const coreCommands = {
|
|||
}
|
||||
|
||||
// Add with explicit type
|
||||
const result = await brain.add({
|
||||
const addParams: any = {
|
||||
data: text,
|
||||
type: nounType,
|
||||
metadata
|
||||
})
|
||||
|
||||
}
|
||||
|
||||
// v4.3.x: Add confidence and weight if provided
|
||||
if (options.confidence) {
|
||||
addParams.confidence = parseFloat(options.confidence)
|
||||
}
|
||||
if (options.weight) {
|
||||
addParams.weight = parseFloat(options.weight)
|
||||
}
|
||||
|
||||
const result = await brain.add(addParams)
|
||||
|
||||
spinner.succeed('Added successfully')
|
||||
|
||||
|
||||
if (!options.json) {
|
||||
console.log(chalk.green(`✓ Added with ID: ${result}`))
|
||||
if (options.type) {
|
||||
console.log(chalk.dim(` Type: ${options.type}`))
|
||||
}
|
||||
if (options.confidence) {
|
||||
console.log(chalk.dim(` Confidence: ${options.confidence}`))
|
||||
}
|
||||
if (options.weight) {
|
||||
console.log(chalk.dim(` Weight: ${options.weight}`))
|
||||
}
|
||||
if (Object.keys(metadata).length > 0) {
|
||||
console.log(chalk.dim(` Metadata: ${JSON.stringify(metadata)}`))
|
||||
}
|
||||
} else {
|
||||
formatOutput({ id: result, metadata }, options)
|
||||
formatOutput({ id: result, metadata, confidence: addParams.confidence, weight: addParams.weight }, options)
|
||||
}
|
||||
} catch (error: any) {
|
||||
if (spinner) spinner.fail('Failed to add data')
|
||||
|
|
@ -328,6 +347,11 @@ export const coreCommands = {
|
|||
searchParams.includeRelations = true
|
||||
}
|
||||
|
||||
// Include VFS files (v4.4.0 - find excludes VFS by default)
|
||||
if (options.includeVfs) {
|
||||
searchParams.includeVFS = true
|
||||
}
|
||||
|
||||
// Triple Intelligence Fusion - custom weighting
|
||||
if (options.fusion || options.vectorWeight || options.graphWeight || options.fieldWeight) {
|
||||
searchParams.fusion = {
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue