feat: comprehensive import progress tracking for all 7 formats
Add real-time progress reporting throughout the entire import pipeline
with a standardized API that works across all supported formats.
Workshop Team Feature Request:
- Eliminates "0% complete" hangs during AI extraction
- Shows continuous progress with entities/sec, throughput, ETA
- Reports contextual messages ("Processing page 5 of 23")
- Standardized progress API for CSV, PDF, Excel, JSON, Markdown, YAML, DOCX
Core Changes:
- Add FormatHandlerProgressHooks interface for extensible progress
- Wire up all 3 binary format handlers (CSV, PDF, Excel) with 7+ progress points
- Wire up all 4 text format importers (JSON, Markdown, YAML, DOCX)
- Add ImportProgress interface with stage, message, counts, throughput, ETA
- ImportCoordinator normalizes all format progress to standard interface
CLI Improvements:
- Import command now uses brain.import() directly with full progress
- Add --include-vfs flag to find command (v4.4.0 compatibility)
- Add --confidence and --weight options to add command
Documentation:
- docs/guides/standard-import-progress.md - Universal API guide
- docs/guides/import-progress-implementation.md - Developer guide
- docs/guides/import-progress-examples.md - Practical examples
- JSDoc on brain.import() with universal handler examples
Result: ONE progress handler works for ALL 7 formats with zero format-specific code!
🤖 Generated with [Claude Code](https://claude.com/claude-code)
Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
parent
e7ea9c4e4b
commit
d5576ffb56
20 changed files with 3967 additions and 52 deletions
|
|
@ -21,6 +21,8 @@ interface AddOptions extends CoreOptions {
|
|||
id?: string
|
||||
metadata?: string
|
||||
type?: string
|
||||
confidence?: string
|
||||
weight?: string
|
||||
}
|
||||
|
||||
interface SearchOptions extends CoreOptions {
|
||||
|
|
@ -35,6 +37,7 @@ interface SearchOptions extends CoreOptions {
|
|||
via?: string
|
||||
explain?: boolean
|
||||
includeRelations?: boolean
|
||||
includeVfs?: boolean
|
||||
fusion?: string
|
||||
vectorWeight?: string
|
||||
graphWeight?: string
|
||||
|
|
@ -163,24 +166,40 @@ export const coreCommands = {
|
|||
}
|
||||
|
||||
// Add with explicit type
|
||||
const result = await brain.add({
|
||||
const addParams: any = {
|
||||
data: text,
|
||||
type: nounType,
|
||||
metadata
|
||||
})
|
||||
|
||||
}
|
||||
|
||||
// v4.3.x: Add confidence and weight if provided
|
||||
if (options.confidence) {
|
||||
addParams.confidence = parseFloat(options.confidence)
|
||||
}
|
||||
if (options.weight) {
|
||||
addParams.weight = parseFloat(options.weight)
|
||||
}
|
||||
|
||||
const result = await brain.add(addParams)
|
||||
|
||||
spinner.succeed('Added successfully')
|
||||
|
||||
|
||||
if (!options.json) {
|
||||
console.log(chalk.green(`✓ Added with ID: ${result}`))
|
||||
if (options.type) {
|
||||
console.log(chalk.dim(` Type: ${options.type}`))
|
||||
}
|
||||
if (options.confidence) {
|
||||
console.log(chalk.dim(` Confidence: ${options.confidence}`))
|
||||
}
|
||||
if (options.weight) {
|
||||
console.log(chalk.dim(` Weight: ${options.weight}`))
|
||||
}
|
||||
if (Object.keys(metadata).length > 0) {
|
||||
console.log(chalk.dim(` Metadata: ${JSON.stringify(metadata)}`))
|
||||
}
|
||||
} else {
|
||||
formatOutput({ id: result, metadata }, options)
|
||||
formatOutput({ id: result, metadata, confidence: addParams.confidence, weight: addParams.weight }, options)
|
||||
}
|
||||
} catch (error: any) {
|
||||
if (spinner) spinner.fail('Failed to add data')
|
||||
|
|
@ -328,6 +347,11 @@ export const coreCommands = {
|
|||
searchParams.includeRelations = true
|
||||
}
|
||||
|
||||
// Include VFS files (v4.4.0 - find excludes VFS by default)
|
||||
if (options.includeVfs) {
|
||||
searchParams.includeVFS = true
|
||||
}
|
||||
|
||||
// Triple Intelligence Fusion - custom weighting
|
||||
if (options.fusion || options.vectorWeight || options.graphWeight || options.fieldWeight) {
|
||||
searchParams.fusion = {
|
||||
|
|
|
|||
|
|
@ -149,23 +149,28 @@ export const importCommands = {
|
|||
options.recursive = answer.recursive
|
||||
}
|
||||
|
||||
spinner = ora('Initializing neural import...').start()
|
||||
spinner = ora('Initializing import...').start()
|
||||
const brain = getBrainy()
|
||||
|
||||
// Load UniversalImportAPI
|
||||
const { UniversalImportAPI } = await import('../../api/UniversalImportAPI.js')
|
||||
const universalImport = new UniversalImportAPI(brain)
|
||||
await universalImport.init()
|
||||
|
||||
spinner.text = 'Processing import...'
|
||||
|
||||
// Handle different source types
|
||||
let result: any
|
||||
|
||||
if (isURL) {
|
||||
// URL import
|
||||
// URL import - fetch first
|
||||
spinner.text = `Fetching from ${source}...`
|
||||
result = await universalImport.importFromURL(source)
|
||||
const response = await fetch(source)
|
||||
const buffer = Buffer.from(await response.arrayBuffer())
|
||||
|
||||
spinner.text = 'Importing...'
|
||||
result = await brain.import(buffer, {
|
||||
enableNeuralExtraction: options.extractEntities !== false,
|
||||
enableRelationshipInference: options.detectRelationships !== false,
|
||||
enableConceptExtraction: options.extractConcepts || false,
|
||||
confidenceThreshold: options.confidence ? parseFloat(options.confidence) : 0.6,
|
||||
onProgress: options.progress ? (p) => {
|
||||
spinner.text = `${p.message}${p.entities ? ` (${p.entities} entities)` : ''}`
|
||||
} : undefined
|
||||
})
|
||||
} else if (isDirectory) {
|
||||
// Directory import - process each file
|
||||
spinner.text = `Scanning directory: ${source}...`
|
||||
|
|
@ -202,34 +207,45 @@ export const importCommands = {
|
|||
|
||||
spinner.succeed(`Found ${files.length} files`)
|
||||
|
||||
// Process files in batches
|
||||
const batchSize = options.batchSize ? parseInt(options.batchSize) : 100
|
||||
// Process files with progress
|
||||
let totalEntities = 0
|
||||
let totalRelationships = 0
|
||||
let filesProcessed = 0
|
||||
|
||||
for (let i = 0; i < files.length; i += batchSize) {
|
||||
const batch = files.slice(i, i + batchSize)
|
||||
for (const file of files) {
|
||||
try {
|
||||
if (options.progress) {
|
||||
spinner = ora(`[${filesProcessed + 1}/${files.length}] Importing ${file}...`).start()
|
||||
}
|
||||
|
||||
if (options.progress) {
|
||||
spinner = ora(`Processing batch ${Math.floor(i / batchSize) + 1}/${Math.ceil(files.length / batchSize)} (${filesProcessed}/${files.length} files)...`).start()
|
||||
}
|
||||
const fileResult = await brain.import(file, {
|
||||
enableNeuralExtraction: options.extractEntities !== false,
|
||||
enableRelationshipInference: options.detectRelationships !== false,
|
||||
enableConceptExtraction: options.extractConcepts || false,
|
||||
confidenceThreshold: options.confidence ? parseFloat(options.confidence) : 0.6,
|
||||
onProgress: options.progress ? (p) => {
|
||||
spinner.text = `[${filesProcessed + 1}/${files.length}] ${p.message}`
|
||||
} : undefined
|
||||
})
|
||||
|
||||
for (const file of batch) {
|
||||
try {
|
||||
const fileResult = await universalImport.importFromFile(file)
|
||||
totalEntities += fileResult.stats.entitiesCreated
|
||||
totalRelationships += fileResult.stats.relationshipsCreated
|
||||
filesProcessed++
|
||||
} catch (error: any) {
|
||||
if (options.verbose) {
|
||||
console.log(chalk.yellow(`⚠️ Failed to import ${file}: ${error.message}`))
|
||||
}
|
||||
totalEntities += fileResult.entities.length
|
||||
totalRelationships += fileResult.relationships.length
|
||||
filesProcessed++
|
||||
|
||||
if (options.progress) {
|
||||
spinner.succeed(`[${filesProcessed}/${files.length}] ${file}`)
|
||||
}
|
||||
} catch (error: any) {
|
||||
if (options.verbose) {
|
||||
if (spinner) spinner.fail(`Failed: ${file}`)
|
||||
console.log(chalk.yellow(`⚠️ ${error.message}`))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
result = {
|
||||
entities: [],
|
||||
relationships: [],
|
||||
stats: {
|
||||
filesProcessed,
|
||||
entitiesCreated: totalEntities,
|
||||
|
|
@ -238,10 +254,22 @@ export const importCommands = {
|
|||
}
|
||||
}
|
||||
|
||||
spinner.succeed('Directory import complete')
|
||||
spinner = ora().succeed(`Directory import complete: ${filesProcessed} files`)
|
||||
} else {
|
||||
// File import
|
||||
result = await universalImport.importFromFile(source)
|
||||
// File import with progress
|
||||
result = await brain.import(source, {
|
||||
format: options.format as any,
|
||||
enableNeuralExtraction: options.extractEntities !== false,
|
||||
enableRelationshipInference: options.detectRelationships !== false,
|
||||
enableConceptExtraction: options.extractConcepts || false,
|
||||
confidenceThreshold: options.confidence ? parseFloat(options.confidence) : 0.6,
|
||||
onProgress: options.progress ? (p) => {
|
||||
spinner.text = `${p.message}${p.entities ? ` (${p.entities} entities, ${p.relationships || 0} relationships)` : ''}`
|
||||
if (p.throughput && p.eta) {
|
||||
spinner.text += ` - ${p.throughput.toFixed(1)}/sec, ETA: ${Math.round(p.eta / 1000)}s`
|
||||
}
|
||||
} : undefined
|
||||
})
|
||||
}
|
||||
|
||||
spinner.succeed('Import complete')
|
||||
|
|
@ -323,18 +351,26 @@ export const importCommands = {
|
|||
console.log(chalk.cyan('\n📊 Import Results:\n'))
|
||||
|
||||
console.log(chalk.bold('Statistics:'))
|
||||
console.log(` Entities created: ${chalk.green(result.stats.entitiesCreated)}`)
|
||||
const entitiesCount = result.stats?.entitiesCreated || result.entities?.length || 0
|
||||
const relationshipsCount = result.stats?.relationshipsCreated || result.relationships?.length || 0
|
||||
|
||||
if (result.stats.relationshipsCreated > 0) {
|
||||
console.log(` Relationships created: ${chalk.green(result.stats.relationshipsCreated)}`)
|
||||
console.log(` Entities created: ${chalk.green(entitiesCount)}`)
|
||||
|
||||
if (relationshipsCount > 0) {
|
||||
console.log(` Relationships created: ${chalk.green(relationshipsCount)}`)
|
||||
}
|
||||
|
||||
if (result.stats.filesProcessed) {
|
||||
if (result.stats?.filesProcessed) {
|
||||
console.log(` Files processed: ${chalk.green(result.stats.filesProcessed)}`)
|
||||
}
|
||||
|
||||
console.log(` Average confidence: ${chalk.yellow((result.stats.averageConfidence * 100).toFixed(1))}%`)
|
||||
console.log(` Processing time: ${chalk.dim(result.stats.processingTimeMs)}ms`)
|
||||
if (result.stats?.averageConfidence) {
|
||||
console.log(` Average confidence: ${chalk.yellow((result.stats.averageConfidence * 100).toFixed(1))}%`)
|
||||
}
|
||||
|
||||
if (result.stats?.processingTimeMs) {
|
||||
console.log(` Processing time: ${chalk.dim(result.stats.processingTimeMs)}ms`)
|
||||
}
|
||||
|
||||
if (options.verbose && result.entities && result.entities.length > 0) {
|
||||
console.log(chalk.bold('\n📦 Imported Entities (first 10):'))
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue