refactor: remove augmentation system and semantic type matching
Remove the entire augmentation pipeline infrastructure (52 files, ~15,000 lines) and the semantic type matching system. These were unused middleware layers adding complexity without value. What was removed: - src/augmentations/ directory (all augmentation implementations) - src/augmentationManager.ts (pipeline orchestrator) - src/types/augmentations.ts, src/types/pipelineTypes.ts - src/shared/default-augmentations.ts - Semantic type suggestion (BrainyTypes.suggestNoun/suggestVerb) - src/utils/typeMatching/ (embedding-based type matcher) What was preserved by relocating: - Import handlers (CSV, PDF, Excel) -> src/importers/handlers/ - NeuralImportAugmentation -> src/cortex/neuralImportAugmentation.ts - Type matching utilities -> heuristic inference in consumers What was simplified: - brainy.ts: operations call storage directly (no execute() wrapper) - IntegrationBase: standalone class (no BaseAugmentation parent) - BrainyTypes: validation-only (nouns, verbs, isValid*, get*) - Pipeline: direct execution (no augmentation interception) - index.ts: removed TypeSuggestion, suggestType exports - package.json: removed stale types/augmentations export Build passes, 1176 tests pass, 0 failures.
This commit is contained in:
parent
ac7a1f772c
commit
d1db3510be
97 changed files with 349 additions and 19705 deletions
|
|
@ -16,8 +16,8 @@ import { NeuralEntityExtractor, ExtractedEntity } from '../neural/entityExtracto
|
|||
import { NaturalLanguageProcessor } from '../neural/naturalLanguageProcessor.js'
|
||||
import { SmartRelationshipExtractor } from '../neural/SmartRelationshipExtractor.js'
|
||||
import { NounType, VerbType } from '../types/graphTypes.js'
|
||||
import { CSVHandler } from '../augmentations/intelligentImport/handlers/csvHandler.js'
|
||||
import type { FormatHandlerOptions } from '../augmentations/intelligentImport/types.js'
|
||||
import { CSVHandler } from './handlers/csvHandler.js'
|
||||
import type { FormatHandlerOptions } from './handlers/types.js'
|
||||
|
||||
export interface SmartCSVOptions extends FormatHandlerOptions {
|
||||
/** Enable neural entity extraction */
|
||||
|
|
|
|||
|
|
@ -14,8 +14,8 @@ import { NeuralEntityExtractor, ExtractedEntity } from '../neural/entityExtracto
|
|||
import { NaturalLanguageProcessor } from '../neural/naturalLanguageProcessor.js'
|
||||
import { SmartRelationshipExtractor } from '../neural/SmartRelationshipExtractor.js'
|
||||
import { NounType, VerbType } from '../types/graphTypes.js'
|
||||
import { ExcelHandler } from '../augmentations/intelligentImport/handlers/excelHandler.js'
|
||||
import type { FormatHandlerOptions } from '../augmentations/intelligentImport/types.js'
|
||||
import { ExcelHandler } from './handlers/excelHandler.js'
|
||||
import type { FormatHandlerOptions } from './handlers/types.js'
|
||||
|
||||
export interface SmartExcelOptions extends FormatHandlerOptions {
|
||||
/** Enable neural entity extraction */
|
||||
|
|
|
|||
|
|
@ -15,8 +15,8 @@ import { NeuralEntityExtractor, ExtractedEntity } from '../neural/entityExtracto
|
|||
import { NaturalLanguageProcessor } from '../neural/naturalLanguageProcessor.js'
|
||||
import { SmartRelationshipExtractor } from '../neural/SmartRelationshipExtractor.js'
|
||||
import { NounType, VerbType } from '../types/graphTypes.js'
|
||||
import { PDFHandler } from '../augmentations/intelligentImport/handlers/pdfHandler.js'
|
||||
import type { FormatHandlerOptions } from '../augmentations/intelligentImport/types.js'
|
||||
import { PDFHandler } from './handlers/pdfHandler.js'
|
||||
import type { FormatHandlerOptions } from './handlers/types.js'
|
||||
|
||||
export interface SmartPDFOptions extends FormatHandlerOptions {
|
||||
/** Enable neural entity extraction */
|
||||
|
|
|
|||
204
src/importers/handlers/base.ts
Normal file
204
src/importers/handlers/base.ts
Normal file
|
|
@ -0,0 +1,204 @@
|
|||
/**
|
||||
* Base Format Handler
|
||||
* Abstract class providing common functionality for all format handlers
|
||||
*
|
||||
* Uses MimeTypeDetector for comprehensive file type detection (2000+ types)
|
||||
*/
|
||||
|
||||
import { FormatHandler, FormatHandlerOptions, ProcessedData } from './types.js'
|
||||
import { mimeDetector } from '../../vfs/MimeTypeDetector.js'
|
||||
|
||||
export abstract class BaseFormatHandler implements FormatHandler {
|
||||
abstract readonly format: string
|
||||
|
||||
abstract process(data: Buffer | string, options: FormatHandlerOptions): Promise<ProcessedData>
|
||||
|
||||
abstract canHandle(data: Buffer | string | { filename?: string, ext?: string }): boolean
|
||||
|
||||
/**
|
||||
* Detect file extension from various inputs
|
||||
*/
|
||||
protected detectExtension(data: Buffer | string | { filename?: string, ext?: string }): string | null {
|
||||
if (typeof data === 'object' && 'filename' in data && data.filename) {
|
||||
return this.getExtension(data.filename)
|
||||
}
|
||||
if (typeof data === 'object' && 'ext' in data && data.ext) {
|
||||
return data.ext.toLowerCase().replace(/^\./, '')
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
/**
|
||||
* Extract extension from filename
|
||||
*/
|
||||
protected getExtension(filename: string): string {
|
||||
const match = filename.match(/\.([^.]+)$/)
|
||||
return match ? match[1].toLowerCase() : ''
|
||||
}
|
||||
|
||||
/**
|
||||
* Get MIME type using MimeTypeDetector
|
||||
*
|
||||
* Supports 2000+ file types via mime library + custom developer types
|
||||
*/
|
||||
protected getMimeType(data: Buffer | string | { filename?: string }): string {
|
||||
if (typeof data === 'object' && 'filename' in data && data.filename) {
|
||||
return mimeDetector.detectMimeType(data.filename)
|
||||
}
|
||||
if (Buffer.isBuffer(data)) {
|
||||
// For buffers, we don't have a filename, so return generic
|
||||
return 'application/octet-stream'
|
||||
}
|
||||
return 'text/plain'
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if MIME type matches expected format
|
||||
*
|
||||
* @param mimeType - MIME type to check
|
||||
* @param patterns - Patterns to match (e.g., ['text/csv', 'application/vnd.ms-excel'])
|
||||
*/
|
||||
protected mimeTypeMatches(mimeType: string, patterns: string[]): boolean {
|
||||
return patterns.some(pattern => {
|
||||
if (pattern.endsWith('/*')) {
|
||||
const prefix = pattern.slice(0, -2)
|
||||
return mimeType.startsWith(prefix)
|
||||
}
|
||||
return mimeType === pattern
|
||||
})
|
||||
}
|
||||
|
||||
/**
|
||||
* Infer field types from data
|
||||
* Analyzes multiple rows to determine the most appropriate type
|
||||
*/
|
||||
protected inferFieldTypes(data: Array<Record<string, any>>): Record<string, string> {
|
||||
if (data.length === 0) return {}
|
||||
|
||||
const types: Record<string, string> = {}
|
||||
const firstRow = data[0]
|
||||
const sampleSize = Math.min(10, data.length)
|
||||
|
||||
for (const key of Object.keys(firstRow)) {
|
||||
// Check first few rows to get more accurate type
|
||||
const sampleTypes = new Set<string>()
|
||||
|
||||
for (let i = 0; i < sampleSize; i++) {
|
||||
const value = data[i][key]
|
||||
const type = this.inferType(value)
|
||||
sampleTypes.add(type)
|
||||
}
|
||||
|
||||
// If we see both integer and float, use float
|
||||
if (sampleTypes.has('float') || (sampleTypes.has('integer') && sampleTypes.has('float'))) {
|
||||
types[key] = 'float'
|
||||
} else if (sampleTypes.has('integer')) {
|
||||
types[key] = 'integer'
|
||||
} else if (sampleTypes.has('date')) {
|
||||
types[key] = 'date'
|
||||
} else if (sampleTypes.has('boolean')) {
|
||||
types[key] = 'boolean'
|
||||
} else {
|
||||
types[key] = 'string'
|
||||
}
|
||||
}
|
||||
|
||||
return types
|
||||
}
|
||||
|
||||
/**
|
||||
* Infer type of a single value
|
||||
*/
|
||||
protected inferType(value: any): string {
|
||||
if (value === null || value === undefined || value === '') return 'string'
|
||||
|
||||
if (typeof value === 'number') return 'number'
|
||||
if (typeof value === 'boolean') return 'boolean'
|
||||
|
||||
if (typeof value === 'string') {
|
||||
// Check if it's a number
|
||||
if (/^-?\d+$/.test(value)) return 'integer'
|
||||
if (/^-?\d+\.\d+$/.test(value)) return 'float'
|
||||
|
||||
// Check if it's a date
|
||||
if (this.isDateString(value)) return 'date'
|
||||
|
||||
// Check if it's a boolean
|
||||
if (/^(true|false|yes|no|y|n)$/i.test(value)) return 'boolean'
|
||||
}
|
||||
|
||||
return 'string'
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if string looks like a date
|
||||
*/
|
||||
protected isDateString(value: string): boolean {
|
||||
// ISO 8601
|
||||
if (/^\d{4}-\d{2}-\d{2}/.test(value)) return true
|
||||
|
||||
// Common date formats
|
||||
if (/^\d{1,2}\/\d{1,2}\/\d{2,4}$/.test(value)) return true
|
||||
if (/^\d{1,2}-\d{1,2}-\d{2,4}$/.test(value)) return true
|
||||
|
||||
return false
|
||||
}
|
||||
|
||||
/**
|
||||
* Sanitize field names for use as object keys
|
||||
*/
|
||||
protected sanitizeFieldName(name: string): string {
|
||||
return name
|
||||
.trim()
|
||||
.replace(/[^a-zA-Z0-9_\s-]/g, '')
|
||||
.replace(/\s+/g, '_')
|
||||
.replace(/-+/g, '_')
|
||||
.replace(/_+/g, '_')
|
||||
.replace(/^_|_$/g, '')
|
||||
|| 'field'
|
||||
}
|
||||
|
||||
/**
|
||||
* Convert value to appropriate type
|
||||
*/
|
||||
protected convertValue(value: any, type: string): any {
|
||||
if (value === null || value === undefined || value === '') return null
|
||||
|
||||
switch (type) {
|
||||
case 'integer':
|
||||
return parseInt(String(value), 10)
|
||||
|
||||
case 'float':
|
||||
case 'number':
|
||||
return parseFloat(String(value))
|
||||
|
||||
case 'boolean':
|
||||
if (typeof value === 'boolean') return value
|
||||
const str = String(value).toLowerCase()
|
||||
return ['true', 'yes', 'y', '1'].includes(str)
|
||||
|
||||
case 'date':
|
||||
return new Date(value)
|
||||
|
||||
default:
|
||||
return value
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Create metadata object with common fields
|
||||
*/
|
||||
protected createMetadata(
|
||||
rowCount: number,
|
||||
fields: string[],
|
||||
processingTime: number,
|
||||
extra: Record<string, any> = {}
|
||||
): ProcessedData['metadata'] {
|
||||
return {
|
||||
rowCount,
|
||||
fields,
|
||||
processingTime,
|
||||
...extra
|
||||
}
|
||||
}
|
||||
}
|
||||
249
src/importers/handlers/csvHandler.ts
Normal file
249
src/importers/handlers/csvHandler.ts
Normal file
|
|
@ -0,0 +1,249 @@
|
|||
/**
|
||||
* CSV Format Handler
|
||||
* Handles CSV files with:
|
||||
* - Automatic encoding detection
|
||||
* - Automatic delimiter detection
|
||||
* - Streaming for large files
|
||||
* - Type inference
|
||||
*/
|
||||
|
||||
import { parse } from 'csv-parse/sync'
|
||||
import { detect as detectEncoding } from 'chardet'
|
||||
import { BaseFormatHandler } from './base.js'
|
||||
import { FormatHandlerOptions, ProcessedData } from './types.js'
|
||||
|
||||
export class CSVHandler extends BaseFormatHandler {
|
||||
readonly format = 'csv'
|
||||
|
||||
canHandle(data: Buffer | string | { filename?: string, ext?: string }): boolean {
|
||||
const ext = this.detectExtension(data)
|
||||
if (ext === 'csv' || ext === 'tsv' || ext === 'txt') return true
|
||||
|
||||
// Check content if it's a buffer
|
||||
if (Buffer.isBuffer(data)) {
|
||||
const sample = data.slice(0, 1024).toString('utf-8')
|
||||
return this.looksLikeCSV(sample)
|
||||
}
|
||||
|
||||
if (typeof data === 'string') {
|
||||
return this.looksLikeCSV(data.slice(0, 1024))
|
||||
}
|
||||
|
||||
return false
|
||||
}
|
||||
|
||||
async process(data: Buffer | string, options: FormatHandlerOptions): Promise<ProcessedData> {
|
||||
const startTime = Date.now()
|
||||
const progressHooks = options.progressHooks
|
||||
|
||||
// Convert to buffer if string
|
||||
const buffer = Buffer.isBuffer(data) ? data : Buffer.from(data, 'utf-8')
|
||||
const totalBytes = buffer.length
|
||||
|
||||
// Report total bytes for progress tracking
|
||||
if (progressHooks?.onBytesProcessed) {
|
||||
progressHooks.onBytesProcessed(0)
|
||||
}
|
||||
if (progressHooks?.onCurrentItem) {
|
||||
progressHooks.onCurrentItem('Detecting CSV encoding and delimiter...')
|
||||
}
|
||||
|
||||
// Detect encoding
|
||||
const detectedEncoding = options.encoding || this.detectEncodingSafe(buffer)
|
||||
const text = buffer.toString(detectedEncoding as BufferEncoding)
|
||||
|
||||
// Detect delimiter if not specified
|
||||
const delimiter = options.csvDelimiter || this.detectDelimiter(text)
|
||||
|
||||
// Report progress - parsing started
|
||||
if (progressHooks?.onCurrentItem) {
|
||||
progressHooks.onCurrentItem(`Parsing CSV rows (delimiter: "${delimiter}")...`)
|
||||
}
|
||||
|
||||
// Parse CSV
|
||||
const hasHeaders = options.csvHeaders !== false
|
||||
const maxRows = options.maxRows
|
||||
|
||||
try {
|
||||
const records = parse(text, {
|
||||
columns: hasHeaders,
|
||||
skip_empty_lines: true,
|
||||
trim: true,
|
||||
delimiter,
|
||||
relax_column_count: true,
|
||||
to: maxRows,
|
||||
cast: false // We'll do type inference ourselves
|
||||
})
|
||||
|
||||
// Report bytes processed (entire file parsed)
|
||||
if (progressHooks?.onBytesProcessed) {
|
||||
progressHooks.onBytesProcessed(totalBytes)
|
||||
}
|
||||
|
||||
// Convert to array of objects
|
||||
const data = Array.isArray(records) ? records : [records]
|
||||
|
||||
// Report data extraction progress
|
||||
if (progressHooks?.onDataExtracted) {
|
||||
progressHooks.onDataExtracted(data.length, data.length)
|
||||
}
|
||||
if (progressHooks?.onCurrentItem) {
|
||||
progressHooks.onCurrentItem(`Extracted ${data.length} rows, inferring types...`)
|
||||
}
|
||||
|
||||
// Infer types and convert values
|
||||
const fields = data.length > 0 ? Object.keys(data[0]) : []
|
||||
const types = this.inferFieldTypes(data)
|
||||
|
||||
const convertedData = data.map((row, index) => {
|
||||
const converted: Record<string, any> = {}
|
||||
for (const [key, value] of Object.entries(row)) {
|
||||
converted[key] = this.convertValue(value, types[key] || 'string')
|
||||
}
|
||||
|
||||
// Report progress every 1000 rows
|
||||
if (progressHooks?.onCurrentItem && index > 0 && index % 1000 === 0) {
|
||||
progressHooks.onCurrentItem(`Converting types: ${index}/${data.length} rows...`)
|
||||
}
|
||||
|
||||
return converted
|
||||
})
|
||||
|
||||
const processingTime = Date.now() - startTime
|
||||
|
||||
// Final progress update
|
||||
if (progressHooks?.onCurrentItem) {
|
||||
progressHooks.onCurrentItem(`CSV processing complete: ${convertedData.length} rows`)
|
||||
}
|
||||
|
||||
return {
|
||||
format: this.format,
|
||||
data: convertedData,
|
||||
metadata: this.createMetadata(
|
||||
convertedData.length,
|
||||
fields,
|
||||
processingTime,
|
||||
{
|
||||
encoding: detectedEncoding,
|
||||
delimiter,
|
||||
hasHeaders,
|
||||
types
|
||||
}
|
||||
),
|
||||
filename: options.filename
|
||||
}
|
||||
} catch (error) {
|
||||
throw new Error(`CSV parsing failed: ${error instanceof Error ? error.message : String(error)}`)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if text looks like CSV
|
||||
*/
|
||||
private looksLikeCSV(text: string): boolean {
|
||||
const lines = text.split('\n').filter(l => l.trim())
|
||||
if (lines.length < 2) return false
|
||||
|
||||
// Check for common delimiters
|
||||
const delimiters = [',', ';', '\t', '|']
|
||||
for (const delimiter of delimiters) {
|
||||
const firstCount = (lines[0].match(new RegExp(`\\${delimiter}`, 'g')) || []).length
|
||||
if (firstCount === 0) continue
|
||||
|
||||
const secondCount = (lines[1].match(new RegExp(`\\${delimiter}`, 'g')) || []).length
|
||||
if (firstCount === secondCount) return true
|
||||
}
|
||||
|
||||
return false
|
||||
}
|
||||
|
||||
/**
|
||||
* Detect CSV delimiter
|
||||
*/
|
||||
private detectDelimiter(text: string): string {
|
||||
const sample = text.split('\n').slice(0, 10).join('\n')
|
||||
const delimiters = [',', ';', '\t', '|']
|
||||
const counts: Record<string, number> = {}
|
||||
|
||||
for (const delimiter of delimiters) {
|
||||
const lines = sample.split('\n').filter(l => l.trim())
|
||||
if (lines.length < 2) continue
|
||||
|
||||
// Count delimiter in first line
|
||||
const firstCount = (lines[0].match(new RegExp(`\\${delimiter}`, 'g')) || []).length
|
||||
if (firstCount === 0) continue
|
||||
|
||||
// Check if count is consistent across lines
|
||||
let consistent = true
|
||||
for (let i = 1; i < Math.min(5, lines.length); i++) {
|
||||
const count = (lines[i].match(new RegExp(`\\${delimiter}`, 'g')) || []).length
|
||||
if (count !== firstCount) {
|
||||
consistent = false
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
if (consistent) {
|
||||
counts[delimiter] = firstCount
|
||||
}
|
||||
}
|
||||
|
||||
// Return delimiter with highest count
|
||||
const best = Object.entries(counts).sort((a, b) => b[1] - a[1])[0]
|
||||
return best ? best[0] : ','
|
||||
}
|
||||
|
||||
/**
|
||||
* Detect encoding safely (with fallback)
|
||||
*/
|
||||
private detectEncodingSafe(buffer: Buffer): string {
|
||||
try {
|
||||
const detected = detectEncoding(buffer)
|
||||
if (!detected) return 'utf-8'
|
||||
|
||||
// Normalize encoding to Node.js-supported names
|
||||
return this.normalizeEncoding(detected)
|
||||
} catch {
|
||||
return 'utf-8'
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Normalize encoding names to Node.js-supported encodings
|
||||
*/
|
||||
private normalizeEncoding(encoding: string): string {
|
||||
const normalized = encoding.toLowerCase().replace(/[_-]/g, '')
|
||||
|
||||
// Map common encodings to Node.js names
|
||||
const mappings: Record<string, string> = {
|
||||
'iso88591': 'latin1',
|
||||
'iso88592': 'latin1',
|
||||
'iso88593': 'latin1',
|
||||
'iso88594': 'latin1',
|
||||
'iso88595': 'latin1',
|
||||
'iso88596': 'latin1',
|
||||
'iso88597': 'latin1',
|
||||
'iso88598': 'latin1',
|
||||
'iso88599': 'latin1',
|
||||
'iso885910': 'latin1',
|
||||
'iso885913': 'latin1',
|
||||
'iso885914': 'latin1',
|
||||
'iso885915': 'latin1',
|
||||
'iso885916': 'latin1',
|
||||
'usascii': 'ascii',
|
||||
'utf8': 'utf8',
|
||||
'utf16le': 'utf16le',
|
||||
'utf16be': 'utf16le',
|
||||
'windows1252': 'latin1',
|
||||
'windows1251': 'utf8', // Cyrillic - best effort
|
||||
'big5': 'utf8', // Chinese - best effort
|
||||
'gbk': 'utf8', // Chinese - best effort
|
||||
'gb2312': 'utf8', // Chinese - best effort
|
||||
'shiftjis': 'utf8', // Japanese - best effort
|
||||
'eucjp': 'utf8', // Japanese - best effort
|
||||
'euckr': 'utf8' // Korean - best effort
|
||||
}
|
||||
|
||||
return mappings[normalized] || 'utf8'
|
||||
}
|
||||
}
|
||||
237
src/importers/handlers/excelHandler.ts
Normal file
237
src/importers/handlers/excelHandler.ts
Normal file
|
|
@ -0,0 +1,237 @@
|
|||
/**
|
||||
* Excel Format Handler
|
||||
* Handles Excel files (.xlsx, .xls, .xlsb) with:
|
||||
* - Multi-sheet extraction
|
||||
* - Type inference
|
||||
* - Formula evaluation
|
||||
* - Metadata extraction
|
||||
*/
|
||||
|
||||
import * as XLSX from 'xlsx'
|
||||
import { BaseFormatHandler } from './base.js'
|
||||
import { FormatHandlerOptions, ProcessedData } from './types.js'
|
||||
|
||||
export class ExcelHandler extends BaseFormatHandler {
|
||||
readonly format = 'excel'
|
||||
|
||||
canHandle(data: Buffer | string | { filename?: string, ext?: string }): boolean {
|
||||
const ext = this.detectExtension(data)
|
||||
return ['xlsx', 'xls', 'xlsb', 'xlsm', 'xlt', 'xltx', 'xltm'].includes(ext || '')
|
||||
}
|
||||
|
||||
async process(data: Buffer | string, options: FormatHandlerOptions): Promise<ProcessedData> {
|
||||
const startTime = Date.now()
|
||||
const progressHooks = options.progressHooks
|
||||
|
||||
// Convert to buffer if string (though Excel should always be binary)
|
||||
const buffer = Buffer.isBuffer(data) ? data : Buffer.from(data, 'binary')
|
||||
const totalBytes = buffer.length
|
||||
|
||||
// Report start
|
||||
if (progressHooks?.onBytesProcessed) {
|
||||
progressHooks.onBytesProcessed(0)
|
||||
}
|
||||
if (progressHooks?.onCurrentItem) {
|
||||
progressHooks.onCurrentItem('Loading Excel workbook...')
|
||||
}
|
||||
|
||||
try {
|
||||
// Read workbook
|
||||
const workbook = XLSX.read(buffer, {
|
||||
type: 'buffer',
|
||||
cellDates: true,
|
||||
cellNF: true,
|
||||
cellStyles: true
|
||||
})
|
||||
|
||||
// Determine which sheets to process
|
||||
const sheetsToProcess = this.getSheetsToProcess(workbook, options)
|
||||
|
||||
// Report workbook loaded
|
||||
if (progressHooks?.onCurrentItem) {
|
||||
progressHooks.onCurrentItem(`Processing ${sheetsToProcess.length} sheets...`)
|
||||
}
|
||||
|
||||
// Extract data from sheets
|
||||
const allData: Array<Record<string, any>> = []
|
||||
const sheetMetadata: Record<string, any> = {}
|
||||
|
||||
for (let sheetIndex = 0; sheetIndex < sheetsToProcess.length; sheetIndex++) {
|
||||
const sheetName = sheetsToProcess[sheetIndex]
|
||||
|
||||
// Report current sheet
|
||||
if (progressHooks?.onCurrentItem) {
|
||||
progressHooks.onCurrentItem(
|
||||
`Reading sheet: ${sheetName} (${sheetIndex + 1}/${sheetsToProcess.length})`
|
||||
)
|
||||
}
|
||||
|
||||
const sheet = workbook.Sheets[sheetName]
|
||||
if (!sheet) continue
|
||||
|
||||
// Convert sheet to JSON with headers
|
||||
const sheetData = XLSX.utils.sheet_to_json(sheet, {
|
||||
header: 1, // Get as array of arrays first
|
||||
defval: null,
|
||||
blankrows: false,
|
||||
raw: false // Convert to formatted strings
|
||||
}) as any[][]
|
||||
|
||||
if (sheetData.length === 0) continue
|
||||
|
||||
// First row is headers
|
||||
const headers = sheetData[0].map((h: any) =>
|
||||
this.sanitizeFieldName(String(h || ''))
|
||||
)
|
||||
|
||||
// Skip if no headers
|
||||
if (headers.length === 0) continue
|
||||
|
||||
// Convert rows to objects
|
||||
for (let i = 1; i < sheetData.length; i++) {
|
||||
const row = sheetData[i]
|
||||
const rowObj: Record<string, any> = {}
|
||||
|
||||
// Add sheet name to each row
|
||||
rowObj._sheet = sheetName
|
||||
|
||||
for (let j = 0; j < headers.length; j++) {
|
||||
const header = headers[j]
|
||||
let value = row[j]
|
||||
|
||||
// Convert Excel dates
|
||||
if (value && typeof value === 'number' && this.isExcelDate(value)) {
|
||||
value = this.excelDateToJSDate(value)
|
||||
}
|
||||
|
||||
rowObj[header] = value === undefined ? null : value
|
||||
}
|
||||
|
||||
allData.push(rowObj)
|
||||
}
|
||||
|
||||
// Store sheet metadata
|
||||
sheetMetadata[sheetName] = {
|
||||
rowCount: sheetData.length - 1, // Exclude header row
|
||||
columnCount: headers.length,
|
||||
headers
|
||||
}
|
||||
|
||||
// Estimate bytes processed (sheets are sequential)
|
||||
const bytesProcessed = Math.floor(((sheetIndex + 1) / sheetsToProcess.length) * totalBytes)
|
||||
if (progressHooks?.onBytesProcessed) {
|
||||
progressHooks.onBytesProcessed(bytesProcessed)
|
||||
}
|
||||
|
||||
// Report extraction progress
|
||||
if (progressHooks?.onDataExtracted) {
|
||||
progressHooks.onDataExtracted(allData.length, undefined) // Total unknown until complete
|
||||
}
|
||||
}
|
||||
|
||||
// Report data extraction complete
|
||||
if (progressHooks?.onCurrentItem) {
|
||||
progressHooks.onCurrentItem(`Extracted ${allData.length} rows, inferring types...`)
|
||||
}
|
||||
if (progressHooks?.onDataExtracted) {
|
||||
progressHooks.onDataExtracted(allData.length, allData.length)
|
||||
}
|
||||
|
||||
// Infer types (excluding _sheet field)
|
||||
const fields = allData.length > 0 ? Object.keys(allData[0]).filter(k => k !== '_sheet') : []
|
||||
const types = this.inferFieldTypes(allData)
|
||||
|
||||
// Convert values to appropriate types
|
||||
const convertedData = allData.map((row, index) => {
|
||||
const converted: Record<string, any> = {}
|
||||
for (const [key, value] of Object.entries(row)) {
|
||||
if (key === '_sheet') {
|
||||
converted[key] = value
|
||||
} else {
|
||||
converted[key] = this.convertValue(value, types[key] || 'string')
|
||||
}
|
||||
}
|
||||
|
||||
// Report progress every 1000 rows (avoid spam)
|
||||
if (progressHooks?.onCurrentItem && index > 0 && index % 1000 === 0) {
|
||||
progressHooks.onCurrentItem(`Converting types: ${index}/${allData.length} rows...`)
|
||||
}
|
||||
|
||||
return converted
|
||||
})
|
||||
|
||||
// Final progress - all bytes processed
|
||||
if (progressHooks?.onBytesProcessed) {
|
||||
progressHooks.onBytesProcessed(totalBytes)
|
||||
}
|
||||
|
||||
const processingTime = Date.now() - startTime
|
||||
|
||||
// Report completion
|
||||
if (progressHooks?.onCurrentItem) {
|
||||
progressHooks.onCurrentItem(
|
||||
`Excel complete: ${sheetsToProcess.length} sheets, ${convertedData.length} rows`
|
||||
)
|
||||
}
|
||||
|
||||
return {
|
||||
format: this.format,
|
||||
data: convertedData,
|
||||
metadata: this.createMetadata(
|
||||
convertedData.length,
|
||||
fields,
|
||||
processingTime,
|
||||
{
|
||||
sheets: sheetsToProcess,
|
||||
sheetCount: sheetsToProcess.length,
|
||||
sheetMetadata,
|
||||
types,
|
||||
workbookInfo: {
|
||||
sheetNames: workbook.SheetNames,
|
||||
properties: workbook.Props || {}
|
||||
}
|
||||
}
|
||||
),
|
||||
filename: options.filename
|
||||
}
|
||||
} catch (error) {
|
||||
throw new Error(`Excel parsing failed: ${error instanceof Error ? error.message : String(error)}`)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Determine which sheets to process
|
||||
*/
|
||||
private getSheetsToProcess(workbook: XLSX.WorkBook, options: FormatHandlerOptions): string[] {
|
||||
const allSheets = workbook.SheetNames
|
||||
|
||||
// If specific sheets requested
|
||||
if (options.excelSheets && options.excelSheets !== 'all') {
|
||||
return options.excelSheets.filter(name => allSheets.includes(name))
|
||||
}
|
||||
|
||||
// Otherwise process all sheets
|
||||
return allSheets
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if a number is likely an Excel date
|
||||
* Excel stores dates as days since 1900-01-01
|
||||
*/
|
||||
private isExcelDate(value: number): boolean {
|
||||
// Excel dates are typically between 1 and 60000 (1900 to 2064)
|
||||
// This is a heuristic - not perfect but catches most cases
|
||||
return value > 0 && value < 100000 && Number.isInteger(value)
|
||||
}
|
||||
|
||||
/**
|
||||
* Convert Excel date (days since 1900-01-01) to JS Date
|
||||
*/
|
||||
private excelDateToJSDate(excelDate: number): Date {
|
||||
// Excel's epoch is 1900-01-01, but there's a bug where it thinks 1900 is a leap year
|
||||
// So dates before March 1, 1900 are off by one day
|
||||
const epoch = new Date(1899, 11, 30) // Dec 30, 1899
|
||||
const msPerDay = 24 * 60 * 60 * 1000
|
||||
return new Date(epoch.getTime() + excelDate * msPerDay)
|
||||
}
|
||||
}
|
||||
333
src/importers/handlers/pdfHandler.ts
Normal file
333
src/importers/handlers/pdfHandler.ts
Normal file
|
|
@ -0,0 +1,333 @@
|
|||
/**
|
||||
* PDF Format Handler
|
||||
* Handles PDF files with:
|
||||
* - Text extraction with layout preservation
|
||||
* - Table detection and extraction
|
||||
* - Metadata extraction (author, dates, etc.)
|
||||
* - Page-by-page processing
|
||||
*/
|
||||
|
||||
import * as pdfjsLib from 'pdfjs-dist/legacy/build/pdf.mjs'
|
||||
import { BaseFormatHandler } from './base.js'
|
||||
import { FormatHandlerOptions, ProcessedData } from './types.js'
|
||||
|
||||
// Use built-in worker for Node.js environments
|
||||
// In production, this can be customized via options
|
||||
const initializeWorker = () => {
|
||||
if (typeof pdfjsLib.GlobalWorkerOptions.workerSrc === 'undefined' ||
|
||||
pdfjsLib.GlobalWorkerOptions.workerSrc === '') {
|
||||
// Use a data URL to avoid file system dependencies
|
||||
// This tells pdfjs to use the built-in fallback worker
|
||||
try {
|
||||
pdfjsLib.GlobalWorkerOptions.workerSrc = 'data:,'
|
||||
} catch {
|
||||
// Ignore if already set or in incompatible environment
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
initializeWorker()
|
||||
|
||||
export class PDFHandler extends BaseFormatHandler {
|
||||
readonly format = 'pdf'
|
||||
|
||||
canHandle(data: Buffer | string | { filename?: string, ext?: string }): boolean {
|
||||
const ext = this.detectExtension(data)
|
||||
if (ext === 'pdf') return true
|
||||
|
||||
// Check for PDF magic bytes
|
||||
if (Buffer.isBuffer(data)) {
|
||||
const header = data.slice(0, 5).toString('ascii')
|
||||
return header === '%PDF-'
|
||||
}
|
||||
|
||||
return false
|
||||
}
|
||||
|
||||
async process(data: Buffer | string, options: FormatHandlerOptions): Promise<ProcessedData> {
|
||||
const startTime = Date.now()
|
||||
const progressHooks = options.progressHooks
|
||||
|
||||
// Convert to buffer
|
||||
const buffer = Buffer.isBuffer(data) ? data : Buffer.from(data, 'binary')
|
||||
const totalBytes = buffer.length
|
||||
|
||||
// Report start
|
||||
if (progressHooks?.onBytesProcessed) {
|
||||
progressHooks.onBytesProcessed(0)
|
||||
}
|
||||
if (progressHooks?.onCurrentItem) {
|
||||
progressHooks.onCurrentItem('Loading PDF document...')
|
||||
}
|
||||
|
||||
try {
|
||||
// Load PDF document
|
||||
const loadingTask = pdfjsLib.getDocument({
|
||||
data: new Uint8Array(buffer),
|
||||
useSystemFonts: true,
|
||||
standardFontDataUrl: undefined
|
||||
})
|
||||
|
||||
const pdfDoc = await loadingTask.promise
|
||||
|
||||
// Extract metadata
|
||||
const metadata = await pdfDoc.getMetadata()
|
||||
const numPages = pdfDoc.numPages
|
||||
|
||||
// Report document loaded
|
||||
if (progressHooks?.onCurrentItem) {
|
||||
progressHooks.onCurrentItem(`Processing ${numPages} pages...`)
|
||||
}
|
||||
|
||||
// Extract text and structure from all pages
|
||||
const allData: Array<Record<string, any>> = []
|
||||
let totalTextLength = 0
|
||||
let detectedTables = 0
|
||||
|
||||
for (let pageNum = 1; pageNum <= numPages; pageNum++) {
|
||||
// Report current page
|
||||
if (progressHooks?.onCurrentItem) {
|
||||
progressHooks.onCurrentItem(`Processing page ${pageNum} of ${numPages}`)
|
||||
}
|
||||
|
||||
const page = await pdfDoc.getPage(pageNum)
|
||||
const textContent = await page.getTextContent()
|
||||
|
||||
// Extract text items with positions
|
||||
const textItems = textContent.items.map((item: any) => ({
|
||||
text: item.str,
|
||||
x: item.transform[4],
|
||||
y: item.transform[5],
|
||||
width: item.width,
|
||||
height: item.height
|
||||
}))
|
||||
|
||||
// Combine text items into lines (group by similar Y position)
|
||||
const lines = this.groupIntoLines(textItems)
|
||||
|
||||
// Detect tables if requested
|
||||
if (options.pdfExtractTables !== false) {
|
||||
const tables = this.detectTables(lines)
|
||||
if (tables.length > 0) {
|
||||
detectedTables += tables.length
|
||||
for (const table of tables) {
|
||||
allData.push(...table.rows)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Extract paragraphs from non-table lines
|
||||
const paragraphs = this.extractParagraphs(lines)
|
||||
for (let i = 0; i < paragraphs.length; i++) {
|
||||
const text = paragraphs[i].trim()
|
||||
if (text.length > 0) {
|
||||
totalTextLength += text.length
|
||||
allData.push({
|
||||
_page: pageNum,
|
||||
_type: 'paragraph',
|
||||
_index: i,
|
||||
text
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// Estimate bytes processed (pages are sequential)
|
||||
const bytesProcessed = Math.floor((pageNum / numPages) * totalBytes)
|
||||
if (progressHooks?.onBytesProcessed) {
|
||||
progressHooks.onBytesProcessed(bytesProcessed)
|
||||
}
|
||||
|
||||
// Report extraction progress
|
||||
if (progressHooks?.onDataExtracted) {
|
||||
progressHooks.onDataExtracted(allData.length, undefined) // Total unknown until complete
|
||||
}
|
||||
}
|
||||
|
||||
// Final progress - all bytes processed
|
||||
if (progressHooks?.onBytesProcessed) {
|
||||
progressHooks.onBytesProcessed(totalBytes)
|
||||
}
|
||||
if (progressHooks?.onDataExtracted) {
|
||||
progressHooks.onDataExtracted(allData.length, allData.length)
|
||||
}
|
||||
|
||||
const processingTime = Date.now() - startTime
|
||||
|
||||
// Report completion
|
||||
if (progressHooks?.onCurrentItem) {
|
||||
progressHooks.onCurrentItem(
|
||||
`PDF complete: ${numPages} pages, ${allData.length} items extracted`
|
||||
)
|
||||
}
|
||||
|
||||
// Get all unique fields (excluding metadata fields)
|
||||
const fields = allData.length > 0
|
||||
? Object.keys(allData[0]).filter(k => !k.startsWith('_'))
|
||||
: []
|
||||
|
||||
return {
|
||||
format: this.format,
|
||||
data: allData,
|
||||
metadata: this.createMetadata(
|
||||
allData.length,
|
||||
fields,
|
||||
processingTime,
|
||||
{
|
||||
pageCount: numPages,
|
||||
textLength: totalTextLength,
|
||||
tableCount: detectedTables,
|
||||
pdfMetadata: {
|
||||
title: (metadata.info as any)?.Title || null,
|
||||
author: (metadata.info as any)?.Author || null,
|
||||
subject: (metadata.info as any)?.Subject || null,
|
||||
creator: (metadata.info as any)?.Creator || null,
|
||||
producer: (metadata.info as any)?.Producer || null,
|
||||
creationDate: (metadata.info as any)?.CreationDate || null,
|
||||
modificationDate: (metadata.info as any)?.ModDate || null
|
||||
}
|
||||
}
|
||||
),
|
||||
filename: options.filename
|
||||
}
|
||||
} catch (error) {
|
||||
throw new Error(`PDF parsing failed: ${error instanceof Error ? error.message : String(error)}`)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Group text items into lines based on Y position
|
||||
*/
|
||||
private groupIntoLines(items: Array<{ text: string, x: number, y: number, width: number, height: number }>): Array<Array<{ text: string, x: number }>> {
|
||||
if (items.length === 0) return []
|
||||
|
||||
// Sort by Y position (descending, since PDF coordinates go bottom-up)
|
||||
const sorted = [...items].sort((a, b) => b.y - a.y)
|
||||
|
||||
const lines: Array<Array<{ text: string, x: number }>> = []
|
||||
let currentLine: Array<{ text: string, x: number }> = []
|
||||
let currentY = sorted[0].y
|
||||
|
||||
for (const item of sorted) {
|
||||
// If Y position differs by more than half the height, it's a new line
|
||||
if (Math.abs(item.y - currentY) > (item.height / 2)) {
|
||||
if (currentLine.length > 0) {
|
||||
// Sort line items by X position
|
||||
currentLine.sort((a, b) => a.x - b.x)
|
||||
lines.push(currentLine)
|
||||
}
|
||||
currentLine = []
|
||||
currentY = item.y
|
||||
}
|
||||
|
||||
if (item.text.trim()) {
|
||||
currentLine.push({ text: item.text, x: item.x })
|
||||
}
|
||||
}
|
||||
|
||||
// Add last line
|
||||
if (currentLine.length > 0) {
|
||||
currentLine.sort((a, b) => a.x - b.x)
|
||||
lines.push(currentLine)
|
||||
}
|
||||
|
||||
return lines
|
||||
}
|
||||
|
||||
/**
|
||||
* Detect tables from lines
|
||||
* Tables are detected when multiple consecutive lines have similar structure
|
||||
*/
|
||||
private detectTables(lines: Array<Array<{ text: string, x: number }>>): Array<{ rows: Array<Record<string, any>> }> {
|
||||
const tables: Array<{ rows: Array<Record<string, any>> }> = []
|
||||
let potentialTable: Array<Array<{ text: string, x: number }>> = []
|
||||
|
||||
for (let i = 0; i < lines.length; i++) {
|
||||
const line = lines[i]
|
||||
|
||||
// A line with multiple items could be part of a table
|
||||
if (line.length >= 2) {
|
||||
potentialTable.push(line)
|
||||
} else {
|
||||
// End of potential table
|
||||
if (potentialTable.length >= 3) { // Need at least header + 2 rows
|
||||
const table = this.parseTable(potentialTable)
|
||||
if (table) {
|
||||
tables.push(table)
|
||||
}
|
||||
}
|
||||
potentialTable = []
|
||||
}
|
||||
}
|
||||
|
||||
// Check last potential table
|
||||
if (potentialTable.length >= 3) {
|
||||
const table = this.parseTable(potentialTable)
|
||||
if (table) {
|
||||
tables.push(table)
|
||||
}
|
||||
}
|
||||
|
||||
return tables
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse a potential table into structured rows
|
||||
*/
|
||||
private parseTable(lines: Array<Array<{ text: string, x: number }>>): { rows: Array<Record<string, any>> } | null {
|
||||
if (lines.length < 2) return null
|
||||
|
||||
// First line is headers
|
||||
const headerLine = lines[0]
|
||||
const headers = headerLine.map(item => this.sanitizeFieldName(item.text))
|
||||
|
||||
// Remaining lines are data
|
||||
const rows: Array<Record<string, any>> = []
|
||||
|
||||
for (let i = 1; i < lines.length; i++) {
|
||||
const line = lines[i]
|
||||
const row: Record<string, any> = { _type: 'table_row' }
|
||||
|
||||
// Match each item to closest header by X position
|
||||
for (let j = 0; j < line.length && j < headers.length; j++) {
|
||||
const header = headers[j]
|
||||
const value = line[j].text.trim()
|
||||
row[header] = value || null
|
||||
}
|
||||
|
||||
if (Object.keys(row).length > 1) { // More than just _type
|
||||
rows.push(row)
|
||||
}
|
||||
}
|
||||
|
||||
return rows.length > 0 ? { rows } : null
|
||||
}
|
||||
|
||||
/**
|
||||
* Extract paragraphs from lines
|
||||
*/
|
||||
private extractParagraphs(lines: Array<Array<{ text: string, x: number }>>): string[] {
|
||||
const paragraphs: string[] = []
|
||||
let currentParagraph: string[] = []
|
||||
|
||||
for (const line of lines) {
|
||||
const lineText = line.map(item => item.text).join(' ').trim()
|
||||
|
||||
if (lineText.length === 0) {
|
||||
// Empty line - end paragraph
|
||||
if (currentParagraph.length > 0) {
|
||||
paragraphs.push(currentParagraph.join(' '))
|
||||
currentParagraph = []
|
||||
}
|
||||
} else {
|
||||
currentParagraph.push(lineText)
|
||||
}
|
||||
}
|
||||
|
||||
// Add last paragraph
|
||||
if (currentParagraph.length > 0) {
|
||||
paragraphs.push(currentParagraph.join(' '))
|
||||
}
|
||||
|
||||
return paragraphs
|
||||
}
|
||||
}
|
||||
194
src/importers/handlers/types.ts
Normal file
194
src/importers/handlers/types.ts
Normal file
|
|
@ -0,0 +1,194 @@
|
|||
/**
|
||||
* Types for Intelligent Import Augmentation
|
||||
* Handles Excel, PDF, and CSV import with intelligent extraction
|
||||
*/
|
||||
|
||||
import { ImportProgressTracker } from '../../utils/import-progress-tracker.js'
|
||||
|
||||
/**
|
||||
* Progress hooks for format handlers
|
||||
*
|
||||
* Handlers call these hooks to report progress during processing.
|
||||
* This enables real-time progress tracking for any file format.
|
||||
*/
|
||||
export interface FormatHandlerProgressHooks {
|
||||
/**
|
||||
* Report bytes processed
|
||||
* Call this as you read/parse the file
|
||||
*/
|
||||
onBytesProcessed?: (bytes: number) => void
|
||||
|
||||
/**
|
||||
* Set current processing context
|
||||
* Examples: "Processing page 5", "Reading sheet: Q2 Sales"
|
||||
*/
|
||||
onCurrentItem?: (item: string) => void
|
||||
|
||||
/**
|
||||
* Report structured data extraction progress
|
||||
* Examples: "Extracted 100 rows", "Parsed 50 paragraphs"
|
||||
*/
|
||||
onDataExtracted?: (count: number, total?: number) => void
|
||||
}
|
||||
|
||||
export interface FormatHandler {
|
||||
/**
|
||||
* Format name (e.g., 'csv', 'xlsx', 'pdf')
|
||||
*/
|
||||
readonly format: string
|
||||
|
||||
/**
|
||||
* Process raw data into structured format
|
||||
* @param data Raw file data (Buffer or string)
|
||||
* @param options Format-specific options
|
||||
* @returns Structured data ready for entity extraction
|
||||
*/
|
||||
process(data: Buffer | string, options: FormatHandlerOptions): Promise<ProcessedData>
|
||||
|
||||
/**
|
||||
* Detect if this handler can process the given data
|
||||
* @param data Raw data or filename
|
||||
* @returns true if handler supports this format
|
||||
*/
|
||||
canHandle(data: Buffer | string | { filename?: string, ext?: string }): boolean
|
||||
}
|
||||
|
||||
export interface FormatHandlerOptions {
|
||||
/** Source filename (for extension detection) */
|
||||
filename?: string
|
||||
|
||||
/** File extension (if known) */
|
||||
ext?: string
|
||||
|
||||
/** Encoding (auto-detected if not specified) */
|
||||
encoding?: string
|
||||
|
||||
/** CSV-specific: delimiter character */
|
||||
csvDelimiter?: string
|
||||
|
||||
/** CSV-specific: whether first row is headers */
|
||||
csvHeaders?: boolean
|
||||
|
||||
/** Excel-specific: sheet names to extract (or 'all') */
|
||||
excelSheets?: string[] | 'all'
|
||||
|
||||
/** Excel-specific: whether to evaluate formulas */
|
||||
excelEvaluateFormulas?: boolean
|
||||
|
||||
/** PDF-specific: whether to extract tables */
|
||||
pdfExtractTables?: boolean
|
||||
|
||||
/** PDF-specific: whether to preserve layout */
|
||||
pdfPreserveLayout?: boolean
|
||||
|
||||
/** Maximum rows to process (for large files) */
|
||||
maxRows?: number
|
||||
|
||||
/** Whether to stream large files */
|
||||
streaming?: boolean
|
||||
|
||||
/**
|
||||
* Progress hooks
|
||||
* Handlers call these to report progress during processing
|
||||
*/
|
||||
progressHooks?: FormatHandlerProgressHooks
|
||||
|
||||
/**
|
||||
* Total file size in bytes
|
||||
* Used for progress percentage calculation
|
||||
*/
|
||||
totalBytes?: number
|
||||
}
|
||||
|
||||
export interface ProcessedData {
|
||||
/** Format that was processed */
|
||||
format: string
|
||||
|
||||
/** Structured data (array of objects) */
|
||||
data: Array<Record<string, any>>
|
||||
|
||||
/** Metadata about the processed data */
|
||||
metadata: {
|
||||
/** Number of rows/entities extracted */
|
||||
rowCount: number
|
||||
|
||||
/** Column/field names */
|
||||
fields: string[]
|
||||
|
||||
/** Detected encoding (for text formats) */
|
||||
encoding?: string
|
||||
|
||||
/** Excel: sheet names */
|
||||
sheets?: string[]
|
||||
|
||||
/** PDF: page count */
|
||||
pageCount?: number
|
||||
|
||||
/** PDF: extracted text length */
|
||||
textLength?: number
|
||||
|
||||
/** PDF: number of tables detected */
|
||||
tableCount?: number
|
||||
|
||||
/** Processing time in milliseconds */
|
||||
processingTime: number
|
||||
|
||||
/** Any warnings during processing */
|
||||
warnings?: string[]
|
||||
|
||||
/** Format-specific metadata */
|
||||
[key: string]: any
|
||||
}
|
||||
|
||||
/** Original filename (if available) */
|
||||
filename?: string
|
||||
}
|
||||
|
||||
export interface HandlerRegistry {
|
||||
/** Registered handlers by format extension */
|
||||
handlers: Map<string, () => Promise<FormatHandler>>
|
||||
|
||||
/** Loaded handler instances (lazy-loaded) */
|
||||
loaded: Map<string, FormatHandler>
|
||||
|
||||
/** Register a new handler */
|
||||
register(extensions: string[], loader: () => Promise<FormatHandler>): void
|
||||
|
||||
/** Get handler for a file/format */
|
||||
getHandler(filenameOrExt: string): Promise<FormatHandler | null>
|
||||
}
|
||||
|
||||
export interface IntelligentImportConfig {
|
||||
/** Enable CSV handler */
|
||||
enableCSV: boolean
|
||||
|
||||
/** Enable Excel handler */
|
||||
enableExcel: boolean
|
||||
|
||||
/** Enable PDF handler */
|
||||
enablePDF: boolean
|
||||
|
||||
/** Enable Image handler */
|
||||
enableImage: boolean
|
||||
|
||||
/** Default options for CSV */
|
||||
csvDefaults?: Partial<FormatHandlerOptions>
|
||||
|
||||
/** Default options for Excel */
|
||||
excelDefaults?: Partial<FormatHandlerOptions>
|
||||
|
||||
/** Default options for PDF */
|
||||
pdfDefaults?: Partial<FormatHandlerOptions>
|
||||
|
||||
/** Default options for Image */
|
||||
imageDefaults?: Partial<FormatHandlerOptions>
|
||||
|
||||
/** Maximum file size to process (bytes) */
|
||||
maxFileSize?: number
|
||||
|
||||
/** Enable caching of processed data */
|
||||
enableCache?: boolean
|
||||
|
||||
/** Cache TTL in milliseconds */
|
||||
cacheTTL?: number
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue