feat: add intelligent import for CSV, Excel, and PDF files

Add IntelligentImportAugmentation with support for:
- CSV: auto-detection of encoding, delimiters, and field types
- Excel: multi-sheet extraction with metadata preservation
- PDF: text extraction, table detection, and metadata extraction

Features:
- Automatic format detection from file extension or content
- Intelligent type inference (string, number, boolean, date)
- Seamless integration with neural entity extraction
- Production-ready with 69 comprehensive tests

Dependencies added:
- xlsx@^0.18.5 for Excel parsing
- pdfjs-dist@^4.0.379 for PDF parsing
- csv-parse@^6.1.0 for CSV parsing
- chardet@^2.0.0 for encoding detection

Documentation:
- Updated README with import examples
- Updated API_REFERENCE with comprehensive import() docs
- Updated import-anything.md guide
- Added working example: import-excel-pdf-csv.ts
- Updated augmentations README
This commit is contained in:
David Snelling 2025-10-01 16:51:03 -07:00
parent aaf8e0f411
commit 814cbb48ee
33 changed files with 4664 additions and 28 deletions

View file

@ -14,6 +14,7 @@ import { CacheAugmentation } from './cacheAugmentation.js'
import { MetricsAugmentation } from './metricsAugmentation.js'
import { MonitoringAugmentation } from './monitoringAugmentation.js'
import { UniversalDisplayAugmentation } from './universalDisplayAugmentation.js'
import { IntelligentImportAugmentation } from './intelligentImport/index.js'
/**
* Create default augmentations for zero-config operation
@ -28,10 +29,17 @@ export function createDefaultAugmentations(
metrics?: boolean | Record<string, any>
monitoring?: boolean | Record<string, any>
display?: boolean | Record<string, any>
intelligentImport?: boolean | Record<string, any>
} = {}
): BaseAugmentation[] {
const augmentations: BaseAugmentation[] = []
// Intelligent Import augmentation (CSV, Excel, PDF)
if (config.intelligentImport !== false) {
const importConfig = typeof config.intelligentImport === 'object' ? config.intelligentImport : {}
augmentations.push(new IntelligentImportAugmentation(importConfig))
}
// Cache augmentation (was SearchCache)
if (config.cache !== false) {
const cacheConfig = typeof config.cache === 'object' ? config.cache : {}
@ -111,5 +119,12 @@ export const AugmentationHelpers = {
*/
getDisplay(brain: Brainy): UniversalDisplayAugmentation | null {
return getAugmentation<UniversalDisplayAugmentation>(brain, 'display')
},
/**
* Get intelligent import augmentation
*/
getIntelligentImport(brain: Brainy): IntelligentImportAugmentation | null {
return getAugmentation<IntelligentImportAugmentation>(brain, 'intelligent-import')
}
}