feat: add intelligent import for CSV, Excel, and PDF files
Add IntelligentImportAugmentation with support for: - CSV: auto-detection of encoding, delimiters, and field types - Excel: multi-sheet extraction with metadata preservation - PDF: text extraction, table detection, and metadata extraction Features: - Automatic format detection from file extension or content - Intelligent type inference (string, number, boolean, date) - Seamless integration with neural entity extraction - Production-ready with 69 comprehensive tests Dependencies added: - xlsx@^0.18.5 for Excel parsing - pdfjs-dist@^4.0.379 for PDF parsing - csv-parse@^6.1.0 for CSV parsing - chardet@^2.0.0 for encoding detection Documentation: - Updated README with import examples - Updated API_REFERENCE with comprehensive import() docs - Updated import-anything.md guide - Added working example: import-excel-pdf-csv.ts - Updated augmentations README
This commit is contained in:
parent
aaf8e0f411
commit
814cbb48ee
33 changed files with 4664 additions and 28 deletions
233
tests/fixtures/import/empty.pdf
vendored
Normal file
233
tests/fixtures/import/empty.pdf
vendored
Normal file
|
|
@ -0,0 +1,233 @@
|
|||
%PDF-1.3
|
||||
%ºß¬à
|
||||
3 0 obj
|
||||
<</Type /Page
|
||||
/Parent 1 0 R
|
||||
/Resources 2 0 R
|
||||
/MediaBox [0 0 595.2799999999999727 841.8899999999999864]
|
||||
/Contents 4 0 R
|
||||
>>
|
||||
endobj
|
||||
4 0 obj
|
||||
<<
|
||||
/Length 24
|
||||
>>
|
||||
stream
|
||||
0.5670000000000001 w
|
||||
0 G
|
||||
endstream
|
||||
endobj
|
||||
1 0 obj
|
||||
<</Type /Pages
|
||||
/Kids [3 0 R ]
|
||||
/Count 1
|
||||
>>
|
||||
endobj
|
||||
5 0 obj
|
||||
<<
|
||||
/Type /Font
|
||||
/BaseFont /Helvetica
|
||||
/Subtype /Type1
|
||||
/Encoding /WinAnsiEncoding
|
||||
/FirstChar 32
|
||||
/LastChar 255
|
||||
>>
|
||||
endobj
|
||||
6 0 obj
|
||||
<<
|
||||
/Type /Font
|
||||
/BaseFont /Helvetica-Bold
|
||||
/Subtype /Type1
|
||||
/Encoding /WinAnsiEncoding
|
||||
/FirstChar 32
|
||||
/LastChar 255
|
||||
>>
|
||||
endobj
|
||||
7 0 obj
|
||||
<<
|
||||
/Type /Font
|
||||
/BaseFont /Helvetica-Oblique
|
||||
/Subtype /Type1
|
||||
/Encoding /WinAnsiEncoding
|
||||
/FirstChar 32
|
||||
/LastChar 255
|
||||
>>
|
||||
endobj
|
||||
8 0 obj
|
||||
<<
|
||||
/Type /Font
|
||||
/BaseFont /Helvetica-BoldOblique
|
||||
/Subtype /Type1
|
||||
/Encoding /WinAnsiEncoding
|
||||
/FirstChar 32
|
||||
/LastChar 255
|
||||
>>
|
||||
endobj
|
||||
9 0 obj
|
||||
<<
|
||||
/Type /Font
|
||||
/BaseFont /Courier
|
||||
/Subtype /Type1
|
||||
/Encoding /WinAnsiEncoding
|
||||
/FirstChar 32
|
||||
/LastChar 255
|
||||
>>
|
||||
endobj
|
||||
10 0 obj
|
||||
<<
|
||||
/Type /Font
|
||||
/BaseFont /Courier-Bold
|
||||
/Subtype /Type1
|
||||
/Encoding /WinAnsiEncoding
|
||||
/FirstChar 32
|
||||
/LastChar 255
|
||||
>>
|
||||
endobj
|
||||
11 0 obj
|
||||
<<
|
||||
/Type /Font
|
||||
/BaseFont /Courier-Oblique
|
||||
/Subtype /Type1
|
||||
/Encoding /WinAnsiEncoding
|
||||
/FirstChar 32
|
||||
/LastChar 255
|
||||
>>
|
||||
endobj
|
||||
12 0 obj
|
||||
<<
|
||||
/Type /Font
|
||||
/BaseFont /Courier-BoldOblique
|
||||
/Subtype /Type1
|
||||
/Encoding /WinAnsiEncoding
|
||||
/FirstChar 32
|
||||
/LastChar 255
|
||||
>>
|
||||
endobj
|
||||
13 0 obj
|
||||
<<
|
||||
/Type /Font
|
||||
/BaseFont /Times-Roman
|
||||
/Subtype /Type1
|
||||
/Encoding /WinAnsiEncoding
|
||||
/FirstChar 32
|
||||
/LastChar 255
|
||||
>>
|
||||
endobj
|
||||
14 0 obj
|
||||
<<
|
||||
/Type /Font
|
||||
/BaseFont /Times-Bold
|
||||
/Subtype /Type1
|
||||
/Encoding /WinAnsiEncoding
|
||||
/FirstChar 32
|
||||
/LastChar 255
|
||||
>>
|
||||
endobj
|
||||
15 0 obj
|
||||
<<
|
||||
/Type /Font
|
||||
/BaseFont /Times-Italic
|
||||
/Subtype /Type1
|
||||
/Encoding /WinAnsiEncoding
|
||||
/FirstChar 32
|
||||
/LastChar 255
|
||||
>>
|
||||
endobj
|
||||
16 0 obj
|
||||
<<
|
||||
/Type /Font
|
||||
/BaseFont /Times-BoldItalic
|
||||
/Subtype /Type1
|
||||
/Encoding /WinAnsiEncoding
|
||||
/FirstChar 32
|
||||
/LastChar 255
|
||||
>>
|
||||
endobj
|
||||
17 0 obj
|
||||
<<
|
||||
/Type /Font
|
||||
/BaseFont /ZapfDingbats
|
||||
/Subtype /Type1
|
||||
/FirstChar 32
|
||||
/LastChar 255
|
||||
>>
|
||||
endobj
|
||||
18 0 obj
|
||||
<<
|
||||
/Type /Font
|
||||
/BaseFont /Symbol
|
||||
/Subtype /Type1
|
||||
/FirstChar 32
|
||||
/LastChar 255
|
||||
>>
|
||||
endobj
|
||||
2 0 obj
|
||||
<<
|
||||
/ProcSet [/PDF /Text /ImageB /ImageC /ImageI]
|
||||
/Font <<
|
||||
/F1 5 0 R
|
||||
/F2 6 0 R
|
||||
/F3 7 0 R
|
||||
/F4 8 0 R
|
||||
/F5 9 0 R
|
||||
/F6 10 0 R
|
||||
/F7 11 0 R
|
||||
/F8 12 0 R
|
||||
/F9 13 0 R
|
||||
/F10 14 0 R
|
||||
/F11 15 0 R
|
||||
/F12 16 0 R
|
||||
/F13 17 0 R
|
||||
/F14 18 0 R
|
||||
>>
|
||||
/XObject <<
|
||||
>>
|
||||
>>
|
||||
endobj
|
||||
19 0 obj
|
||||
<<
|
||||
/Producer (jsPDF 3.0.3)
|
||||
/CreationDate (D:20251001162659-07'00')
|
||||
>>
|
||||
endobj
|
||||
20 0 obj
|
||||
<<
|
||||
/Type /Catalog
|
||||
/Pages 1 0 R
|
||||
/OpenAction [3 0 R /FitH null]
|
||||
/PageLayout /OneColumn
|
||||
>>
|
||||
endobj
|
||||
xref
|
||||
0 21
|
||||
0000000000 65535 f
|
||||
0000000226 00000 n
|
||||
0000002043 00000 n
|
||||
0000000015 00000 n
|
||||
0000000152 00000 n
|
||||
0000000283 00000 n
|
||||
0000000408 00000 n
|
||||
0000000538 00000 n
|
||||
0000000671 00000 n
|
||||
0000000808 00000 n
|
||||
0000000931 00000 n
|
||||
0000001060 00000 n
|
||||
0000001192 00000 n
|
||||
0000001328 00000 n
|
||||
0000001456 00000 n
|
||||
0000001583 00000 n
|
||||
0000001712 00000 n
|
||||
0000001845 00000 n
|
||||
0000001947 00000 n
|
||||
0000002291 00000 n
|
||||
0000002377 00000 n
|
||||
trailer
|
||||
<<
|
||||
/Size 21
|
||||
/Root 20 0 R
|
||||
/Info 19 0 R
|
||||
/ID [ <49861707DD6C8FE0691713F0D6CEE800> <49861707DD6C8FE0691713F0D6CEE800> ]
|
||||
>>
|
||||
startxref
|
||||
2481
|
||||
%%EOF
|
||||
Loading…
Add table
Add a link
Reference in a new issue