feat: add intelligent import for CSV, Excel, and PDF files
Add IntelligentImportAugmentation with support for: - CSV: auto-detection of encoding, delimiters, and field types - Excel: multi-sheet extraction with metadata preservation - PDF: text extraction, table detection, and metadata extraction Features: - Automatic format detection from file extension or content - Intelligent type inference (string, number, boolean, date) - Seamless integration with neural entity extraction - Production-ready with 69 comprehensive tests Dependencies added: - xlsx@^0.18.5 for Excel parsing - pdfjs-dist@^4.0.379 for PDF parsing - csv-parse@^6.1.0 for CSV parsing - chardet@^2.0.0 for encoding detection Documentation: - Updated README with import examples - Updated API_REFERENCE with comprehensive import() docs - Updated import-anything.md guide - Added working example: import-excel-pdf-csv.ts - Updated augmentations README
This commit is contained in:
parent
aaf8e0f411
commit
814cbb48ee
33 changed files with 4664 additions and 28 deletions
327
tests/fixtures/import/multi-page.pdf
vendored
Normal file
327
tests/fixtures/import/multi-page.pdf
vendored
Normal file
|
|
@ -0,0 +1,327 @@
|
|||
%PDF-1.3
|
||||
%ºß¬à
|
||||
3 0 obj
|
||||
<</Type /Page
|
||||
/Parent 1 0 R
|
||||
/Resources 2 0 R
|
||||
/MediaBox [0 0 595.2799999999999727 841.8899999999999864]
|
||||
/Contents 4 0 R
|
||||
>>
|
||||
endobj
|
||||
4 0 obj
|
||||
<<
|
||||
/Length 437
|
||||
>>
|
||||
stream
|
||||
0.5670000000000001 w
|
||||
0 G
|
||||
BT
|
||||
/F1 16 Tf
|
||||
18.3999999999999986 TL
|
||||
0 g
|
||||
56.6929133858267775 785.1970866141732586 Td
|
||||
(Page 1: Introduction) Tj
|
||||
ET
|
||||
BT
|
||||
/F1 12 Tf
|
||||
13.7999999999999989 TL
|
||||
0 g
|
||||
56.6929133858267775 728.5041732283464171 Td
|
||||
(This is the first page of a multi-page PDF document.) Tj
|
||||
ET
|
||||
BT
|
||||
/F1 12 Tf
|
||||
13.7999999999999989 TL
|
||||
0 g
|
||||
56.6929133858267775 700.15771653543311 Td
|
||||
(It demonstrates that the PDF handler can process multiple pages.) Tj
|
||||
ET
|
||||
endstream
|
||||
endobj
|
||||
5 0 obj
|
||||
<</Type /Page
|
||||
/Parent 1 0 R
|
||||
/Resources 2 0 R
|
||||
/MediaBox [0 0 595.2799999999999727 841.8899999999999864]
|
||||
/Contents 6 0 R
|
||||
>>
|
||||
endobj
|
||||
6 0 obj
|
||||
<<
|
||||
/Length 404
|
||||
>>
|
||||
stream
|
||||
0.5670000000000001 w
|
||||
0 G
|
||||
BT
|
||||
/F1 16 Tf
|
||||
18.3999999999999986 TL
|
||||
0 g
|
||||
56.6929133858267775 785.1970866141732586 Td
|
||||
(Page 2: Content) Tj
|
||||
ET
|
||||
BT
|
||||
/F1 12 Tf
|
||||
13.7999999999999989 TL
|
||||
0 g
|
||||
56.6929133858267775 728.5041732283464171 Td
|
||||
(This is the second page with different content.) Tj
|
||||
ET
|
||||
BT
|
||||
/F1 12 Tf
|
||||
13.7999999999999989 TL
|
||||
0 g
|
||||
56.6929133858267775 700.15771653543311 Td
|
||||
(Each page should be extracted separately.) Tj
|
||||
ET
|
||||
endstream
|
||||
endobj
|
||||
7 0 obj
|
||||
<</Type /Page
|
||||
/Parent 1 0 R
|
||||
/Resources 2 0 R
|
||||
/MediaBox [0 0 595.2799999999999727 841.8899999999999864]
|
||||
/Contents 8 0 R
|
||||
>>
|
||||
endobj
|
||||
8 0 obj
|
||||
<<
|
||||
/Length 267
|
||||
>>
|
||||
stream
|
||||
0.5670000000000001 w
|
||||
0 G
|
||||
BT
|
||||
/F1 16 Tf
|
||||
18.3999999999999986 TL
|
||||
0 g
|
||||
56.6929133858267775 785.1970866141732586 Td
|
||||
(Page 3: Conclusion) Tj
|
||||
ET
|
||||
BT
|
||||
/F1 12 Tf
|
||||
13.7999999999999989 TL
|
||||
0 g
|
||||
56.6929133858267775 728.5041732283464171 Td
|
||||
(This is the final page of the document.) Tj
|
||||
ET
|
||||
endstream
|
||||
endobj
|
||||
1 0 obj
|
||||
<</Type /Pages
|
||||
/Kids [3 0 R 5 0 R 7 0 R ]
|
||||
/Count 3
|
||||
>>
|
||||
endobj
|
||||
9 0 obj
|
||||
<<
|
||||
/Type /Font
|
||||
/BaseFont /Helvetica
|
||||
/Subtype /Type1
|
||||
/Encoding /WinAnsiEncoding
|
||||
/FirstChar 32
|
||||
/LastChar 255
|
||||
>>
|
||||
endobj
|
||||
10 0 obj
|
||||
<<
|
||||
/Type /Font
|
||||
/BaseFont /Helvetica-Bold
|
||||
/Subtype /Type1
|
||||
/Encoding /WinAnsiEncoding
|
||||
/FirstChar 32
|
||||
/LastChar 255
|
||||
>>
|
||||
endobj
|
||||
11 0 obj
|
||||
<<
|
||||
/Type /Font
|
||||
/BaseFont /Helvetica-Oblique
|
||||
/Subtype /Type1
|
||||
/Encoding /WinAnsiEncoding
|
||||
/FirstChar 32
|
||||
/LastChar 255
|
||||
>>
|
||||
endobj
|
||||
12 0 obj
|
||||
<<
|
||||
/Type /Font
|
||||
/BaseFont /Helvetica-BoldOblique
|
||||
/Subtype /Type1
|
||||
/Encoding /WinAnsiEncoding
|
||||
/FirstChar 32
|
||||
/LastChar 255
|
||||
>>
|
||||
endobj
|
||||
13 0 obj
|
||||
<<
|
||||
/Type /Font
|
||||
/BaseFont /Courier
|
||||
/Subtype /Type1
|
||||
/Encoding /WinAnsiEncoding
|
||||
/FirstChar 32
|
||||
/LastChar 255
|
||||
>>
|
||||
endobj
|
||||
14 0 obj
|
||||
<<
|
||||
/Type /Font
|
||||
/BaseFont /Courier-Bold
|
||||
/Subtype /Type1
|
||||
/Encoding /WinAnsiEncoding
|
||||
/FirstChar 32
|
||||
/LastChar 255
|
||||
>>
|
||||
endobj
|
||||
15 0 obj
|
||||
<<
|
||||
/Type /Font
|
||||
/BaseFont /Courier-Oblique
|
||||
/Subtype /Type1
|
||||
/Encoding /WinAnsiEncoding
|
||||
/FirstChar 32
|
||||
/LastChar 255
|
||||
>>
|
||||
endobj
|
||||
16 0 obj
|
||||
<<
|
||||
/Type /Font
|
||||
/BaseFont /Courier-BoldOblique
|
||||
/Subtype /Type1
|
||||
/Encoding /WinAnsiEncoding
|
||||
/FirstChar 32
|
||||
/LastChar 255
|
||||
>>
|
||||
endobj
|
||||
17 0 obj
|
||||
<<
|
||||
/Type /Font
|
||||
/BaseFont /Times-Roman
|
||||
/Subtype /Type1
|
||||
/Encoding /WinAnsiEncoding
|
||||
/FirstChar 32
|
||||
/LastChar 255
|
||||
>>
|
||||
endobj
|
||||
18 0 obj
|
||||
<<
|
||||
/Type /Font
|
||||
/BaseFont /Times-Bold
|
||||
/Subtype /Type1
|
||||
/Encoding /WinAnsiEncoding
|
||||
/FirstChar 32
|
||||
/LastChar 255
|
||||
>>
|
||||
endobj
|
||||
19 0 obj
|
||||
<<
|
||||
/Type /Font
|
||||
/BaseFont /Times-Italic
|
||||
/Subtype /Type1
|
||||
/Encoding /WinAnsiEncoding
|
||||
/FirstChar 32
|
||||
/LastChar 255
|
||||
>>
|
||||
endobj
|
||||
20 0 obj
|
||||
<<
|
||||
/Type /Font
|
||||
/BaseFont /Times-BoldItalic
|
||||
/Subtype /Type1
|
||||
/Encoding /WinAnsiEncoding
|
||||
/FirstChar 32
|
||||
/LastChar 255
|
||||
>>
|
||||
endobj
|
||||
21 0 obj
|
||||
<<
|
||||
/Type /Font
|
||||
/BaseFont /ZapfDingbats
|
||||
/Subtype /Type1
|
||||
/FirstChar 32
|
||||
/LastChar 255
|
||||
>>
|
||||
endobj
|
||||
22 0 obj
|
||||
<<
|
||||
/Type /Font
|
||||
/BaseFont /Symbol
|
||||
/Subtype /Type1
|
||||
/FirstChar 32
|
||||
/LastChar 255
|
||||
>>
|
||||
endobj
|
||||
2 0 obj
|
||||
<<
|
||||
/ProcSet [/PDF /Text /ImageB /ImageC /ImageI]
|
||||
/Font <<
|
||||
/F1 9 0 R
|
||||
/F2 10 0 R
|
||||
/F3 11 0 R
|
||||
/F4 12 0 R
|
||||
/F5 13 0 R
|
||||
/F6 14 0 R
|
||||
/F7 15 0 R
|
||||
/F8 16 0 R
|
||||
/F9 17 0 R
|
||||
/F10 18 0 R
|
||||
/F11 19 0 R
|
||||
/F12 20 0 R
|
||||
/F13 21 0 R
|
||||
/F14 22 0 R
|
||||
>>
|
||||
/XObject <<
|
||||
>>
|
||||
>>
|
||||
endobj
|
||||
23 0 obj
|
||||
<<
|
||||
/Producer (jsPDF 3.0.3)
|
||||
/CreationDate (D:20251001162659-07'00')
|
||||
>>
|
||||
endobj
|
||||
24 0 obj
|
||||
<<
|
||||
/Type /Catalog
|
||||
/Pages 1 0 R
|
||||
/OpenAction [3 0 R /FitH null]
|
||||
/PageLayout /OneColumn
|
||||
>>
|
||||
endobj
|
||||
xref
|
||||
0 25
|
||||
0000000000 65535 f
|
||||
0000001687 00000 n
|
||||
0000003520 00000 n
|
||||
0000000015 00000 n
|
||||
0000000152 00000 n
|
||||
0000000640 00000 n
|
||||
0000000777 00000 n
|
||||
0000001232 00000 n
|
||||
0000001369 00000 n
|
||||
0000001756 00000 n
|
||||
0000001881 00000 n
|
||||
0000002012 00000 n
|
||||
0000002146 00000 n
|
||||
0000002284 00000 n
|
||||
0000002408 00000 n
|
||||
0000002537 00000 n
|
||||
0000002669 00000 n
|
||||
0000002805 00000 n
|
||||
0000002933 00000 n
|
||||
0000003060 00000 n
|
||||
0000003189 00000 n
|
||||
0000003322 00000 n
|
||||
0000003424 00000 n
|
||||
0000003772 00000 n
|
||||
0000003858 00000 n
|
||||
trailer
|
||||
<<
|
||||
/Size 25
|
||||
/Root 24 0 R
|
||||
/Info 23 0 R
|
||||
/ID [ <04E6AF3F674215986B311755247B62DD> <04E6AF3F674215986B311755247B62DD> ]
|
||||
>>
|
||||
startxref
|
||||
3962
|
||||
%%EOF
|
||||
Loading…
Add table
Add a link
Reference in a new issue