feat: add intelligent import for CSV, Excel, and PDF files
Add IntelligentImportAugmentation with support for: - CSV: auto-detection of encoding, delimiters, and field types - Excel: multi-sheet extraction with metadata preservation - PDF: text extraction, table detection, and metadata extraction Features: - Automatic format detection from file extension or content - Intelligent type inference (string, number, boolean, date) - Seamless integration with neural entity extraction - Production-ready with 69 comprehensive tests Dependencies added: - xlsx@^0.18.5 for Excel parsing - pdfjs-dist@^4.0.379 for PDF parsing - csv-parse@^6.1.0 for CSV parsing - chardet@^2.0.0 for encoding detection Documentation: - Updated README with import examples - Updated API_REFERENCE with comprehensive import() docs - Updated import-anything.md guide - Added working example: import-excel-pdf-csv.ts - Updated augmentations README
This commit is contained in:
parent
aaf8e0f411
commit
814cbb48ee
33 changed files with 4664 additions and 28 deletions
275
tests/fixtures/import/simple.pdf
vendored
Normal file
275
tests/fixtures/import/simple.pdf
vendored
Normal file
|
|
@ -0,0 +1,275 @@
|
|||
%PDF-1.3
|
||||
%ºß¬à
|
||||
3 0 obj
|
||||
<</Type /Page
|
||||
/Parent 1 0 R
|
||||
/Resources 2 0 R
|
||||
/MediaBox [0 0 595.2799999999999727 841.8899999999999864]
|
||||
/Contents 4 0 R
|
||||
>>
|
||||
endobj
|
||||
4 0 obj
|
||||
<<
|
||||
/Length 847
|
||||
>>
|
||||
stream
|
||||
0.5670000000000001 w
|
||||
0 G
|
||||
BT
|
||||
/F1 16 Tf
|
||||
18.3999999999999986 TL
|
||||
0 g
|
||||
56.6929133858267775 785.1970866141732586 Td
|
||||
(Simple PDF Document) Tj
|
||||
ET
|
||||
BT
|
||||
/F1 12 Tf
|
||||
13.7999999999999989 TL
|
||||
0 g
|
||||
56.6929133858267775 728.5041732283464171 Td
|
||||
(This is a test PDF with simple text content.) Tj
|
||||
ET
|
||||
BT
|
||||
/F1 12 Tf
|
||||
13.7999999999999989 TL
|
||||
0 g
|
||||
56.6929133858267775 700.15771653543311 Td
|
||||
(It contains multiple paragraphs to test text extraction.) Tj
|
||||
ET
|
||||
BT
|
||||
/F1 12 Tf
|
||||
13.7999999999999989 TL
|
||||
0 g
|
||||
56.6929133858267775 643.4648031496062686 Td
|
||||
(Second paragraph starts here.) Tj
|
||||
ET
|
||||
BT
|
||||
/F1 12 Tf
|
||||
13.7999999999999989 TL
|
||||
0 g
|
||||
56.6929133858267775 615.1183464566928478 Td
|
||||
(This paragraph has multiple sentences. Each sentence adds more text.) Tj
|
||||
ET
|
||||
BT
|
||||
/F1 12 Tf
|
||||
13.7999999999999989 TL
|
||||
0 g
|
||||
56.6929133858267775 558.42543307086612 Td
|
||||
(Third paragraph demonstrates extraction capabilities.) Tj
|
||||
ET
|
||||
endstream
|
||||
endobj
|
||||
1 0 obj
|
||||
<</Type /Pages
|
||||
/Kids [3 0 R ]
|
||||
/Count 1
|
||||
>>
|
||||
endobj
|
||||
5 0 obj
|
||||
<<
|
||||
/Type /Font
|
||||
/BaseFont /Helvetica
|
||||
/Subtype /Type1
|
||||
/Encoding /WinAnsiEncoding
|
||||
/FirstChar 32
|
||||
/LastChar 255
|
||||
>>
|
||||
endobj
|
||||
6 0 obj
|
||||
<<
|
||||
/Type /Font
|
||||
/BaseFont /Helvetica-Bold
|
||||
/Subtype /Type1
|
||||
/Encoding /WinAnsiEncoding
|
||||
/FirstChar 32
|
||||
/LastChar 255
|
||||
>>
|
||||
endobj
|
||||
7 0 obj
|
||||
<<
|
||||
/Type /Font
|
||||
/BaseFont /Helvetica-Oblique
|
||||
/Subtype /Type1
|
||||
/Encoding /WinAnsiEncoding
|
||||
/FirstChar 32
|
||||
/LastChar 255
|
||||
>>
|
||||
endobj
|
||||
8 0 obj
|
||||
<<
|
||||
/Type /Font
|
||||
/BaseFont /Helvetica-BoldOblique
|
||||
/Subtype /Type1
|
||||
/Encoding /WinAnsiEncoding
|
||||
/FirstChar 32
|
||||
/LastChar 255
|
||||
>>
|
||||
endobj
|
||||
9 0 obj
|
||||
<<
|
||||
/Type /Font
|
||||
/BaseFont /Courier
|
||||
/Subtype /Type1
|
||||
/Encoding /WinAnsiEncoding
|
||||
/FirstChar 32
|
||||
/LastChar 255
|
||||
>>
|
||||
endobj
|
||||
10 0 obj
|
||||
<<
|
||||
/Type /Font
|
||||
/BaseFont /Courier-Bold
|
||||
/Subtype /Type1
|
||||
/Encoding /WinAnsiEncoding
|
||||
/FirstChar 32
|
||||
/LastChar 255
|
||||
>>
|
||||
endobj
|
||||
11 0 obj
|
||||
<<
|
||||
/Type /Font
|
||||
/BaseFont /Courier-Oblique
|
||||
/Subtype /Type1
|
||||
/Encoding /WinAnsiEncoding
|
||||
/FirstChar 32
|
||||
/LastChar 255
|
||||
>>
|
||||
endobj
|
||||
12 0 obj
|
||||
<<
|
||||
/Type /Font
|
||||
/BaseFont /Courier-BoldOblique
|
||||
/Subtype /Type1
|
||||
/Encoding /WinAnsiEncoding
|
||||
/FirstChar 32
|
||||
/LastChar 255
|
||||
>>
|
||||
endobj
|
||||
13 0 obj
|
||||
<<
|
||||
/Type /Font
|
||||
/BaseFont /Times-Roman
|
||||
/Subtype /Type1
|
||||
/Encoding /WinAnsiEncoding
|
||||
/FirstChar 32
|
||||
/LastChar 255
|
||||
>>
|
||||
endobj
|
||||
14 0 obj
|
||||
<<
|
||||
/Type /Font
|
||||
/BaseFont /Times-Bold
|
||||
/Subtype /Type1
|
||||
/Encoding /WinAnsiEncoding
|
||||
/FirstChar 32
|
||||
/LastChar 255
|
||||
>>
|
||||
endobj
|
||||
15 0 obj
|
||||
<<
|
||||
/Type /Font
|
||||
/BaseFont /Times-Italic
|
||||
/Subtype /Type1
|
||||
/Encoding /WinAnsiEncoding
|
||||
/FirstChar 32
|
||||
/LastChar 255
|
||||
>>
|
||||
endobj
|
||||
16 0 obj
|
||||
<<
|
||||
/Type /Font
|
||||
/BaseFont /Times-BoldItalic
|
||||
/Subtype /Type1
|
||||
/Encoding /WinAnsiEncoding
|
||||
/FirstChar 32
|
||||
/LastChar 255
|
||||
>>
|
||||
endobj
|
||||
17 0 obj
|
||||
<<
|
||||
/Type /Font
|
||||
/BaseFont /ZapfDingbats
|
||||
/Subtype /Type1
|
||||
/FirstChar 32
|
||||
/LastChar 255
|
||||
>>
|
||||
endobj
|
||||
18 0 obj
|
||||
<<
|
||||
/Type /Font
|
||||
/BaseFont /Symbol
|
||||
/Subtype /Type1
|
||||
/FirstChar 32
|
||||
/LastChar 255
|
||||
>>
|
||||
endobj
|
||||
2 0 obj
|
||||
<<
|
||||
/ProcSet [/PDF /Text /ImageB /ImageC /ImageI]
|
||||
/Font <<
|
||||
/F1 5 0 R
|
||||
/F2 6 0 R
|
||||
/F3 7 0 R
|
||||
/F4 8 0 R
|
||||
/F5 9 0 R
|
||||
/F6 10 0 R
|
||||
/F7 11 0 R
|
||||
/F8 12 0 R
|
||||
/F9 13 0 R
|
||||
/F10 14 0 R
|
||||
/F11 15 0 R
|
||||
/F12 16 0 R
|
||||
/F13 17 0 R
|
||||
/F14 18 0 R
|
||||
>>
|
||||
/XObject <<
|
||||
>>
|
||||
>>
|
||||
endobj
|
||||
19 0 obj
|
||||
<<
|
||||
/Producer (jsPDF 3.0.3)
|
||||
/CreationDate (D:20251001162659-07'00')
|
||||
>>
|
||||
endobj
|
||||
20 0 obj
|
||||
<<
|
||||
/Type /Catalog
|
||||
/Pages 1 0 R
|
||||
/OpenAction [3 0 R /FitH null]
|
||||
/PageLayout /OneColumn
|
||||
>>
|
||||
endobj
|
||||
xref
|
||||
0 21
|
||||
0000000000 65535 f
|
||||
0000001050 00000 n
|
||||
0000002867 00000 n
|
||||
0000000015 00000 n
|
||||
0000000152 00000 n
|
||||
0000001107 00000 n
|
||||
0000001232 00000 n
|
||||
0000001362 00000 n
|
||||
0000001495 00000 n
|
||||
0000001632 00000 n
|
||||
0000001755 00000 n
|
||||
0000001884 00000 n
|
||||
0000002016 00000 n
|
||||
0000002152 00000 n
|
||||
0000002280 00000 n
|
||||
0000002407 00000 n
|
||||
0000002536 00000 n
|
||||
0000002669 00000 n
|
||||
0000002771 00000 n
|
||||
0000003115 00000 n
|
||||
0000003201 00000 n
|
||||
trailer
|
||||
<<
|
||||
/Size 21
|
||||
/Root 20 0 R
|
||||
/Info 19 0 R
|
||||
/ID [ <0F5A6C1D434F4F388C03021EE84E0E57> <0F5A6C1D434F4F388C03021EE84E0E57> ]
|
||||
>>
|
||||
startxref
|
||||
3305
|
||||
%%EOF
|
||||
Loading…
Add table
Add a link
Reference in a new issue