feat: add intelligent import for CSV, Excel, and PDF files
Add IntelligentImportAugmentation with support for: - CSV: auto-detection of encoding, delimiters, and field types - Excel: multi-sheet extraction with metadata preservation - PDF: text extraction, table detection, and metadata extraction Features: - Automatic format detection from file extension or content - Intelligent type inference (string, number, boolean, date) - Seamless integration with neural entity extraction - Production-ready with 69 comprehensive tests Dependencies added: - xlsx@^0.18.5 for Excel parsing - pdfjs-dist@^4.0.379 for PDF parsing - csv-parse@^6.1.0 for CSV parsing - chardet@^2.0.0 for encoding detection Documentation: - Updated README with import examples - Updated API_REFERENCE with comprehensive import() docs - Updated import-anything.md guide - Added working example: import-excel-pdf-csv.ts - Updated augmentations README
This commit is contained in:
parent
aaf8e0f411
commit
814cbb48ee
33 changed files with 4664 additions and 28 deletions
324
tests/fixtures/import/table.pdf
vendored
Normal file
324
tests/fixtures/import/table.pdf
vendored
Normal file
|
|
@ -0,0 +1,324 @@
|
|||
%PDF-1.3
|
||||
%ºß¬à
|
||||
3 0 obj
|
||||
<</Type /Page
|
||||
/Parent 1 0 R
|
||||
/Resources 2 0 R
|
||||
/MediaBox [0 0 595.2799999999999727 841.8899999999999864]
|
||||
/Contents 4 0 R
|
||||
>>
|
||||
endobj
|
||||
4 0 obj
|
||||
<<
|
||||
/Length 1152
|
||||
>>
|
||||
stream
|
||||
0.5670000000000001 w
|
||||
0 G
|
||||
BT
|
||||
/F1 14 Tf
|
||||
16.0999999999999979 TL
|
||||
0 g
|
||||
56.6929133858267775 785.1970866141732586 Td
|
||||
(Employee Records) Tj
|
||||
ET
|
||||
BT
|
||||
/F2 10 Tf
|
||||
11.5 TL
|
||||
0 g
|
||||
56.6929133858267775 728.5041732283464171 Td
|
||||
(Name) Tj
|
||||
ET
|
||||
BT
|
||||
/F2 10 Tf
|
||||
11.5 TL
|
||||
0 g
|
||||
226.7716535433071101 728.5041732283464171 Td
|
||||
(Age) Tj
|
||||
ET
|
||||
BT
|
||||
/F2 10 Tf
|
||||
11.5 TL
|
||||
0 g
|
||||
340.157480314960651 728.5041732283464171 Td
|
||||
(Department) Tj
|
||||
ET
|
||||
BT
|
||||
/F1 10 Tf
|
||||
11.5 TL
|
||||
0 g
|
||||
56.6929133858267775 700.15771653543311 Td
|
||||
(Alice Johnson) Tj
|
||||
ET
|
||||
BT
|
||||
/F1 10 Tf
|
||||
11.5 TL
|
||||
0 g
|
||||
226.7716535433071101 700.15771653543311 Td
|
||||
(28) Tj
|
||||
ET
|
||||
BT
|
||||
/F1 10 Tf
|
||||
11.5 TL
|
||||
0 g
|
||||
340.157480314960651 700.15771653543311 Td
|
||||
(Engineering) Tj
|
||||
ET
|
||||
BT
|
||||
/F1 10 Tf
|
||||
11.5 TL
|
||||
0 g
|
||||
56.6929133858267775 671.8112598425196893 Td
|
||||
(Bob Smith) Tj
|
||||
ET
|
||||
BT
|
||||
/F1 10 Tf
|
||||
11.5 TL
|
||||
0 g
|
||||
226.7716535433071101 671.8112598425196893 Td
|
||||
(35) Tj
|
||||
ET
|
||||
BT
|
||||
/F1 10 Tf
|
||||
11.5 TL
|
||||
0 g
|
||||
340.157480314960651 671.8112598425196893 Td
|
||||
(Sales) Tj
|
||||
ET
|
||||
BT
|
||||
/F1 10 Tf
|
||||
11.5 TL
|
||||
0 g
|
||||
56.6929133858267775 643.4648031496062686 Td
|
||||
(Charlie Brown) Tj
|
||||
ET
|
||||
BT
|
||||
/F1 10 Tf
|
||||
11.5 TL
|
||||
0 g
|
||||
226.7716535433071101 643.4648031496062686 Td
|
||||
(42) Tj
|
||||
ET
|
||||
BT
|
||||
/F1 10 Tf
|
||||
11.5 TL
|
||||
0 g
|
||||
340.157480314960651 643.4648031496062686 Td
|
||||
(Engineering) Tj
|
||||
ET
|
||||
endstream
|
||||
endobj
|
||||
1 0 obj
|
||||
<</Type /Pages
|
||||
/Kids [3 0 R ]
|
||||
/Count 1
|
||||
>>
|
||||
endobj
|
||||
5 0 obj
|
||||
<<
|
||||
/Type /Font
|
||||
/BaseFont /Helvetica
|
||||
/Subtype /Type1
|
||||
/Encoding /WinAnsiEncoding
|
||||
/FirstChar 32
|
||||
/LastChar 255
|
||||
>>
|
||||
endobj
|
||||
6 0 obj
|
||||
<<
|
||||
/Type /Font
|
||||
/BaseFont /Helvetica-Bold
|
||||
/Subtype /Type1
|
||||
/Encoding /WinAnsiEncoding
|
||||
/FirstChar 32
|
||||
/LastChar 255
|
||||
>>
|
||||
endobj
|
||||
7 0 obj
|
||||
<<
|
||||
/Type /Font
|
||||
/BaseFont /Helvetica-Oblique
|
||||
/Subtype /Type1
|
||||
/Encoding /WinAnsiEncoding
|
||||
/FirstChar 32
|
||||
/LastChar 255
|
||||
>>
|
||||
endobj
|
||||
8 0 obj
|
||||
<<
|
||||
/Type /Font
|
||||
/BaseFont /Helvetica-BoldOblique
|
||||
/Subtype /Type1
|
||||
/Encoding /WinAnsiEncoding
|
||||
/FirstChar 32
|
||||
/LastChar 255
|
||||
>>
|
||||
endobj
|
||||
9 0 obj
|
||||
<<
|
||||
/Type /Font
|
||||
/BaseFont /Courier
|
||||
/Subtype /Type1
|
||||
/Encoding /WinAnsiEncoding
|
||||
/FirstChar 32
|
||||
/LastChar 255
|
||||
>>
|
||||
endobj
|
||||
10 0 obj
|
||||
<<
|
||||
/Type /Font
|
||||
/BaseFont /Courier-Bold
|
||||
/Subtype /Type1
|
||||
/Encoding /WinAnsiEncoding
|
||||
/FirstChar 32
|
||||
/LastChar 255
|
||||
>>
|
||||
endobj
|
||||
11 0 obj
|
||||
<<
|
||||
/Type /Font
|
||||
/BaseFont /Courier-Oblique
|
||||
/Subtype /Type1
|
||||
/Encoding /WinAnsiEncoding
|
||||
/FirstChar 32
|
||||
/LastChar 255
|
||||
>>
|
||||
endobj
|
||||
12 0 obj
|
||||
<<
|
||||
/Type /Font
|
||||
/BaseFont /Courier-BoldOblique
|
||||
/Subtype /Type1
|
||||
/Encoding /WinAnsiEncoding
|
||||
/FirstChar 32
|
||||
/LastChar 255
|
||||
>>
|
||||
endobj
|
||||
13 0 obj
|
||||
<<
|
||||
/Type /Font
|
||||
/BaseFont /Times-Roman
|
||||
/Subtype /Type1
|
||||
/Encoding /WinAnsiEncoding
|
||||
/FirstChar 32
|
||||
/LastChar 255
|
||||
>>
|
||||
endobj
|
||||
14 0 obj
|
||||
<<
|
||||
/Type /Font
|
||||
/BaseFont /Times-Bold
|
||||
/Subtype /Type1
|
||||
/Encoding /WinAnsiEncoding
|
||||
/FirstChar 32
|
||||
/LastChar 255
|
||||
>>
|
||||
endobj
|
||||
15 0 obj
|
||||
<<
|
||||
/Type /Font
|
||||
/BaseFont /Times-Italic
|
||||
/Subtype /Type1
|
||||
/Encoding /WinAnsiEncoding
|
||||
/FirstChar 32
|
||||
/LastChar 255
|
||||
>>
|
||||
endobj
|
||||
16 0 obj
|
||||
<<
|
||||
/Type /Font
|
||||
/BaseFont /Times-BoldItalic
|
||||
/Subtype /Type1
|
||||
/Encoding /WinAnsiEncoding
|
||||
/FirstChar 32
|
||||
/LastChar 255
|
||||
>>
|
||||
endobj
|
||||
17 0 obj
|
||||
<<
|
||||
/Type /Font
|
||||
/BaseFont /ZapfDingbats
|
||||
/Subtype /Type1
|
||||
/FirstChar 32
|
||||
/LastChar 255
|
||||
>>
|
||||
endobj
|
||||
18 0 obj
|
||||
<<
|
||||
/Type /Font
|
||||
/BaseFont /Symbol
|
||||
/Subtype /Type1
|
||||
/FirstChar 32
|
||||
/LastChar 255
|
||||
>>
|
||||
endobj
|
||||
2 0 obj
|
||||
<<
|
||||
/ProcSet [/PDF /Text /ImageB /ImageC /ImageI]
|
||||
/Font <<
|
||||
/F1 5 0 R
|
||||
/F2 6 0 R
|
||||
/F3 7 0 R
|
||||
/F4 8 0 R
|
||||
/F5 9 0 R
|
||||
/F6 10 0 R
|
||||
/F7 11 0 R
|
||||
/F8 12 0 R
|
||||
/F9 13 0 R
|
||||
/F10 14 0 R
|
||||
/F11 15 0 R
|
||||
/F12 16 0 R
|
||||
/F13 17 0 R
|
||||
/F14 18 0 R
|
||||
>>
|
||||
/XObject <<
|
||||
>>
|
||||
>>
|
||||
endobj
|
||||
19 0 obj
|
||||
<<
|
||||
/Producer (jsPDF 3.0.3)
|
||||
/CreationDate (D:20251001162659-07'00')
|
||||
>>
|
||||
endobj
|
||||
20 0 obj
|
||||
<<
|
||||
/Type /Catalog
|
||||
/Pages 1 0 R
|
||||
/OpenAction [3 0 R /FitH null]
|
||||
/PageLayout /OneColumn
|
||||
>>
|
||||
endobj
|
||||
xref
|
||||
0 21
|
||||
0000000000 65535 f
|
||||
0000001356 00000 n
|
||||
0000003173 00000 n
|
||||
0000000015 00000 n
|
||||
0000000152 00000 n
|
||||
0000001413 00000 n
|
||||
0000001538 00000 n
|
||||
0000001668 00000 n
|
||||
0000001801 00000 n
|
||||
0000001938 00000 n
|
||||
0000002061 00000 n
|
||||
0000002190 00000 n
|
||||
0000002322 00000 n
|
||||
0000002458 00000 n
|
||||
0000002586 00000 n
|
||||
0000002713 00000 n
|
||||
0000002842 00000 n
|
||||
0000002975 00000 n
|
||||
0000003077 00000 n
|
||||
0000003421 00000 n
|
||||
0000003507 00000 n
|
||||
trailer
|
||||
<<
|
||||
/Size 21
|
||||
/Root 20 0 R
|
||||
/Info 19 0 R
|
||||
/ID [ <A34A2614AA2553099D2A387E65B754AA> <A34A2614AA2553099D2A387E65B754AA> ]
|
||||
>>
|
||||
startxref
|
||||
3611
|
||||
%%EOF
|
||||
Loading…
Add table
Add a link
Reference in a new issue