feat: add intelligent import for CSV, Excel, and PDF files

Add IntelligentImportAugmentation with support for:
- CSV: auto-detection of encoding, delimiters, and field types
- Excel: multi-sheet extraction with metadata preservation
- PDF: text extraction, table detection, and metadata extraction

Features:
- Automatic format detection from file extension or content
- Intelligent type inference (string, number, boolean, date)
- Seamless integration with neural entity extraction
- Production-ready with 69 comprehensive tests

Dependencies added:
- xlsx@^0.18.5 for Excel parsing
- pdfjs-dist@^4.0.379 for PDF parsing
- csv-parse@^6.1.0 for CSV parsing
- chardet@^2.0.0 for encoding detection

Documentation:
- Updated README with import examples
- Updated API_REFERENCE with comprehensive import() docs
- Updated import-anything.md guide
- Added working example: import-excel-pdf-csv.ts
- Updated augmentations README
This commit is contained in:
David Snelling 2025-10-01 16:51:03 -07:00
parent aaf8e0f411
commit 814cbb48ee
33 changed files with 4664 additions and 28 deletions

View file

@ -0,0 +1,281 @@
/**
* CSV Handler Tests
* Comprehensive tests for CSV parsing, encoding detection, and type inference
*/
import { describe, it, expect, beforeAll } from 'vitest'
import { CSVHandler } from '../../../src/augmentations/intelligentImport/handlers/csvHandler.js'
import { promises as fs } from 'fs'
import * as path from 'path'
describe('CSVHandler', () => {
let handler: CSVHandler
const fixturesPath = path.join(process.cwd(), 'tests/fixtures/import')
beforeAll(() => {
handler = new CSVHandler()
})
describe('canHandle', () => {
it('should handle .csv extension', () => {
expect(handler.canHandle({ filename: 'data.csv' })).toBe(true)
})
it('should handle .tsv extension', () => {
expect(handler.canHandle({ filename: 'data.tsv' })).toBe(true)
})
it('should handle .txt extension', () => {
expect(handler.canHandle({ filename: 'data.txt' })).toBe(true)
})
it('should handle CSV content (comma)', () => {
const content = 'name,age,email\nJohn,30,john@example.com'
expect(handler.canHandle(content)).toBe(true)
})
it('should handle CSV content (semicolon)', () => {
const content = 'name;age;email\nJohn;30;john@example.com'
expect(handler.canHandle(content)).toBe(true)
})
it('should not handle non-CSV content', () => {
const content = 'This is just plain text without any structure'
expect(handler.canHandle(content)).toBe(false)
})
it('should not handle .xlsx extension', () => {
expect(handler.canHandle({ filename: 'data.xlsx' })).toBe(false)
})
})
describe('process - simple CSV', () => {
it('should parse simple comma-delimited CSV', async () => {
const filePath = path.join(fixturesPath, 'simple.csv')
const buffer = await fs.readFile(filePath)
const result = await handler.process(buffer, { filename: 'simple.csv' })
expect(result.format).toBe('csv')
expect(result.data).toHaveLength(4)
expect(result.data[0]).toEqual({
name: 'John Doe',
age: 30,
email: 'john@example.com',
active: true
})
expect(result.metadata.rowCount).toBe(4)
expect(result.metadata.fields).toEqual(['name', 'age', 'email', 'active'])
expect(result.metadata.delimiter).toBe(',')
expect(result.metadata.hasHeaders).toBe(true)
})
it('should infer correct types', async () => {
const filePath = path.join(fixturesPath, 'simple.csv')
const buffer = await fs.readFile(filePath)
const result = await handler.process(buffer, { filename: 'simple.csv' })
expect(typeof result.data[0].name).toBe('string')
expect(typeof result.data[0].age).toBe('number')
expect(typeof result.data[0].active).toBe('boolean')
})
})
describe('process - delimiter detection', () => {
it('should detect semicolon delimiter', async () => {
const filePath = path.join(fixturesPath, 'semicolon.csv')
const buffer = await fs.readFile(filePath)
const result = await handler.process(buffer, { filename: 'semicolon.csv' })
expect(result.metadata.delimiter).toBe(';')
expect(result.data).toHaveLength(4)
expect(result.data[0]).toEqual({
product: 'Laptop',
price: 999.99,
quantity: 10,
inStock: true
})
})
it('should detect tab delimiter', async () => {
const filePath = path.join(fixturesPath, 'tab-delimited.csv')
const buffer = await fs.readFile(filePath)
const result = await handler.process(buffer, { filename: 'tab-delimited.csv' })
expect(result.metadata.delimiter).toBe('\t')
expect(result.data).toHaveLength(4)
expect(result.data[0].city).toBe('Tokyo')
expect(result.data[0].population).toBe(13960000)
})
it('should use specified delimiter when provided', async () => {
const content = 'name|age|email\nJohn|30|john@example.com'
const result = await handler.process(content, {
csvDelimiter: '|',
filename: 'pipe.csv'
})
expect(result.data).toHaveLength(1)
expect(result.data[0].name).toBe('John')
expect(result.data[0].age).toBe(30)
})
})
describe('process - type inference', () => {
it('should infer integer, float, date, and boolean types', async () => {
const filePath = path.join(fixturesPath, 'types.csv')
const buffer = await fs.readFile(filePath)
const result = await handler.process(buffer, { filename: 'types.csv' })
expect(result.metadata.types).toBeDefined()
expect(result.metadata.types.id).toBe('integer')
expect(result.metadata.types.name).toBe('string')
// Score is 95 (integer), not 95.0, so it's detected as integer
expect(['integer', 'float']).toContain(result.metadata.types.score)
expect(result.metadata.types.active).toBe('boolean')
// Check actual values
expect(result.data[0].id).toBe(1)
expect([95, 95.0]).toContain(result.data[0].score) // Can be integer or float
expect(result.data[0].active).toBe(true)
expect(result.data[1].score).toBe(87.5) // This one is definitely a float
})
it('should handle null values', async () => {
const content = 'name,age,email\nJohn,30,john@example.com\nJane,,jane@example.com\nBob,45,'
const result = await handler.process(content, { filename: 'nulls.csv' })
expect(result.data[1].age).toBeNull()
expect(result.data[2].email).toBeNull()
})
})
describe('process - encoding detection', () => {
it('should detect UTF-8 encoding', async () => {
const content = 'name,description\nProduct 1,Description with émojis 🎉'
const result = await handler.process(content, { filename: 'utf8.csv' })
// Encoding can be 'UTF-8', 'utf-8', or 'utf8' depending on detection
expect(result.metadata.encoding.toLowerCase()).toContain('utf')
expect(result.data[0].description).toContain('émojis')
expect(result.data[0].description).toContain('🎉')
})
it('should use specified encoding when provided', async () => {
const content = 'name,age\nJohn,30'
const result = await handler.process(content, {
encoding: 'ascii',
filename: 'ascii.csv'
})
expect(result.metadata.encoding).toBe('ascii')
})
})
describe('process - options', () => {
it('should respect csvHeaders=false option', async () => {
const content = 'John,30,john@example.com\nJane,25,jane@example.com'
const result = await handler.process(content, {
csvHeaders: false,
filename: 'no-headers.csv'
})
expect(result.metadata.hasHeaders).toBe(false)
// csv-parse will generate column names like '0', '1', '2'
expect(Object.keys(result.data[0])).toHaveLength(3)
})
it('should respect maxRows option', async () => {
const filePath = path.join(fixturesPath, 'simple.csv')
const buffer = await fs.readFile(filePath)
const result = await handler.process(buffer, {
maxRows: 2,
filename: 'simple.csv'
})
expect(result.data).toHaveLength(2)
expect(result.metadata.rowCount).toBe(2)
})
})
describe('process - edge cases', () => {
it('should handle empty CSV', async () => {
const content = ''
const result = await handler.process(content, { filename: 'empty.csv' })
expect(result.data).toHaveLength(0)
expect(result.metadata.rowCount).toBe(0)
expect(result.metadata.fields).toEqual([])
})
it('should handle CSV with only headers', async () => {
const content = 'name,age,email'
const result = await handler.process(content, { filename: 'headers-only.csv' })
expect(result.data).toHaveLength(0)
expect(result.metadata.rowCount).toBe(0)
})
it('should handle CSV with single row', async () => {
const content = 'name,age,email\nJohn,30,john@example.com'
const result = await handler.process(content, { filename: 'single-row.csv' })
expect(result.data).toHaveLength(1)
expect(result.data[0].name).toBe('John')
})
it('should handle CSV with varying column counts', async () => {
const content = 'name,age,email\nJohn,30,john@example.com\nJane,25\nBob,45,bob@example.com,extra'
const result = await handler.process(content, { filename: 'varying.csv' })
// csv-parse with relax_column_count should handle this
expect(result.data).toHaveLength(3)
})
it('should trim whitespace', async () => {
const content = 'name, age , email \n John , 30 , john@example.com '
const result = await handler.process(content, { filename: 'whitespace.csv' })
expect(result.data[0].name).toBe('John')
expect(result.data[0].age).toBe(30)
expect(result.data[0].email).toBe('john@example.com')
})
})
describe('process - performance', () => {
it('should measure processing time', async () => {
const filePath = path.join(fixturesPath, 'simple.csv')
const buffer = await fs.readFile(filePath)
const result = await handler.process(buffer, { filename: 'simple.csv' })
expect(result.metadata.processingTime).toBeGreaterThanOrEqual(0)
expect(typeof result.metadata.processingTime).toBe('number')
})
})
describe('process - error handling', () => {
it('should throw error for malformed CSV', async () => {
const content = 'name,age,email\nJohn,30,"unclosed quote'
await expect(
handler.process(content, { filename: 'malformed.csv' })
).rejects.toThrow('CSV parsing failed')
})
})
})

View file

@ -0,0 +1,269 @@
/**
* Excel Handler Tests
* Comprehensive tests for Excel parsing, multi-sheet extraction, and type inference
*/
import { describe, it, expect, beforeAll } from 'vitest'
import { ExcelHandler } from '../../../src/augmentations/intelligentImport/handlers/excelHandler.js'
import { promises as fs } from 'fs'
import * as path from 'path'
describe('ExcelHandler', () => {
let handler: ExcelHandler
const fixturesPath = path.join(process.cwd(), 'tests/fixtures/import')
beforeAll(() => {
handler = new ExcelHandler()
})
describe('canHandle', () => {
it('should handle .xlsx extension', () => {
expect(handler.canHandle({ filename: 'data.xlsx' })).toBe(true)
})
it('should handle .xls extension', () => {
expect(handler.canHandle({ filename: 'data.xls' })).toBe(true)
})
it('should handle .xlsb extension', () => {
expect(handler.canHandle({ filename: 'data.xlsb' })).toBe(true)
})
it('should handle .xlsm extension', () => {
expect(handler.canHandle({ filename: 'data.xlsm' })).toBe(true)
})
it('should not handle .csv extension', () => {
expect(handler.canHandle({ filename: 'data.csv' })).toBe(false)
})
it('should not handle .pdf extension', () => {
expect(handler.canHandle({ filename: 'data.pdf' })).toBe(false)
})
})
describe('process - simple Excel', () => {
it('should parse simple single-sheet Excel file', async () => {
const filePath = path.join(fixturesPath, 'simple.xlsx')
const buffer = await fs.readFile(filePath)
const result = await handler.process(buffer, { filename: 'simple.xlsx' })
expect(result.format).toBe('excel')
expect(result.data).toHaveLength(4)
expect(result.data[0]).toMatchObject({
Name: 'Alice Johnson',
Age: 28,
Department: 'Engineering',
Salary: 95000,
Active: true
})
expect(result.metadata.rowCount).toBe(4)
expect(result.metadata.sheetCount).toBe(1)
expect(result.metadata.sheets).toEqual(['Employees'])
})
it('should infer correct types from Excel data', async () => {
const filePath = path.join(fixturesPath, 'simple.xlsx')
const buffer = await fs.readFile(filePath)
const result = await handler.process(buffer, { filename: 'simple.xlsx' })
expect(result.metadata.types).toBeDefined()
expect(result.metadata.types.Name).toBe('string')
expect(result.metadata.types.Age).toBe('integer')
expect(['integer', 'float']).toContain(result.metadata.types.Salary)
expect(result.metadata.types.Active).toBe('boolean')
})
})
describe('process - multi-sheet Excel', () => {
it('should extract data from all sheets by default', async () => {
const filePath = path.join(fixturesPath, 'multi-sheet.xlsx')
const buffer = await fs.readFile(filePath)
const result = await handler.process(buffer, { filename: 'multi-sheet.xlsx' })
expect(result.metadata.sheetCount).toBe(3)
expect(result.metadata.sheets).toEqual(['Products', 'Orders', 'Customers'])
// Total rows from all sheets
const totalRows = 4 + 3 + 3 // Products + Orders + Customers
expect(result.data).toHaveLength(totalRows)
// Check _sheet field is added
expect(result.data[0]._sheet).toBe('Products')
})
it('should filter data by sheet name', async () => {
const filePath = path.join(fixturesPath, 'multi-sheet.xlsx')
const buffer = await fs.readFile(filePath)
const result = await handler.process(buffer, {
filename: 'multi-sheet.xlsx',
excelSheets: ['Products']
})
expect(result.metadata.sheetCount).toBe(1)
expect(result.metadata.sheets).toEqual(['Products'])
expect(result.data).toHaveLength(4)
expect(result.data.every(row => row._sheet === 'Products')).toBe(true)
})
it('should extract data from multiple specified sheets', async () => {
const filePath = path.join(fixturesPath, 'multi-sheet.xlsx')
const buffer = await fs.readFile(filePath)
const result = await handler.process(buffer, {
filename: 'multi-sheet.xlsx',
excelSheets: ['Products', 'Customers']
})
expect(result.metadata.sheetCount).toBe(2)
expect(result.metadata.sheets).toEqual(['Products', 'Customers'])
expect(result.data).toHaveLength(7) // 4 + 3
const sheets = new Set(result.data.map(row => row._sheet))
expect(sheets.has('Products')).toBe(true)
expect(sheets.has('Customers')).toBe(true)
expect(sheets.has('Orders')).toBe(false)
})
it('should include sheet metadata for each sheet', async () => {
const filePath = path.join(fixturesPath, 'multi-sheet.xlsx')
const buffer = await fs.readFile(filePath)
const result = await handler.process(buffer, { filename: 'multi-sheet.xlsx' })
expect(result.metadata.sheetMetadata).toBeDefined()
expect(result.metadata.sheetMetadata.Products).toMatchObject({
rowCount: 4,
columnCount: 5
})
expect(result.metadata.sheetMetadata.Products.headers).toEqual([
'ID', 'Product', 'Price', 'Stock', 'Category'
])
})
})
describe('process - type inference', () => {
it('should infer integer, float, date, and boolean types', async () => {
const filePath = path.join(fixturesPath, 'types.xlsx')
const buffer = await fs.readFile(filePath)
const result = await handler.process(buffer, { filename: 'types.xlsx' })
expect(result.metadata.types).toBeDefined()
expect(result.metadata.types.ID).toBe('integer')
expect(result.metadata.types.Name).toBe('string')
expect(['integer', 'float']).toContain(result.metadata.types.Score)
expect(['integer', 'float']).toContain(result.metadata.types.Percentage)
expect(result.metadata.types.Active).toBe('boolean')
// Check actual values
expect(result.data[0].ID).toBe(1)
expect(result.data[0].Active).toBe(true)
expect(typeof result.data[0].Score).toBe('number')
})
it('should handle null/empty values', async () => {
const filePath = path.join(fixturesPath, 'empty.xlsx')
const buffer = await fs.readFile(filePath)
const result = await handler.process(buffer, { filename: 'empty.xlsx' })
expect(result.data).toHaveLength(0)
expect(result.metadata.rowCount).toBe(0)
})
})
describe('process - workbook metadata', () => {
it('should extract workbook information', async () => {
const filePath = path.join(fixturesPath, 'multi-sheet.xlsx')
const buffer = await fs.readFile(filePath)
const result = await handler.process(buffer, { filename: 'multi-sheet.xlsx' })
expect(result.metadata.workbookInfo).toBeDefined()
expect(result.metadata.workbookInfo.sheetNames).toEqual([
'Products', 'Orders', 'Customers'
])
})
})
describe('process - edge cases', () => {
it('should handle Excel file with only headers', async () => {
const filePath = path.join(fixturesPath, 'empty.xlsx')
const buffer = await fs.readFile(filePath)
const result = await handler.process(buffer, { filename: 'empty.xlsx' })
expect(result.data).toHaveLength(0)
expect(result.metadata.sheetCount).toBe(1)
})
it('should skip non-existent sheets when filtering', async () => {
const filePath = path.join(fixturesPath, 'simple.xlsx')
const buffer = await fs.readFile(filePath)
const result = await handler.process(buffer, {
filename: 'simple.xlsx',
excelSheets: ['Employees', 'NonExistent']
})
expect(result.metadata.sheets).toEqual(['Employees'])
expect(result.data).toHaveLength(4)
})
})
describe('process - performance', () => {
it('should measure processing time', async () => {
const filePath = path.join(fixturesPath, 'simple.xlsx')
const buffer = await fs.readFile(filePath)
const result = await handler.process(buffer, { filename: 'simple.xlsx' })
expect(result.metadata.processingTime).toBeGreaterThanOrEqual(0)
expect(typeof result.metadata.processingTime).toBe('number')
})
})
describe('process - field sanitization', () => {
it('should sanitize column names', async () => {
const filePath = path.join(fixturesPath, 'simple.xlsx')
const buffer = await fs.readFile(filePath)
const result = await handler.process(buffer, { filename: 'simple.xlsx' })
// All field names should be valid identifiers
const fields = Object.keys(result.data[0]).filter(k => k !== '_sheet')
for (const field of fields) {
expect(field).toMatch(/^[a-zA-Z_][a-zA-Z0-9_]*$/)
}
})
})
describe('process - error handling', () => {
it('should handle corrupted Excel file gracefully', async () => {
// XLSX library is very forgiving, so we test that it returns empty data
// rather than crashing
const invalidData = Buffer.from('This is not an Excel file')
const result = await handler.process(invalidData, { filename: 'invalid.xlsx' })
// Should return empty data instead of crashing
expect(result.data).toHaveLength(0)
expect(result.metadata.rowCount).toBe(0)
})
it('should handle binary garbage', async () => {
const garbage = Buffer.from(new Uint8Array(256).fill(255))
const result = await handler.process(garbage, { filename: 'garbage.xlsx' })
// Should not crash
expect(result).toBeDefined()
expect(result.format).toBe('excel')
})
})
})

View file

@ -0,0 +1,292 @@
/**
* PDF Handler Tests
* Comprehensive tests for PDF text extraction, table detection, and metadata
*/
import { describe, it, expect, beforeAll } from 'vitest'
import { PDFHandler } from '../../../src/augmentations/intelligentImport/handlers/pdfHandler.js'
import { promises as fs } from 'fs'
import * as path from 'path'
describe('PDFHandler', () => {
let handler: PDFHandler
const fixturesPath = path.join(process.cwd(), 'tests/fixtures/import')
beforeAll(() => {
handler = new PDFHandler()
})
describe('canHandle', () => {
it('should handle .pdf extension', () => {
expect(handler.canHandle({ filename: 'document.pdf' })).toBe(true)
})
it('should handle PDF magic bytes', () => {
const pdfBuffer = Buffer.from('%PDF-1.4\n')
expect(handler.canHandle(pdfBuffer)).toBe(true)
})
it('should not handle .xlsx extension', () => {
expect(handler.canHandle({ filename: 'data.xlsx' })).toBe(false)
})
it('should not handle .csv extension', () => {
expect(handler.canHandle({ filename: 'data.csv' })).toBe(false)
})
it('should not handle non-PDF buffer', () => {
const buffer = Buffer.from('This is not a PDF')
expect(handler.canHandle(buffer)).toBe(false)
})
})
describe('process - simple text PDF', () => {
it('should extract text from simple PDF', async () => {
const filePath = path.join(fixturesPath, 'simple.pdf')
const buffer = await fs.readFile(filePath)
const result = await handler.process(buffer, { filename: 'simple.pdf' })
expect(result.format).toBe('pdf')
expect(result.data.length).toBeGreaterThan(0)
// Should have extracted paragraphs
const paragraphs = result.data.filter(item => item._type === 'paragraph')
expect(paragraphs.length).toBeGreaterThan(0)
// Check that text was extracted
const hasText = paragraphs.some(p => p.text && p.text.length > 0)
expect(hasText).toBe(true)
})
it('should include page numbers', async () => {
const filePath = path.join(fixturesPath, 'simple.pdf')
const buffer = await fs.readFile(filePath)
const result = await handler.process(buffer, { filename: 'simple.pdf' })
// All items should have _page field
expect(result.data.every(item => typeof item._page === 'number')).toBe(true)
expect(result.data.every(item => item._page >= 1)).toBe(true)
})
it('should count pages correctly', async () => {
const filePath = path.join(fixturesPath, 'simple.pdf')
const buffer = await fs.readFile(filePath)
const result = await handler.process(buffer, { filename: 'simple.pdf' })
expect(result.metadata.pageCount).toBe(1)
})
})
describe('process - multi-page PDF', () => {
it('should extract text from all pages', async () => {
const filePath = path.join(fixturesPath, 'multi-page.pdf')
const buffer = await fs.readFile(filePath)
const result = await handler.process(buffer, { filename: 'multi-page.pdf' })
expect(result.metadata.pageCount).toBe(3)
// Should have content from multiple pages
const pages = new Set(result.data.map(item => item._page))
expect(pages.size).toBeGreaterThan(1)
expect(pages.has(1)).toBe(true)
expect(pages.has(2)).toBe(true)
expect(pages.has(3)).toBe(true)
})
it('should preserve page order', async () => {
const filePath = path.join(fixturesPath, 'multi-page.pdf')
const buffer = await fs.readFile(filePath)
const result = await handler.process(buffer, { filename: 'multi-page.pdf' })
// Page numbers should be in order
let lastPage = 0
let pageChanged = false
for (const item of result.data) {
if (item._page !== lastPage && lastPage > 0) {
pageChanged = true
}
expect(item._page).toBeGreaterThanOrEqual(lastPage)
lastPage = item._page
}
expect(pageChanged).toBe(true) // Should have moved through pages
})
})
describe('process - table detection', () => {
it('should detect tables in PDF', async () => {
const filePath = path.join(fixturesPath, 'table.pdf')
const buffer = await fs.readFile(filePath)
const result = await handler.process(buffer, { filename: 'table.pdf' })
// Should have detected at least one table
expect(result.metadata.tableCount).toBeGreaterThan(0)
})
it('should extract table rows when tables detected', async () => {
const filePath = path.join(fixturesPath, 'table.pdf')
const buffer = await fs.readFile(filePath)
const result = await handler.process(buffer, {
filename: 'table.pdf',
pdfExtractTables: true
})
// Should have table_row items
const tableRows = result.data.filter(item => item._type === 'table_row')
expect(tableRows.length).toBeGreaterThan(0)
// Table rows should have structured data
if (tableRows.length > 0) {
const firstRow = tableRows[0]
const fields = Object.keys(firstRow).filter(k => !k.startsWith('_'))
expect(fields.length).toBeGreaterThan(0)
}
})
it('should skip table extraction when disabled', async () => {
const filePath = path.join(fixturesPath, 'table.pdf')
const buffer = await fs.readFile(filePath)
const result = await handler.process(buffer, {
filename: 'table.pdf',
pdfExtractTables: false
})
// Should not have detected tables
expect(result.metadata.tableCount).toBe(0)
// Should not have table_row items
const tableRows = result.data.filter(item => item._type === 'table_row')
expect(tableRows.length).toBe(0)
})
})
describe('process - metadata extraction', () => {
it('should extract PDF metadata', async () => {
const filePath = path.join(fixturesPath, 'metadata.pdf')
const buffer = await fs.readFile(filePath)
const result = await handler.process(buffer, { filename: 'metadata.pdf' })
expect(result.metadata.pdfMetadata).toBeDefined()
expect(result.metadata.pdfMetadata.title).toBe('Test Document')
expect(result.metadata.pdfMetadata.author).toBe('Test Author')
expect(result.metadata.pdfMetadata.creator).toBe('Brainy Test Suite')
})
it('should handle PDFs without metadata gracefully', async () => {
const filePath = path.join(fixturesPath, 'simple.pdf')
const buffer = await fs.readFile(filePath)
const result = await handler.process(buffer, { filename: 'simple.pdf' })
expect(result.metadata.pdfMetadata).toBeDefined()
// Some fields may be null
expect(result.metadata.pdfMetadata).toHaveProperty('title')
expect(result.metadata.pdfMetadata).toHaveProperty('author')
})
})
describe('process - text statistics', () => {
it('should track total text length', async () => {
const filePath = path.join(fixturesPath, 'simple.pdf')
const buffer = await fs.readFile(filePath)
const result = await handler.process(buffer, { filename: 'simple.pdf' })
expect(result.metadata.textLength).toBeGreaterThan(0)
expect(typeof result.metadata.textLength).toBe('number')
})
it('should count extracted items', async () => {
const filePath = path.join(fixturesPath, 'simple.pdf')
const buffer = await fs.readFile(filePath)
const result = await handler.process(buffer, { filename: 'simple.pdf' })
expect(result.metadata.rowCount).toBe(result.data.length)
expect(result.metadata.rowCount).toBeGreaterThan(0)
})
})
describe('process - edge cases', () => {
it('should handle empty PDF', async () => {
const filePath = path.join(fixturesPath, 'empty.pdf')
const buffer = await fs.readFile(filePath)
const result = await handler.process(buffer, { filename: 'empty.pdf' })
expect(result.format).toBe('pdf')
expect(result.metadata.pageCount).toBe(1)
// Empty PDF may have 0 or minimal content
expect(result.data).toBeDefined()
})
it('should measure processing time', async () => {
const filePath = path.join(fixturesPath, 'simple.pdf')
const buffer = await fs.readFile(filePath)
const result = await handler.process(buffer, { filename: 'simple.pdf' })
expect(result.metadata.processingTime).toBeGreaterThan(0)
expect(typeof result.metadata.processingTime).toBe('number')
})
})
describe('process - data structure', () => {
it('should include type indicators', async () => {
const filePath = path.join(fixturesPath, 'simple.pdf')
const buffer = await fs.readFile(filePath)
const result = await handler.process(buffer, { filename: 'simple.pdf' })
// All items should have _type field
expect(result.data.every(item => '_type' in item)).toBe(true)
// Types should be valid
const validTypes = ['paragraph', 'table_row']
expect(result.data.every(item => validTypes.includes(item._type))).toBe(true)
})
it('should include index for paragraphs', async () => {
const filePath = path.join(fixturesPath, 'simple.pdf')
const buffer = await fs.readFile(filePath)
const result = await handler.process(buffer, { filename: 'simple.pdf' })
const paragraphs = result.data.filter(item => item._type === 'paragraph')
// Paragraphs should have _index field
expect(paragraphs.every(p => typeof p._index === 'number')).toBe(true)
})
})
describe('process - error handling', () => {
it('should throw error for invalid PDF', async () => {
const invalidData = Buffer.from('This is not a PDF file')
await expect(
handler.process(invalidData, { filename: 'invalid.pdf' })
).rejects.toThrow('PDF parsing failed')
})
it('should throw error for corrupted PDF', async () => {
const corruptedPDF = Buffer.concat([
Buffer.from('%PDF-1.4\n'),
Buffer.from('corrupted data that is not valid PDF structure')
])
await expect(
handler.process(corruptedPDF, { filename: 'corrupted.pdf' })
).rejects.toThrow('PDF parsing failed')
})
})
})

233
tests/fixtures/import/empty.pdf vendored Normal file
View file

@ -0,0 +1,233 @@
%PDF-1.3
%ºß¬à
3 0 obj
<</Type /Page
/Parent 1 0 R
/Resources 2 0 R
/MediaBox [0 0 595.2799999999999727 841.8899999999999864]
/Contents 4 0 R
>>
endobj
4 0 obj
<<
/Length 24
>>
stream
0.5670000000000001 w
0 G
endstream
endobj
1 0 obj
<</Type /Pages
/Kids [3 0 R ]
/Count 1
>>
endobj
5 0 obj
<<
/Type /Font
/BaseFont /Helvetica
/Subtype /Type1
/Encoding /WinAnsiEncoding
/FirstChar 32
/LastChar 255
>>
endobj
6 0 obj
<<
/Type /Font
/BaseFont /Helvetica-Bold
/Subtype /Type1
/Encoding /WinAnsiEncoding
/FirstChar 32
/LastChar 255
>>
endobj
7 0 obj
<<
/Type /Font
/BaseFont /Helvetica-Oblique
/Subtype /Type1
/Encoding /WinAnsiEncoding
/FirstChar 32
/LastChar 255
>>
endobj
8 0 obj
<<
/Type /Font
/BaseFont /Helvetica-BoldOblique
/Subtype /Type1
/Encoding /WinAnsiEncoding
/FirstChar 32
/LastChar 255
>>
endobj
9 0 obj
<<
/Type /Font
/BaseFont /Courier
/Subtype /Type1
/Encoding /WinAnsiEncoding
/FirstChar 32
/LastChar 255
>>
endobj
10 0 obj
<<
/Type /Font
/BaseFont /Courier-Bold
/Subtype /Type1
/Encoding /WinAnsiEncoding
/FirstChar 32
/LastChar 255
>>
endobj
11 0 obj
<<
/Type /Font
/BaseFont /Courier-Oblique
/Subtype /Type1
/Encoding /WinAnsiEncoding
/FirstChar 32
/LastChar 255
>>
endobj
12 0 obj
<<
/Type /Font
/BaseFont /Courier-BoldOblique
/Subtype /Type1
/Encoding /WinAnsiEncoding
/FirstChar 32
/LastChar 255
>>
endobj
13 0 obj
<<
/Type /Font
/BaseFont /Times-Roman
/Subtype /Type1
/Encoding /WinAnsiEncoding
/FirstChar 32
/LastChar 255
>>
endobj
14 0 obj
<<
/Type /Font
/BaseFont /Times-Bold
/Subtype /Type1
/Encoding /WinAnsiEncoding
/FirstChar 32
/LastChar 255
>>
endobj
15 0 obj
<<
/Type /Font
/BaseFont /Times-Italic
/Subtype /Type1
/Encoding /WinAnsiEncoding
/FirstChar 32
/LastChar 255
>>
endobj
16 0 obj
<<
/Type /Font
/BaseFont /Times-BoldItalic
/Subtype /Type1
/Encoding /WinAnsiEncoding
/FirstChar 32
/LastChar 255
>>
endobj
17 0 obj
<<
/Type /Font
/BaseFont /ZapfDingbats
/Subtype /Type1
/FirstChar 32
/LastChar 255
>>
endobj
18 0 obj
<<
/Type /Font
/BaseFont /Symbol
/Subtype /Type1
/FirstChar 32
/LastChar 255
>>
endobj
2 0 obj
<<
/ProcSet [/PDF /Text /ImageB /ImageC /ImageI]
/Font <<
/F1 5 0 R
/F2 6 0 R
/F3 7 0 R
/F4 8 0 R
/F5 9 0 R
/F6 10 0 R
/F7 11 0 R
/F8 12 0 R
/F9 13 0 R
/F10 14 0 R
/F11 15 0 R
/F12 16 0 R
/F13 17 0 R
/F14 18 0 R
>>
/XObject <<
>>
>>
endobj
19 0 obj
<<
/Producer (jsPDF 3.0.3)
/CreationDate (D:20251001162659-07'00')
>>
endobj
20 0 obj
<<
/Type /Catalog
/Pages 1 0 R
/OpenAction [3 0 R /FitH null]
/PageLayout /OneColumn
>>
endobj
xref
0 21
0000000000 65535 f
0000000226 00000 n
0000002043 00000 n
0000000015 00000 n
0000000152 00000 n
0000000283 00000 n
0000000408 00000 n
0000000538 00000 n
0000000671 00000 n
0000000808 00000 n
0000000931 00000 n
0000001060 00000 n
0000001192 00000 n
0000001328 00000 n
0000001456 00000 n
0000001583 00000 n
0000001712 00000 n
0000001845 00000 n
0000001947 00000 n
0000002291 00000 n
0000002377 00000 n
trailer
<<
/Size 21
/Root 20 0 R
/Info 19 0 R
/ID [ <49861707DD6C8FE0691713F0D6CEE800> <49861707DD6C8FE0691713F0D6CEE800> ]
>>
startxref
2481
%%EOF

BIN
tests/fixtures/import/empty.xlsx vendored Normal file

Binary file not shown.

109
tests/fixtures/import/generate-excel.ts vendored Normal file
View file

@ -0,0 +1,109 @@
/**
* Script to generate Excel test fixtures
* Run with: npx tsx tests/fixtures/import/generate-excel.ts
*/
import * as XLSX from 'xlsx'
import * as path from 'path'
import { fileURLToPath } from 'url'
const __filename = fileURLToPath(import.meta.url)
const __dirname = path.dirname(__filename)
// Simple single-sheet workbook
function createSimpleWorkbook() {
const data = [
['Name', 'Age', 'Department', 'Salary', 'Active'],
['Alice Johnson', 28, 'Engineering', 95000, true],
['Bob Smith', 35, 'Sales', 75000, true],
['Charlie Brown', 42, 'Engineering', 105000, false],
['Diana Prince', 31, 'Marketing', 85000, true]
]
const ws = XLSX.utils.aoa_to_sheet(data)
const wb = XLSX.utils.book_new()
XLSX.utils.book_append_sheet(wb, ws, 'Employees')
XLSX.writeFile(wb, path.join(__dirname, 'simple.xlsx'))
console.log('✅ Created simple.xlsx')
}
// Multi-sheet workbook
function createMultiSheetWorkbook() {
const wb = XLSX.utils.book_new()
// Sheet 1: Products
const products = [
['ID', 'Product', 'Price', 'Stock', 'Category'],
[1, 'Laptop', 999.99, 50, 'Electronics'],
[2, 'Mouse', 29.99, 200, 'Electronics'],
[3, 'Desk', 299.99, 30, 'Furniture'],
[4, 'Chair', 199.99, 45, 'Furniture']
]
const ws1 = XLSX.utils.aoa_to_sheet(products)
XLSX.utils.book_append_sheet(wb, ws1, 'Products')
// Sheet 2: Orders
const orders = [
['OrderID', 'ProductID', 'Quantity', 'Date', 'Status'],
[1001, 1, 2, '2024-01-15', 'Shipped'],
[1002, 2, 5, '2024-01-16', 'Delivered'],
[1003, 3, 1, '2024-01-17', 'Processing']
]
const ws2 = XLSX.utils.aoa_to_sheet(orders)
XLSX.utils.book_append_sheet(wb, ws2, 'Orders')
// Sheet 3: Customers
const customers = [
['CustomerID', 'Name', 'Email', 'Country'],
[101, 'John Doe', 'john@example.com', 'USA'],
[102, 'Jane Smith', 'jane@example.com', 'Canada'],
[103, 'Bob Wilson', 'bob@example.com', 'UK']
]
const ws3 = XLSX.utils.aoa_to_sheet(customers)
XLSX.utils.book_append_sheet(wb, ws3, 'Customers')
XLSX.writeFile(wb, path.join(__dirname, 'multi-sheet.xlsx'))
console.log('✅ Created multi-sheet.xlsx')
}
// Workbook with types
function createTypesWorkbook() {
const data = [
['ID', 'Name', 'Score', 'Percentage', 'Date', 'Active'],
[1, 'Alice', 95.5, 0.955, new Date('2024-01-15'), true],
[2, 'Bob', 87.3, 0.873, new Date('2024-02-20'), false],
[3, 'Charlie', 92.8, 0.928, new Date('2024-03-10'), true]
]
const ws = XLSX.utils.aoa_to_sheet(data)
const wb = XLSX.utils.book_new()
XLSX.utils.book_append_sheet(wb, ws, 'Scores')
XLSX.writeFile(wb, path.join(__dirname, 'types.xlsx'))
console.log('✅ Created types.xlsx')
}
// Empty workbook (edge case)
function createEmptyWorkbook() {
const ws = XLSX.utils.aoa_to_sheet([['Header1', 'Header2', 'Header3']])
const wb = XLSX.utils.book_new()
XLSX.utils.book_append_sheet(wb, ws, 'Empty')
XLSX.writeFile(wb, path.join(__dirname, 'empty.xlsx'))
console.log('✅ Created empty.xlsx')
}
// Generate all fixtures
async function main() {
console.log('Generating Excel test fixtures...\n')
createSimpleWorkbook()
createMultiSheetWorkbook()
createTypesWorkbook()
createEmptyWorkbook()
console.log('\n✨ All Excel fixtures generated!')
}
main().catch(console.error)

147
tests/fixtures/import/generate-pdf.ts vendored Normal file
View file

@ -0,0 +1,147 @@
/**
* Script to generate PDF test fixtures
* Run with: npx tsx tests/fixtures/import/generate-pdf.ts
*/
import { jsPDF } from 'jspdf'
import * as path from 'path'
import { fileURLToPath } from 'url'
import { writeFileSync } from 'fs'
const __filename = fileURLToPath(import.meta.url)
const __dirname = path.dirname(__filename)
// Simple text PDF
function createSimpleTextPDF() {
const doc = new jsPDF()
doc.setFontSize(16)
doc.text('Simple PDF Document', 20, 20)
doc.setFontSize(12)
doc.text('This is a test PDF with simple text content.', 20, 40)
doc.text('It contains multiple paragraphs to test text extraction.', 20, 50)
doc.text('Second paragraph starts here.', 20, 70)
doc.text('This paragraph has multiple sentences. Each sentence adds more text.', 20, 80)
doc.text('Third paragraph demonstrates extraction capabilities.', 20, 100)
const pdfBuffer = Buffer.from(doc.output('arraybuffer'))
writeFileSync(path.join(__dirname, 'simple.pdf'), pdfBuffer)
console.log('✅ Created simple.pdf')
}
// PDF with table-like structure
function createTablePDF() {
const doc = new jsPDF()
doc.setFontSize(14)
doc.text('Employee Records', 20, 20)
// Table headers
doc.setFontSize(10)
doc.setFont(undefined, 'bold')
doc.text('Name', 20, 40)
doc.text('Age', 80, 40)
doc.text('Department', 120, 40)
// Table rows
doc.setFont(undefined, 'normal')
doc.text('Alice Johnson', 20, 50)
doc.text('28', 80, 50)
doc.text('Engineering', 120, 50)
doc.text('Bob Smith', 20, 60)
doc.text('35', 80, 60)
doc.text('Sales', 120, 60)
doc.text('Charlie Brown', 20, 70)
doc.text('42', 80, 70)
doc.text('Engineering', 120, 70)
const pdfBuffer = Buffer.from(doc.output('arraybuffer'))
writeFileSync(path.join(__dirname, 'table.pdf'), pdfBuffer)
console.log('✅ Created table.pdf')
}
// Multi-page PDF
function createMultiPagePDF() {
const doc = new jsPDF()
// Page 1
doc.setFontSize(16)
doc.text('Page 1: Introduction', 20, 20)
doc.setFontSize(12)
doc.text('This is the first page of a multi-page PDF document.', 20, 40)
doc.text('It demonstrates that the PDF handler can process multiple pages.', 20, 50)
// Page 2
doc.addPage()
doc.setFontSize(16)
doc.text('Page 2: Content', 20, 20)
doc.setFontSize(12)
doc.text('This is the second page with different content.', 20, 40)
doc.text('Each page should be extracted separately.', 20, 50)
// Page 3
doc.addPage()
doc.setFontSize(16)
doc.text('Page 3: Conclusion', 20, 20)
doc.setFontSize(12)
doc.text('This is the final page of the document.', 20, 40)
const pdfBuffer = Buffer.from(doc.output('arraybuffer'))
writeFileSync(path.join(__dirname, 'multi-page.pdf'), pdfBuffer)
console.log('✅ Created multi-page.pdf')
}
// PDF with metadata
function createMetadataPDF() {
const doc = new jsPDF()
// Set metadata
doc.setProperties({
title: 'Test Document',
subject: 'PDF Testing',
author: 'Test Author',
keywords: 'pdf, test, metadata',
creator: 'Brainy Test Suite'
})
doc.setFontSize(14)
doc.text('PDF with Metadata', 20, 20)
doc.setFontSize(12)
doc.text('This PDF has metadata that should be extracted.', 20, 40)
const pdfBuffer = Buffer.from(doc.output('arraybuffer'))
writeFileSync(path.join(__dirname, 'metadata.pdf'), pdfBuffer)
console.log('✅ Created metadata.pdf')
}
// Empty PDF (edge case)
function createEmptyPDF() {
const doc = new jsPDF()
// Just a blank page
doc.text('', 20, 20)
const pdfBuffer = Buffer.from(doc.output('arraybuffer'))
writeFileSync(path.join(__dirname, 'empty.pdf'), pdfBuffer)
console.log('✅ Created empty.pdf')
}
// Generate all fixtures
async function main() {
console.log('Generating PDF test fixtures...\n')
createSimpleTextPDF()
createTablePDF()
createMultiPagePDF()
createMetadataPDF()
createEmptyPDF()
console.log('\n✨ All PDF fixtures generated!')
}
main().catch(console.error)

252
tests/fixtures/import/metadata.pdf vendored Normal file
View file

@ -0,0 +1,252 @@
%PDF-1.3
%ºß¬à
3 0 obj
<</Type /Page
/Parent 1 0 R
/Resources 2 0 R
/MediaBox [0 0 595.2799999999999727 841.8899999999999864]
/Contents 4 0 R
>>
endobj
4 0 obj
<<
/Length 274
>>
stream
0.5670000000000001 w
0 G
BT
/F1 14 Tf
16.0999999999999979 TL
0 g
56.6929133858267775 785.1970866141732586 Td
(PDF with Metadata) Tj
ET
BT
/F1 12 Tf
13.7999999999999989 TL
0 g
56.6929133858267775 728.5041732283464171 Td
(This PDF has metadata that should be extracted.) Tj
ET
endstream
endobj
1 0 obj
<</Type /Pages
/Kids [3 0 R ]
/Count 1
>>
endobj
5 0 obj
<<
/Type /Font
/BaseFont /Helvetica
/Subtype /Type1
/Encoding /WinAnsiEncoding
/FirstChar 32
/LastChar 255
>>
endobj
6 0 obj
<<
/Type /Font
/BaseFont /Helvetica-Bold
/Subtype /Type1
/Encoding /WinAnsiEncoding
/FirstChar 32
/LastChar 255
>>
endobj
7 0 obj
<<
/Type /Font
/BaseFont /Helvetica-Oblique
/Subtype /Type1
/Encoding /WinAnsiEncoding
/FirstChar 32
/LastChar 255
>>
endobj
8 0 obj
<<
/Type /Font
/BaseFont /Helvetica-BoldOblique
/Subtype /Type1
/Encoding /WinAnsiEncoding
/FirstChar 32
/LastChar 255
>>
endobj
9 0 obj
<<
/Type /Font
/BaseFont /Courier
/Subtype /Type1
/Encoding /WinAnsiEncoding
/FirstChar 32
/LastChar 255
>>
endobj
10 0 obj
<<
/Type /Font
/BaseFont /Courier-Bold
/Subtype /Type1
/Encoding /WinAnsiEncoding
/FirstChar 32
/LastChar 255
>>
endobj
11 0 obj
<<
/Type /Font
/BaseFont /Courier-Oblique
/Subtype /Type1
/Encoding /WinAnsiEncoding
/FirstChar 32
/LastChar 255
>>
endobj
12 0 obj
<<
/Type /Font
/BaseFont /Courier-BoldOblique
/Subtype /Type1
/Encoding /WinAnsiEncoding
/FirstChar 32
/LastChar 255
>>
endobj
13 0 obj
<<
/Type /Font
/BaseFont /Times-Roman
/Subtype /Type1
/Encoding /WinAnsiEncoding
/FirstChar 32
/LastChar 255
>>
endobj
14 0 obj
<<
/Type /Font
/BaseFont /Times-Bold
/Subtype /Type1
/Encoding /WinAnsiEncoding
/FirstChar 32
/LastChar 255
>>
endobj
15 0 obj
<<
/Type /Font
/BaseFont /Times-Italic
/Subtype /Type1
/Encoding /WinAnsiEncoding
/FirstChar 32
/LastChar 255
>>
endobj
16 0 obj
<<
/Type /Font
/BaseFont /Times-BoldItalic
/Subtype /Type1
/Encoding /WinAnsiEncoding
/FirstChar 32
/LastChar 255
>>
endobj
17 0 obj
<<
/Type /Font
/BaseFont /ZapfDingbats
/Subtype /Type1
/FirstChar 32
/LastChar 255
>>
endobj
18 0 obj
<<
/Type /Font
/BaseFont /Symbol
/Subtype /Type1
/FirstChar 32
/LastChar 255
>>
endobj
2 0 obj
<<
/ProcSet [/PDF /Text /ImageB /ImageC /ImageI]
/Font <<
/F1 5 0 R
/F2 6 0 R
/F3 7 0 R
/F4 8 0 R
/F5 9 0 R
/F6 10 0 R
/F7 11 0 R
/F8 12 0 R
/F9 13 0 R
/F10 14 0 R
/F11 15 0 R
/F12 16 0 R
/F13 17 0 R
/F14 18 0 R
>>
/XObject <<
>>
>>
endobj
19 0 obj
<<
/Producer (jsPDF 3.0.3)
/Title (Test Document)
/Subject (PDF Testing)
/Author (Test Author)
/Keywords (pdf, test, metadata)
/Creator (Brainy Test Suite)
/CreationDate (D:20251001162659-07'00')
>>
endobj
20 0 obj
<<
/Type /Catalog
/Pages 1 0 R
/OpenAction [3 0 R /FitH null]
/PageLayout /OneColumn
>>
endobj
xref
0 21
0000000000 65535 f
0000000477 00000 n
0000002294 00000 n
0000000015 00000 n
0000000152 00000 n
0000000534 00000 n
0000000659 00000 n
0000000789 00000 n
0000000922 00000 n
0000001059 00000 n
0000001182 00000 n
0000001311 00000 n
0000001443 00000 n
0000001579 00000 n
0000001707 00000 n
0000001834 00000 n
0000001963 00000 n
0000002096 00000 n
0000002198 00000 n
0000002542 00000 n
0000002757 00000 n
trailer
<<
/Size 21
/Root 20 0 R
/Info 19 0 R
/ID [ <A029D212DBC492DA6593A84070F2BDFD> <A029D212DBC492DA6593A84070F2BDFD> ]
>>
startxref
2861
%%EOF

327
tests/fixtures/import/multi-page.pdf vendored Normal file
View file

@ -0,0 +1,327 @@
%PDF-1.3
%ºß¬à
3 0 obj
<</Type /Page
/Parent 1 0 R
/Resources 2 0 R
/MediaBox [0 0 595.2799999999999727 841.8899999999999864]
/Contents 4 0 R
>>
endobj
4 0 obj
<<
/Length 437
>>
stream
0.5670000000000001 w
0 G
BT
/F1 16 Tf
18.3999999999999986 TL
0 g
56.6929133858267775 785.1970866141732586 Td
(Page 1: Introduction) Tj
ET
BT
/F1 12 Tf
13.7999999999999989 TL
0 g
56.6929133858267775 728.5041732283464171 Td
(This is the first page of a multi-page PDF document.) Tj
ET
BT
/F1 12 Tf
13.7999999999999989 TL
0 g
56.6929133858267775 700.15771653543311 Td
(It demonstrates that the PDF handler can process multiple pages.) Tj
ET
endstream
endobj
5 0 obj
<</Type /Page
/Parent 1 0 R
/Resources 2 0 R
/MediaBox [0 0 595.2799999999999727 841.8899999999999864]
/Contents 6 0 R
>>
endobj
6 0 obj
<<
/Length 404
>>
stream
0.5670000000000001 w
0 G
BT
/F1 16 Tf
18.3999999999999986 TL
0 g
56.6929133858267775 785.1970866141732586 Td
(Page 2: Content) Tj
ET
BT
/F1 12 Tf
13.7999999999999989 TL
0 g
56.6929133858267775 728.5041732283464171 Td
(This is the second page with different content.) Tj
ET
BT
/F1 12 Tf
13.7999999999999989 TL
0 g
56.6929133858267775 700.15771653543311 Td
(Each page should be extracted separately.) Tj
ET
endstream
endobj
7 0 obj
<</Type /Page
/Parent 1 0 R
/Resources 2 0 R
/MediaBox [0 0 595.2799999999999727 841.8899999999999864]
/Contents 8 0 R
>>
endobj
8 0 obj
<<
/Length 267
>>
stream
0.5670000000000001 w
0 G
BT
/F1 16 Tf
18.3999999999999986 TL
0 g
56.6929133858267775 785.1970866141732586 Td
(Page 3: Conclusion) Tj
ET
BT
/F1 12 Tf
13.7999999999999989 TL
0 g
56.6929133858267775 728.5041732283464171 Td
(This is the final page of the document.) Tj
ET
endstream
endobj
1 0 obj
<</Type /Pages
/Kids [3 0 R 5 0 R 7 0 R ]
/Count 3
>>
endobj
9 0 obj
<<
/Type /Font
/BaseFont /Helvetica
/Subtype /Type1
/Encoding /WinAnsiEncoding
/FirstChar 32
/LastChar 255
>>
endobj
10 0 obj
<<
/Type /Font
/BaseFont /Helvetica-Bold
/Subtype /Type1
/Encoding /WinAnsiEncoding
/FirstChar 32
/LastChar 255
>>
endobj
11 0 obj
<<
/Type /Font
/BaseFont /Helvetica-Oblique
/Subtype /Type1
/Encoding /WinAnsiEncoding
/FirstChar 32
/LastChar 255
>>
endobj
12 0 obj
<<
/Type /Font
/BaseFont /Helvetica-BoldOblique
/Subtype /Type1
/Encoding /WinAnsiEncoding
/FirstChar 32
/LastChar 255
>>
endobj
13 0 obj
<<
/Type /Font
/BaseFont /Courier
/Subtype /Type1
/Encoding /WinAnsiEncoding
/FirstChar 32
/LastChar 255
>>
endobj
14 0 obj
<<
/Type /Font
/BaseFont /Courier-Bold
/Subtype /Type1
/Encoding /WinAnsiEncoding
/FirstChar 32
/LastChar 255
>>
endobj
15 0 obj
<<
/Type /Font
/BaseFont /Courier-Oblique
/Subtype /Type1
/Encoding /WinAnsiEncoding
/FirstChar 32
/LastChar 255
>>
endobj
16 0 obj
<<
/Type /Font
/BaseFont /Courier-BoldOblique
/Subtype /Type1
/Encoding /WinAnsiEncoding
/FirstChar 32
/LastChar 255
>>
endobj
17 0 obj
<<
/Type /Font
/BaseFont /Times-Roman
/Subtype /Type1
/Encoding /WinAnsiEncoding
/FirstChar 32
/LastChar 255
>>
endobj
18 0 obj
<<
/Type /Font
/BaseFont /Times-Bold
/Subtype /Type1
/Encoding /WinAnsiEncoding
/FirstChar 32
/LastChar 255
>>
endobj
19 0 obj
<<
/Type /Font
/BaseFont /Times-Italic
/Subtype /Type1
/Encoding /WinAnsiEncoding
/FirstChar 32
/LastChar 255
>>
endobj
20 0 obj
<<
/Type /Font
/BaseFont /Times-BoldItalic
/Subtype /Type1
/Encoding /WinAnsiEncoding
/FirstChar 32
/LastChar 255
>>
endobj
21 0 obj
<<
/Type /Font
/BaseFont /ZapfDingbats
/Subtype /Type1
/FirstChar 32
/LastChar 255
>>
endobj
22 0 obj
<<
/Type /Font
/BaseFont /Symbol
/Subtype /Type1
/FirstChar 32
/LastChar 255
>>
endobj
2 0 obj
<<
/ProcSet [/PDF /Text /ImageB /ImageC /ImageI]
/Font <<
/F1 9 0 R
/F2 10 0 R
/F3 11 0 R
/F4 12 0 R
/F5 13 0 R
/F6 14 0 R
/F7 15 0 R
/F8 16 0 R
/F9 17 0 R
/F10 18 0 R
/F11 19 0 R
/F12 20 0 R
/F13 21 0 R
/F14 22 0 R
>>
/XObject <<
>>
>>
endobj
23 0 obj
<<
/Producer (jsPDF 3.0.3)
/CreationDate (D:20251001162659-07'00')
>>
endobj
24 0 obj
<<
/Type /Catalog
/Pages 1 0 R
/OpenAction [3 0 R /FitH null]
/PageLayout /OneColumn
>>
endobj
xref
0 25
0000000000 65535 f
0000001687 00000 n
0000003520 00000 n
0000000015 00000 n
0000000152 00000 n
0000000640 00000 n
0000000777 00000 n
0000001232 00000 n
0000001369 00000 n
0000001756 00000 n
0000001881 00000 n
0000002012 00000 n
0000002146 00000 n
0000002284 00000 n
0000002408 00000 n
0000002537 00000 n
0000002669 00000 n
0000002805 00000 n
0000002933 00000 n
0000003060 00000 n
0000003189 00000 n
0000003322 00000 n
0000003424 00000 n
0000003772 00000 n
0000003858 00000 n
trailer
<<
/Size 25
/Root 24 0 R
/Info 23 0 R
/ID [ <04E6AF3F674215986B311755247B62DD> <04E6AF3F674215986B311755247B62DD> ]
>>
startxref
3962
%%EOF

BIN
tests/fixtures/import/multi-sheet.xlsx vendored Normal file

Binary file not shown.

5
tests/fixtures/import/semicolon.csv vendored Normal file
View file

@ -0,0 +1,5 @@
product;price;quantity;inStock
Laptop;999.99;10;yes
Mouse;29.99;100;yes
Keyboard;79.99;50;no
Monitor;299.99;25;yes
1 product price quantity inStock
2 Laptop 999.99 10 yes
3 Mouse 29.99 100 yes
4 Keyboard 79.99 50 no
5 Monitor 299.99 25 yes

5
tests/fixtures/import/simple.csv vendored Normal file
View file

@ -0,0 +1,5 @@
name,age,email,active
John Doe,30,john@example.com,true
Jane Smith,25,jane@example.com,false
Bob Johnson,45,bob@example.com,true
Alice Williams,35,alice@example.com,true
1 name age email active
2 John Doe 30 john@example.com true
3 Jane Smith 25 jane@example.com false
4 Bob Johnson 45 bob@example.com true
5 Alice Williams 35 alice@example.com true

275
tests/fixtures/import/simple.pdf vendored Normal file
View file

@ -0,0 +1,275 @@
%PDF-1.3
%ºß¬à
3 0 obj
<</Type /Page
/Parent 1 0 R
/Resources 2 0 R
/MediaBox [0 0 595.2799999999999727 841.8899999999999864]
/Contents 4 0 R
>>
endobj
4 0 obj
<<
/Length 847
>>
stream
0.5670000000000001 w
0 G
BT
/F1 16 Tf
18.3999999999999986 TL
0 g
56.6929133858267775 785.1970866141732586 Td
(Simple PDF Document) Tj
ET
BT
/F1 12 Tf
13.7999999999999989 TL
0 g
56.6929133858267775 728.5041732283464171 Td
(This is a test PDF with simple text content.) Tj
ET
BT
/F1 12 Tf
13.7999999999999989 TL
0 g
56.6929133858267775 700.15771653543311 Td
(It contains multiple paragraphs to test text extraction.) Tj
ET
BT
/F1 12 Tf
13.7999999999999989 TL
0 g
56.6929133858267775 643.4648031496062686 Td
(Second paragraph starts here.) Tj
ET
BT
/F1 12 Tf
13.7999999999999989 TL
0 g
56.6929133858267775 615.1183464566928478 Td
(This paragraph has multiple sentences. Each sentence adds more text.) Tj
ET
BT
/F1 12 Tf
13.7999999999999989 TL
0 g
56.6929133858267775 558.42543307086612 Td
(Third paragraph demonstrates extraction capabilities.) Tj
ET
endstream
endobj
1 0 obj
<</Type /Pages
/Kids [3 0 R ]
/Count 1
>>
endobj
5 0 obj
<<
/Type /Font
/BaseFont /Helvetica
/Subtype /Type1
/Encoding /WinAnsiEncoding
/FirstChar 32
/LastChar 255
>>
endobj
6 0 obj
<<
/Type /Font
/BaseFont /Helvetica-Bold
/Subtype /Type1
/Encoding /WinAnsiEncoding
/FirstChar 32
/LastChar 255
>>
endobj
7 0 obj
<<
/Type /Font
/BaseFont /Helvetica-Oblique
/Subtype /Type1
/Encoding /WinAnsiEncoding
/FirstChar 32
/LastChar 255
>>
endobj
8 0 obj
<<
/Type /Font
/BaseFont /Helvetica-BoldOblique
/Subtype /Type1
/Encoding /WinAnsiEncoding
/FirstChar 32
/LastChar 255
>>
endobj
9 0 obj
<<
/Type /Font
/BaseFont /Courier
/Subtype /Type1
/Encoding /WinAnsiEncoding
/FirstChar 32
/LastChar 255
>>
endobj
10 0 obj
<<
/Type /Font
/BaseFont /Courier-Bold
/Subtype /Type1
/Encoding /WinAnsiEncoding
/FirstChar 32
/LastChar 255
>>
endobj
11 0 obj
<<
/Type /Font
/BaseFont /Courier-Oblique
/Subtype /Type1
/Encoding /WinAnsiEncoding
/FirstChar 32
/LastChar 255
>>
endobj
12 0 obj
<<
/Type /Font
/BaseFont /Courier-BoldOblique
/Subtype /Type1
/Encoding /WinAnsiEncoding
/FirstChar 32
/LastChar 255
>>
endobj
13 0 obj
<<
/Type /Font
/BaseFont /Times-Roman
/Subtype /Type1
/Encoding /WinAnsiEncoding
/FirstChar 32
/LastChar 255
>>
endobj
14 0 obj
<<
/Type /Font
/BaseFont /Times-Bold
/Subtype /Type1
/Encoding /WinAnsiEncoding
/FirstChar 32
/LastChar 255
>>
endobj
15 0 obj
<<
/Type /Font
/BaseFont /Times-Italic
/Subtype /Type1
/Encoding /WinAnsiEncoding
/FirstChar 32
/LastChar 255
>>
endobj
16 0 obj
<<
/Type /Font
/BaseFont /Times-BoldItalic
/Subtype /Type1
/Encoding /WinAnsiEncoding
/FirstChar 32
/LastChar 255
>>
endobj
17 0 obj
<<
/Type /Font
/BaseFont /ZapfDingbats
/Subtype /Type1
/FirstChar 32
/LastChar 255
>>
endobj
18 0 obj
<<
/Type /Font
/BaseFont /Symbol
/Subtype /Type1
/FirstChar 32
/LastChar 255
>>
endobj
2 0 obj
<<
/ProcSet [/PDF /Text /ImageB /ImageC /ImageI]
/Font <<
/F1 5 0 R
/F2 6 0 R
/F3 7 0 R
/F4 8 0 R
/F5 9 0 R
/F6 10 0 R
/F7 11 0 R
/F8 12 0 R
/F9 13 0 R
/F10 14 0 R
/F11 15 0 R
/F12 16 0 R
/F13 17 0 R
/F14 18 0 R
>>
/XObject <<
>>
>>
endobj
19 0 obj
<<
/Producer (jsPDF 3.0.3)
/CreationDate (D:20251001162659-07'00')
>>
endobj
20 0 obj
<<
/Type /Catalog
/Pages 1 0 R
/OpenAction [3 0 R /FitH null]
/PageLayout /OneColumn
>>
endobj
xref
0 21
0000000000 65535 f
0000001050 00000 n
0000002867 00000 n
0000000015 00000 n
0000000152 00000 n
0000001107 00000 n
0000001232 00000 n
0000001362 00000 n
0000001495 00000 n
0000001632 00000 n
0000001755 00000 n
0000001884 00000 n
0000002016 00000 n
0000002152 00000 n
0000002280 00000 n
0000002407 00000 n
0000002536 00000 n
0000002669 00000 n
0000002771 00000 n
0000003115 00000 n
0000003201 00000 n
trailer
<<
/Size 21
/Root 20 0 R
/Info 19 0 R
/ID [ <0F5A6C1D434F4F388C03021EE84E0E57> <0F5A6C1D434F4F388C03021EE84E0E57> ]
>>
startxref
3305
%%EOF

BIN
tests/fixtures/import/simple.xlsx vendored Normal file

Binary file not shown.

View file

@ -0,0 +1,5 @@
city country population area
Tokyo Japan 13960000 2191
New York USA 8336000 783.8
London UK 9002000 1572
Paris France 2161000 105.4
1 city country population area
2 Tokyo Japan 13960000 2191
3 New York USA 8336000 783.8
4 London UK 9002000 1572
5 Paris France 2161000 105.4

324
tests/fixtures/import/table.pdf vendored Normal file
View file

@ -0,0 +1,324 @@
%PDF-1.3
%ºß¬à
3 0 obj
<</Type /Page
/Parent 1 0 R
/Resources 2 0 R
/MediaBox [0 0 595.2799999999999727 841.8899999999999864]
/Contents 4 0 R
>>
endobj
4 0 obj
<<
/Length 1152
>>
stream
0.5670000000000001 w
0 G
BT
/F1 14 Tf
16.0999999999999979 TL
0 g
56.6929133858267775 785.1970866141732586 Td
(Employee Records) Tj
ET
BT
/F2 10 Tf
11.5 TL
0 g
56.6929133858267775 728.5041732283464171 Td
(Name) Tj
ET
BT
/F2 10 Tf
11.5 TL
0 g
226.7716535433071101 728.5041732283464171 Td
(Age) Tj
ET
BT
/F2 10 Tf
11.5 TL
0 g
340.157480314960651 728.5041732283464171 Td
(Department) Tj
ET
BT
/F1 10 Tf
11.5 TL
0 g
56.6929133858267775 700.15771653543311 Td
(Alice Johnson) Tj
ET
BT
/F1 10 Tf
11.5 TL
0 g
226.7716535433071101 700.15771653543311 Td
(28) Tj
ET
BT
/F1 10 Tf
11.5 TL
0 g
340.157480314960651 700.15771653543311 Td
(Engineering) Tj
ET
BT
/F1 10 Tf
11.5 TL
0 g
56.6929133858267775 671.8112598425196893 Td
(Bob Smith) Tj
ET
BT
/F1 10 Tf
11.5 TL
0 g
226.7716535433071101 671.8112598425196893 Td
(35) Tj
ET
BT
/F1 10 Tf
11.5 TL
0 g
340.157480314960651 671.8112598425196893 Td
(Sales) Tj
ET
BT
/F1 10 Tf
11.5 TL
0 g
56.6929133858267775 643.4648031496062686 Td
(Charlie Brown) Tj
ET
BT
/F1 10 Tf
11.5 TL
0 g
226.7716535433071101 643.4648031496062686 Td
(42) Tj
ET
BT
/F1 10 Tf
11.5 TL
0 g
340.157480314960651 643.4648031496062686 Td
(Engineering) Tj
ET
endstream
endobj
1 0 obj
<</Type /Pages
/Kids [3 0 R ]
/Count 1
>>
endobj
5 0 obj
<<
/Type /Font
/BaseFont /Helvetica
/Subtype /Type1
/Encoding /WinAnsiEncoding
/FirstChar 32
/LastChar 255
>>
endobj
6 0 obj
<<
/Type /Font
/BaseFont /Helvetica-Bold
/Subtype /Type1
/Encoding /WinAnsiEncoding
/FirstChar 32
/LastChar 255
>>
endobj
7 0 obj
<<
/Type /Font
/BaseFont /Helvetica-Oblique
/Subtype /Type1
/Encoding /WinAnsiEncoding
/FirstChar 32
/LastChar 255
>>
endobj
8 0 obj
<<
/Type /Font
/BaseFont /Helvetica-BoldOblique
/Subtype /Type1
/Encoding /WinAnsiEncoding
/FirstChar 32
/LastChar 255
>>
endobj
9 0 obj
<<
/Type /Font
/BaseFont /Courier
/Subtype /Type1
/Encoding /WinAnsiEncoding
/FirstChar 32
/LastChar 255
>>
endobj
10 0 obj
<<
/Type /Font
/BaseFont /Courier-Bold
/Subtype /Type1
/Encoding /WinAnsiEncoding
/FirstChar 32
/LastChar 255
>>
endobj
11 0 obj
<<
/Type /Font
/BaseFont /Courier-Oblique
/Subtype /Type1
/Encoding /WinAnsiEncoding
/FirstChar 32
/LastChar 255
>>
endobj
12 0 obj
<<
/Type /Font
/BaseFont /Courier-BoldOblique
/Subtype /Type1
/Encoding /WinAnsiEncoding
/FirstChar 32
/LastChar 255
>>
endobj
13 0 obj
<<
/Type /Font
/BaseFont /Times-Roman
/Subtype /Type1
/Encoding /WinAnsiEncoding
/FirstChar 32
/LastChar 255
>>
endobj
14 0 obj
<<
/Type /Font
/BaseFont /Times-Bold
/Subtype /Type1
/Encoding /WinAnsiEncoding
/FirstChar 32
/LastChar 255
>>
endobj
15 0 obj
<<
/Type /Font
/BaseFont /Times-Italic
/Subtype /Type1
/Encoding /WinAnsiEncoding
/FirstChar 32
/LastChar 255
>>
endobj
16 0 obj
<<
/Type /Font
/BaseFont /Times-BoldItalic
/Subtype /Type1
/Encoding /WinAnsiEncoding
/FirstChar 32
/LastChar 255
>>
endobj
17 0 obj
<<
/Type /Font
/BaseFont /ZapfDingbats
/Subtype /Type1
/FirstChar 32
/LastChar 255
>>
endobj
18 0 obj
<<
/Type /Font
/BaseFont /Symbol
/Subtype /Type1
/FirstChar 32
/LastChar 255
>>
endobj
2 0 obj
<<
/ProcSet [/PDF /Text /ImageB /ImageC /ImageI]
/Font <<
/F1 5 0 R
/F2 6 0 R
/F3 7 0 R
/F4 8 0 R
/F5 9 0 R
/F6 10 0 R
/F7 11 0 R
/F8 12 0 R
/F9 13 0 R
/F10 14 0 R
/F11 15 0 R
/F12 16 0 R
/F13 17 0 R
/F14 18 0 R
>>
/XObject <<
>>
>>
endobj
19 0 obj
<<
/Producer (jsPDF 3.0.3)
/CreationDate (D:20251001162659-07'00')
>>
endobj
20 0 obj
<<
/Type /Catalog
/Pages 1 0 R
/OpenAction [3 0 R /FitH null]
/PageLayout /OneColumn
>>
endobj
xref
0 21
0000000000 65535 f
0000001356 00000 n
0000003173 00000 n
0000000015 00000 n
0000000152 00000 n
0000001413 00000 n
0000001538 00000 n
0000001668 00000 n
0000001801 00000 n
0000001938 00000 n
0000002061 00000 n
0000002190 00000 n
0000002322 00000 n
0000002458 00000 n
0000002586 00000 n
0000002713 00000 n
0000002842 00000 n
0000002975 00000 n
0000003077 00000 n
0000003421 00000 n
0000003507 00000 n
trailer
<<
/Size 21
/Root 20 0 R
/Info 19 0 R
/ID [ <A34A2614AA2553099D2A387E65B754AA> <A34A2614AA2553099D2A387E65B754AA> ]
>>
startxref
3611
%%EOF

4
tests/fixtures/import/types.csv vendored Normal file
View file

@ -0,0 +1,4 @@
id,name,score,percentage,date,timestamp,active
1,Alice,95,0.95,2024-01-15,2024-01-15T10:30:00Z,true
2,Bob,87.5,0.875,2024-02-20,2024-02-20T14:45:00Z,false
3,Charlie,92,0.92,2024-03-10,2024-03-10T09:15:00Z,true
1 id name score percentage date timestamp active
2 1 Alice 95 0.95 2024-01-15 2024-01-15T10:30:00Z true
3 2 Bob 87.5 0.875 2024-02-20 2024-02-20T14:45:00Z false
4 3 Charlie 92 0.92 2024-03-10 2024-03-10T09:15:00Z true

BIN
tests/fixtures/import/types.xlsx vendored Normal file

Binary file not shown.