/** * CSV Format Handler * Handles CSV files with: * - Automatic encoding detection * - Automatic delimiter detection * - Streaming for large files * - Type inference */ import { parse } from 'csv-parse/sync' import { detect as detectEncoding } from 'chardet' import { BaseFormatHandler } from './base.js' import { FormatHandlerOptions, ProcessedData } from '../types.js' export class CSVHandler extends BaseFormatHandler { readonly format = 'csv' canHandle(data: Buffer | string | { filename?: string, ext?: string }): boolean { const ext = this.detectExtension(data) if (ext === 'csv' || ext === 'tsv' || ext === 'txt') return true // Check content if it's a buffer if (Buffer.isBuffer(data)) { const sample = data.slice(0, 1024).toString('utf-8') return this.looksLikeCSV(sample) } if (typeof data === 'string') { return this.looksLikeCSV(data.slice(0, 1024)) } return false } async process(data: Buffer | string, options: FormatHandlerOptions): Promise { const startTime = Date.now() // Convert to buffer if string const buffer = Buffer.isBuffer(data) ? data : Buffer.from(data, 'utf-8') // Detect encoding const detectedEncoding = options.encoding || this.detectEncodingSafe(buffer) const text = buffer.toString(detectedEncoding as BufferEncoding) // Detect delimiter if not specified const delimiter = options.csvDelimiter || this.detectDelimiter(text) // Parse CSV const hasHeaders = options.csvHeaders !== false const maxRows = options.maxRows try { const records = parse(text, { columns: hasHeaders, skip_empty_lines: true, trim: true, delimiter, relax_column_count: true, to: maxRows, cast: false // We'll do type inference ourselves }) // Convert to array of objects const data = Array.isArray(records) ? records : [records] // Infer types and convert values const fields = data.length > 0 ? Object.keys(data[0]) : [] const types = this.inferFieldTypes(data) const convertedData = data.map(row => { const converted: Record = {} for (const [key, value] of Object.entries(row)) { converted[key] = this.convertValue(value, types[key] || 'string') } return converted }) const processingTime = Date.now() - startTime return { format: this.format, data: convertedData, metadata: this.createMetadata( convertedData.length, fields, processingTime, { encoding: detectedEncoding, delimiter, hasHeaders, types } ), filename: options.filename } } catch (error) { throw new Error(`CSV parsing failed: ${error instanceof Error ? error.message : String(error)}`) } } /** * Check if text looks like CSV */ private looksLikeCSV(text: string): boolean { const lines = text.split('\n').filter(l => l.trim()) if (lines.length < 2) return false // Check for common delimiters const delimiters = [',', ';', '\t', '|'] for (const delimiter of delimiters) { const firstCount = (lines[0].match(new RegExp(`\\${delimiter}`, 'g')) || []).length if (firstCount === 0) continue const secondCount = (lines[1].match(new RegExp(`\\${delimiter}`, 'g')) || []).length if (firstCount === secondCount) return true } return false } /** * Detect CSV delimiter */ private detectDelimiter(text: string): string { const sample = text.split('\n').slice(0, 10).join('\n') const delimiters = [',', ';', '\t', '|'] const counts: Record = {} for (const delimiter of delimiters) { const lines = sample.split('\n').filter(l => l.trim()) if (lines.length < 2) continue // Count delimiter in first line const firstCount = (lines[0].match(new RegExp(`\\${delimiter}`, 'g')) || []).length if (firstCount === 0) continue // Check if count is consistent across lines let consistent = true for (let i = 1; i < Math.min(5, lines.length); i++) { const count = (lines[i].match(new RegExp(`\\${delimiter}`, 'g')) || []).length if (count !== firstCount) { consistent = false break } } if (consistent) { counts[delimiter] = firstCount } } // Return delimiter with highest count const best = Object.entries(counts).sort((a, b) => b[1] - a[1])[0] return best ? best[0] : ',' } /** * Detect encoding safely (with fallback) */ private detectEncodingSafe(buffer: Buffer): string { try { const detected = detectEncoding(buffer) if (!detected) return 'utf-8' // Normalize encoding to Node.js-supported names return this.normalizeEncoding(detected) } catch { return 'utf-8' } } /** * Normalize encoding names to Node.js-supported encodings */ private normalizeEncoding(encoding: string): string { const normalized = encoding.toLowerCase().replace(/[_-]/g, '') // Map common encodings to Node.js names const mappings: Record = { 'iso88591': 'latin1', 'iso88592': 'latin1', 'iso88593': 'latin1', 'iso88594': 'latin1', 'iso88595': 'latin1', 'iso88596': 'latin1', 'iso88597': 'latin1', 'iso88598': 'latin1', 'iso88599': 'latin1', 'iso885910': 'latin1', 'iso885913': 'latin1', 'iso885914': 'latin1', 'iso885915': 'latin1', 'iso885916': 'latin1', 'usascii': 'ascii', 'utf8': 'utf8', 'utf16le': 'utf16le', 'utf16be': 'utf16le', 'windows1252': 'latin1', 'windows1251': 'utf8', // Cyrillic - best effort 'big5': 'utf8', // Chinese - best effort 'gbk': 'utf8', // Chinese - best effort 'gb2312': 'utf8', // Chinese - best effort 'shiftjis': 'utf8', // Japanese - best effort 'eucjp': 'utf8', // Japanese - best effort 'euckr': 'utf8' // Korean - best effort } return mappings[normalized] || 'utf8' } }