refactor: remove augmentation system and semantic type matching
Remove the entire augmentation pipeline infrastructure (52 files, ~15,000 lines) and the semantic type matching system. These were unused middleware layers adding complexity without value. What was removed: - src/augmentations/ directory (all augmentation implementations) - src/augmentationManager.ts (pipeline orchestrator) - src/types/augmentations.ts, src/types/pipelineTypes.ts - src/shared/default-augmentations.ts - Semantic type suggestion (BrainyTypes.suggestNoun/suggestVerb) - src/utils/typeMatching/ (embedding-based type matcher) What was preserved by relocating: - Import handlers (CSV, PDF, Excel) -> src/importers/handlers/ - NeuralImportAugmentation -> src/cortex/neuralImportAugmentation.ts - Type matching utilities -> heuristic inference in consumers What was simplified: - brainy.ts: operations call storage directly (no execute() wrapper) - IntegrationBase: standalone class (no BaseAugmentation parent) - BrainyTypes: validation-only (nouns, verbs, isValid*, get*) - Pipeline: direct execution (no augmentation interception) - index.ts: removed TypeSuggestion, suggestType exports - package.json: removed stale types/augmentations export Build passes, 1176 tests pass, 0 failures.
This commit is contained in:
parent
ac7a1f772c
commit
d1db3510be
97 changed files with 349 additions and 19705 deletions
204
src/importers/handlers/base.ts
Normal file
204
src/importers/handlers/base.ts
Normal file
|
|
@ -0,0 +1,204 @@
|
|||
/**
|
||||
* Base Format Handler
|
||||
* Abstract class providing common functionality for all format handlers
|
||||
*
|
||||
* Uses MimeTypeDetector for comprehensive file type detection (2000+ types)
|
||||
*/
|
||||
|
||||
import { FormatHandler, FormatHandlerOptions, ProcessedData } from './types.js'
|
||||
import { mimeDetector } from '../../vfs/MimeTypeDetector.js'
|
||||
|
||||
export abstract class BaseFormatHandler implements FormatHandler {
|
||||
abstract readonly format: string
|
||||
|
||||
abstract process(data: Buffer | string, options: FormatHandlerOptions): Promise<ProcessedData>
|
||||
|
||||
abstract canHandle(data: Buffer | string | { filename?: string, ext?: string }): boolean
|
||||
|
||||
/**
|
||||
* Detect file extension from various inputs
|
||||
*/
|
||||
protected detectExtension(data: Buffer | string | { filename?: string, ext?: string }): string | null {
|
||||
if (typeof data === 'object' && 'filename' in data && data.filename) {
|
||||
return this.getExtension(data.filename)
|
||||
}
|
||||
if (typeof data === 'object' && 'ext' in data && data.ext) {
|
||||
return data.ext.toLowerCase().replace(/^\./, '')
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
/**
|
||||
* Extract extension from filename
|
||||
*/
|
||||
protected getExtension(filename: string): string {
|
||||
const match = filename.match(/\.([^.]+)$/)
|
||||
return match ? match[1].toLowerCase() : ''
|
||||
}
|
||||
|
||||
/**
|
||||
* Get MIME type using MimeTypeDetector
|
||||
*
|
||||
* Supports 2000+ file types via mime library + custom developer types
|
||||
*/
|
||||
protected getMimeType(data: Buffer | string | { filename?: string }): string {
|
||||
if (typeof data === 'object' && 'filename' in data && data.filename) {
|
||||
return mimeDetector.detectMimeType(data.filename)
|
||||
}
|
||||
if (Buffer.isBuffer(data)) {
|
||||
// For buffers, we don't have a filename, so return generic
|
||||
return 'application/octet-stream'
|
||||
}
|
||||
return 'text/plain'
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if MIME type matches expected format
|
||||
*
|
||||
* @param mimeType - MIME type to check
|
||||
* @param patterns - Patterns to match (e.g., ['text/csv', 'application/vnd.ms-excel'])
|
||||
*/
|
||||
protected mimeTypeMatches(mimeType: string, patterns: string[]): boolean {
|
||||
return patterns.some(pattern => {
|
||||
if (pattern.endsWith('/*')) {
|
||||
const prefix = pattern.slice(0, -2)
|
||||
return mimeType.startsWith(prefix)
|
||||
}
|
||||
return mimeType === pattern
|
||||
})
|
||||
}
|
||||
|
||||
/**
|
||||
* Infer field types from data
|
||||
* Analyzes multiple rows to determine the most appropriate type
|
||||
*/
|
||||
protected inferFieldTypes(data: Array<Record<string, any>>): Record<string, string> {
|
||||
if (data.length === 0) return {}
|
||||
|
||||
const types: Record<string, string> = {}
|
||||
const firstRow = data[0]
|
||||
const sampleSize = Math.min(10, data.length)
|
||||
|
||||
for (const key of Object.keys(firstRow)) {
|
||||
// Check first few rows to get more accurate type
|
||||
const sampleTypes = new Set<string>()
|
||||
|
||||
for (let i = 0; i < sampleSize; i++) {
|
||||
const value = data[i][key]
|
||||
const type = this.inferType(value)
|
||||
sampleTypes.add(type)
|
||||
}
|
||||
|
||||
// If we see both integer and float, use float
|
||||
if (sampleTypes.has('float') || (sampleTypes.has('integer') && sampleTypes.has('float'))) {
|
||||
types[key] = 'float'
|
||||
} else if (sampleTypes.has('integer')) {
|
||||
types[key] = 'integer'
|
||||
} else if (sampleTypes.has('date')) {
|
||||
types[key] = 'date'
|
||||
} else if (sampleTypes.has('boolean')) {
|
||||
types[key] = 'boolean'
|
||||
} else {
|
||||
types[key] = 'string'
|
||||
}
|
||||
}
|
||||
|
||||
return types
|
||||
}
|
||||
|
||||
/**
|
||||
* Infer type of a single value
|
||||
*/
|
||||
protected inferType(value: any): string {
|
||||
if (value === null || value === undefined || value === '') return 'string'
|
||||
|
||||
if (typeof value === 'number') return 'number'
|
||||
if (typeof value === 'boolean') return 'boolean'
|
||||
|
||||
if (typeof value === 'string') {
|
||||
// Check if it's a number
|
||||
if (/^-?\d+$/.test(value)) return 'integer'
|
||||
if (/^-?\d+\.\d+$/.test(value)) return 'float'
|
||||
|
||||
// Check if it's a date
|
||||
if (this.isDateString(value)) return 'date'
|
||||
|
||||
// Check if it's a boolean
|
||||
if (/^(true|false|yes|no|y|n)$/i.test(value)) return 'boolean'
|
||||
}
|
||||
|
||||
return 'string'
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if string looks like a date
|
||||
*/
|
||||
protected isDateString(value: string): boolean {
|
||||
// ISO 8601
|
||||
if (/^\d{4}-\d{2}-\d{2}/.test(value)) return true
|
||||
|
||||
// Common date formats
|
||||
if (/^\d{1,2}\/\d{1,2}\/\d{2,4}$/.test(value)) return true
|
||||
if (/^\d{1,2}-\d{1,2}-\d{2,4}$/.test(value)) return true
|
||||
|
||||
return false
|
||||
}
|
||||
|
||||
/**
|
||||
* Sanitize field names for use as object keys
|
||||
*/
|
||||
protected sanitizeFieldName(name: string): string {
|
||||
return name
|
||||
.trim()
|
||||
.replace(/[^a-zA-Z0-9_\s-]/g, '')
|
||||
.replace(/\s+/g, '_')
|
||||
.replace(/-+/g, '_')
|
||||
.replace(/_+/g, '_')
|
||||
.replace(/^_|_$/g, '')
|
||||
|| 'field'
|
||||
}
|
||||
|
||||
/**
|
||||
* Convert value to appropriate type
|
||||
*/
|
||||
protected convertValue(value: any, type: string): any {
|
||||
if (value === null || value === undefined || value === '') return null
|
||||
|
||||
switch (type) {
|
||||
case 'integer':
|
||||
return parseInt(String(value), 10)
|
||||
|
||||
case 'float':
|
||||
case 'number':
|
||||
return parseFloat(String(value))
|
||||
|
||||
case 'boolean':
|
||||
if (typeof value === 'boolean') return value
|
||||
const str = String(value).toLowerCase()
|
||||
return ['true', 'yes', 'y', '1'].includes(str)
|
||||
|
||||
case 'date':
|
||||
return new Date(value)
|
||||
|
||||
default:
|
||||
return value
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Create metadata object with common fields
|
||||
*/
|
||||
protected createMetadata(
|
||||
rowCount: number,
|
||||
fields: string[],
|
||||
processingTime: number,
|
||||
extra: Record<string, any> = {}
|
||||
): ProcessedData['metadata'] {
|
||||
return {
|
||||
rowCount,
|
||||
fields,
|
||||
processingTime,
|
||||
...extra
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in a new issue