refactor(8.0): remove dead, unreachable, and unwired modules
Pre-GA dead-code sweep. A deterministic import-reachability walk from the
package's real entry points (the exports map, the CLI bin, the conversation
surface) found 34 source modules that nothing reachable imports — they
compile and ship as dist output that no consumer or internal path can ever
reach. All were superseded duplicates, abandoned parallel implementations,
or built-but-never-wired features:
- superseded duplicates of live modules: an older "unified" entry, a
standalone neural-import variant, a static NLP processor + its matcher,
a parallel API-types module, a duplicate progress-types module
- an abandoned import path (orchestrator + entity deduplicator + barrels)
left behind when ingestion moved to the coordinator
- unwired feature modules (instance pool, import presets, cached
embeddings, relationship-confidence scorer) reachable only from tests
- dead leaf utilities (write buffer, deleted-items index, bounded
registry, two crypto shims, a cache manager, a structured logger, a
stale v5 type-migration helper, a browser-only FS type shim) and
several dead re-export barrels
- a CLI catalog command wired into no command
Also removed two tests that only exercised deleted modules, repointed two
VFS tests off a deleted barrel onto the implementation module, and deleted
one example that demoed removed features.
Verified: clean build (both tsc passes), unit 1505 + integration 607 green,
and a re-run of the reachability walk reports zero remaining orphans.
Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
parent
03d654061f
commit
bf0afe8563
39 changed files with 2 additions and 10497 deletions
|
|
@ -1,789 +0,0 @@
|
|||
/**
|
||||
* Smart Import Orchestrator
|
||||
*
|
||||
* Coordinates the entire smart import pipeline:
|
||||
* 1. Extract entities/relationships using SmartExcelImporter
|
||||
* 2. Create entities and relationships in Brainy
|
||||
* 3. Organize into VFS structure using VFSStructureGenerator
|
||||
*
|
||||
* NO MOCKS - Production-ready implementation
|
||||
*/
|
||||
|
||||
import { Brainy } from '../brainy.js'
|
||||
import { VirtualFileSystem } from '../vfs/VirtualFileSystem.js'
|
||||
import { NounType, VerbType } from '../types/graphTypes.js'
|
||||
import { splitNounMetadataRecord } from '../types/reservedFields.js'
|
||||
import { SmartExcelImporter, SmartExcelOptions, SmartExcelResult } from './SmartExcelImporter.js'
|
||||
import { SmartPDFImporter, SmartPDFOptions, SmartPDFResult } from './SmartPDFImporter.js'
|
||||
import { SmartCSVImporter, SmartCSVOptions, SmartCSVResult } from './SmartCSVImporter.js'
|
||||
import { SmartJSONImporter, SmartJSONOptions, SmartJSONResult } from './SmartJSONImporter.js'
|
||||
import { SmartMarkdownImporter, SmartMarkdownOptions, SmartMarkdownResult } from './SmartMarkdownImporter.js'
|
||||
import { VFSStructureGenerator, VFSStructureOptions } from './VFSStructureGenerator.js'
|
||||
|
||||
export interface SmartImportOptions extends SmartExcelOptions {
|
||||
/** Create VFS structure */
|
||||
createVFSStructure?: boolean
|
||||
|
||||
/** VFS root path */
|
||||
vfsRootPath?: string
|
||||
|
||||
/** VFS grouping strategy */
|
||||
vfsGroupBy?: 'type' | 'sheet' | 'flat' | 'custom'
|
||||
|
||||
/** Create entities in Brainy */
|
||||
createEntities?: boolean
|
||||
|
||||
/** Create relationships in Brainy */
|
||||
createRelationships?: boolean
|
||||
|
||||
/** Source filename */
|
||||
filename?: string
|
||||
|
||||
/**
|
||||
* Default subtype tag for entities + relationships this importer creates when
|
||||
* the extractor doesn't set one. See `ValidImportOptions.defaultSubtype` —
|
||||
* same semantics, same precedence (extractor > caller default > `'imported'`).
|
||||
* Added 7.30.1.
|
||||
*/
|
||||
defaultSubtype?: string
|
||||
}
|
||||
|
||||
export interface SmartImportProgress {
|
||||
phase: 'parsing' | 'extracting' | 'creating' | 'relationships' | 'organizing' | 'complete'
|
||||
message: string
|
||||
processed: number
|
||||
total: number
|
||||
entities: number
|
||||
relationships: number
|
||||
}
|
||||
|
||||
export interface SmartImportResult {
|
||||
success: boolean
|
||||
|
||||
/** Extraction results */
|
||||
extraction: SmartExcelResult
|
||||
|
||||
/** Created entity IDs */
|
||||
entityIds: string[]
|
||||
|
||||
/** Created relationship IDs */
|
||||
relationshipIds: string[]
|
||||
|
||||
/** VFS structure created */
|
||||
vfsStructure?: {
|
||||
rootPath: string
|
||||
directories: string[]
|
||||
files: number
|
||||
}
|
||||
|
||||
/** Overall statistics */
|
||||
stats: {
|
||||
rowsProcessed: number
|
||||
entitiesCreated: number
|
||||
relationshipsCreated: number
|
||||
filesCreated: number
|
||||
totalTime: number
|
||||
}
|
||||
|
||||
/** Any errors encountered */
|
||||
errors: string[]
|
||||
}
|
||||
|
||||
/**
|
||||
* SmartImportOrchestrator - Main entry point for smart imports
|
||||
*/
|
||||
export class SmartImportOrchestrator {
|
||||
private brain: Brainy
|
||||
private excelImporter: SmartExcelImporter
|
||||
private pdfImporter: SmartPDFImporter
|
||||
private csvImporter: SmartCSVImporter
|
||||
private jsonImporter: SmartJSONImporter
|
||||
private markdownImporter: SmartMarkdownImporter
|
||||
private vfsGenerator: VFSStructureGenerator
|
||||
|
||||
constructor(brain: Brainy) {
|
||||
this.brain = brain
|
||||
this.excelImporter = new SmartExcelImporter(brain)
|
||||
this.pdfImporter = new SmartPDFImporter(brain)
|
||||
this.csvImporter = new SmartCSVImporter(brain)
|
||||
this.jsonImporter = new SmartJSONImporter(brain)
|
||||
this.markdownImporter = new SmartMarkdownImporter(brain)
|
||||
this.vfsGenerator = new VFSStructureGenerator(brain)
|
||||
}
|
||||
|
||||
/**
|
||||
* Initialize the orchestrator
|
||||
*/
|
||||
async init(): Promise<void> {
|
||||
await this.excelImporter.init()
|
||||
await this.pdfImporter.init()
|
||||
await this.csvImporter.init()
|
||||
await this.jsonImporter.init()
|
||||
await this.markdownImporter.init()
|
||||
await this.vfsGenerator.init()
|
||||
}
|
||||
|
||||
/**
|
||||
* Import Excel file with full pipeline
|
||||
*/
|
||||
async importExcel(
|
||||
buffer: Buffer,
|
||||
options: SmartImportOptions = {},
|
||||
onProgress?: (progress: SmartImportProgress) => void
|
||||
): Promise<SmartImportResult> {
|
||||
const startTime = Date.now()
|
||||
const result: SmartImportResult = {
|
||||
success: false,
|
||||
// Typed boundary: populated in the extraction phase below. If extraction
|
||||
// throws, the error path returns with this still null (pre-existing
|
||||
// contract — consumers check `success`/`errors` before reading it).
|
||||
extraction: null as unknown as SmartExcelResult,
|
||||
entityIds: [],
|
||||
relationshipIds: [],
|
||||
stats: {
|
||||
rowsProcessed: 0,
|
||||
entitiesCreated: 0,
|
||||
relationshipsCreated: 0,
|
||||
filesCreated: 0,
|
||||
totalTime: 0
|
||||
},
|
||||
errors: []
|
||||
}
|
||||
|
||||
try {
|
||||
// Phase 1: Extract entities and relationships
|
||||
onProgress?.({
|
||||
phase: 'extracting',
|
||||
message: 'Extracting entities and relationships...',
|
||||
processed: 0,
|
||||
total: 0,
|
||||
entities: 0,
|
||||
relationships: 0
|
||||
})
|
||||
|
||||
result.extraction = await this.excelImporter.extract(buffer, {
|
||||
...options,
|
||||
onProgress: (stats) => {
|
||||
onProgress?.({
|
||||
phase: 'extracting',
|
||||
message: `Processing row ${stats.processed}/${stats.total}...`,
|
||||
processed: stats.processed,
|
||||
total: stats.total,
|
||||
entities: stats.entities,
|
||||
relationships: stats.relationships
|
||||
})
|
||||
}
|
||||
})
|
||||
|
||||
result.stats.rowsProcessed = result.extraction.rowsProcessed
|
||||
|
||||
// Phase 2: Create entities in Brainy
|
||||
if (options.createEntities !== false) {
|
||||
onProgress?.({
|
||||
phase: 'creating',
|
||||
message: 'Creating entities in knowledge graph...',
|
||||
processed: 0,
|
||||
total: result.extraction.rows.length,
|
||||
entities: 0,
|
||||
relationships: 0
|
||||
})
|
||||
|
||||
for (let i = 0; i < result.extraction.rows.length; i++) {
|
||||
const extracted = result.extraction.rows[i]
|
||||
|
||||
try {
|
||||
// Create main entity. Subtype precedence: extractor-set → caller default
|
||||
// → Brainy default `'imported'` (added 7.30.1).
|
||||
const entityId = await this.brain.add({
|
||||
data: extracted.entity.description,
|
||||
type: extracted.entity.type,
|
||||
subtype:
|
||||
(extracted.entity as typeof extracted.entity & { subtype?: string })
|
||||
.subtype ?? options.defaultSubtype ?? 'imported',
|
||||
confidence: extracted.entity.confidence, // reserved field — dedicated param, not metadata
|
||||
metadata: {
|
||||
// Strip reserved keys an extractor may have smuggled into the bag
|
||||
// (8.0 reservedFieldPolicy defaults to 'throw').
|
||||
...splitNounMetadataRecord(extracted.entity.metadata).custom,
|
||||
name: extracted.entity.name,
|
||||
importedFrom: 'smart-import'
|
||||
}
|
||||
})
|
||||
|
||||
result.entityIds.push(entityId)
|
||||
result.stats.entitiesCreated++
|
||||
|
||||
// Update entity ID in extraction result
|
||||
extracted.entity.id = entityId
|
||||
|
||||
onProgress?.({
|
||||
phase: 'creating',
|
||||
message: `Created entity: ${extracted.entity.name}`,
|
||||
processed: i + 1,
|
||||
total: result.extraction.rows.length,
|
||||
entities: result.entityIds.length,
|
||||
relationships: result.relationshipIds.length
|
||||
})
|
||||
} catch (error: any) {
|
||||
result.errors.push(`Failed to create entity ${extracted.entity.name}: ${error.message}`)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Phase 3: Create relationships
|
||||
if (options.createRelationships !== false && options.createEntities !== false) {
|
||||
onProgress?.({
|
||||
phase: 'creating',
|
||||
message: 'Preparing relationships...',
|
||||
processed: 0,
|
||||
total: result.extraction.rows.length,
|
||||
entities: result.entityIds.length,
|
||||
relationships: 0
|
||||
})
|
||||
|
||||
// Build entity name -> ID map
|
||||
const entityMap = new Map<string, string>()
|
||||
for (const extracted of result.extraction.rows) {
|
||||
entityMap.set(extracted.entity.name.toLowerCase(), extracted.entity.id)
|
||||
}
|
||||
|
||||
// Collect all relationship parameters
|
||||
const relationshipParams: Array<{from: string; to: string; type: VerbType; subtype?: string; confidence?: number; metadata?: any}> = []
|
||||
|
||||
for (const extracted of result.extraction.rows) {
|
||||
for (const rel of extracted.relationships) {
|
||||
try {
|
||||
// Find target entity ID
|
||||
let toEntityId: string | undefined
|
||||
|
||||
// Try to find by name in our extracted entities
|
||||
for (const otherExtracted of result.extraction.rows) {
|
||||
if (rel.to.toLowerCase().includes(otherExtracted.entity.name.toLowerCase()) ||
|
||||
otherExtracted.entity.name.toLowerCase().includes(rel.to.toLowerCase())) {
|
||||
toEntityId = otherExtracted.entity.id
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
// If not found, create a placeholder entity. `import-placeholder` marks
|
||||
// these as synthetic targets so consumers can distinguish them from real
|
||||
// imports and downstream dedup can consolidate (added 7.30.1).
|
||||
if (!toEntityId) {
|
||||
toEntityId = await this.brain.add({
|
||||
data: rel.to,
|
||||
type: NounType.Thing,
|
||||
subtype: 'import-placeholder',
|
||||
metadata: {
|
||||
name: rel.to,
|
||||
placeholder: true,
|
||||
extractedFrom: extracted.entity.name
|
||||
}
|
||||
})
|
||||
result.entityIds.push(toEntityId)
|
||||
}
|
||||
|
||||
// Collect relationship parameter. Subtype precedence: extractor-set rel
|
||||
// subtype → caller default → Brainy default `'imported'` (added 7.30.1).
|
||||
relationshipParams.push({
|
||||
from: extracted.entity.id,
|
||||
to: toEntityId,
|
||||
type: rel.type,
|
||||
subtype:
|
||||
(rel as typeof rel & { subtype?: string }).subtype ??
|
||||
options.defaultSubtype ?? 'imported',
|
||||
confidence: rel.confidence, // reserved field — dedicated param, not metadata
|
||||
metadata: {
|
||||
evidence: rel.evidence
|
||||
}
|
||||
})
|
||||
} catch (error: any) {
|
||||
result.errors.push(`Failed to prepare relationship: ${error.message}`)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Batch create all relationships with progress
|
||||
if (relationshipParams.length > 0) {
|
||||
onProgress?.({
|
||||
phase: 'relationships',
|
||||
message: 'Building relationships...',
|
||||
processed: 0,
|
||||
total: relationshipParams.length,
|
||||
entities: result.entityIds.length,
|
||||
relationships: 0
|
||||
})
|
||||
|
||||
try {
|
||||
const relationshipIds = await this.brain.relateMany({
|
||||
items: relationshipParams,
|
||||
parallel: true,
|
||||
chunkSize: 100,
|
||||
continueOnError: true,
|
||||
onProgress: (done, total) => {
|
||||
onProgress?.({
|
||||
phase: 'relationships',
|
||||
message: `Building relationships: ${done}/${total}`,
|
||||
processed: done,
|
||||
total: total,
|
||||
entities: result.entityIds.length,
|
||||
relationships: done
|
||||
})
|
||||
}
|
||||
})
|
||||
|
||||
result.relationshipIds = relationshipIds
|
||||
result.stats.relationshipsCreated = relationshipIds.length
|
||||
} catch (error: any) {
|
||||
result.errors.push(`Failed to create relationships: ${error.message}`)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Phase 4: Create VFS structure
|
||||
if (options.createVFSStructure !== false) {
|
||||
onProgress?.({
|
||||
phase: 'organizing',
|
||||
message: 'Organizing into file structure...',
|
||||
processed: 0,
|
||||
total: result.extraction.rows.length,
|
||||
entities: result.entityIds.length,
|
||||
relationships: result.relationshipIds.length
|
||||
})
|
||||
|
||||
const vfsOptions: VFSStructureOptions = {
|
||||
rootPath: options.vfsRootPath || '/imports/' + (options.filename || 'import'),
|
||||
groupBy: options.vfsGroupBy || 'type',
|
||||
preserveSource: true,
|
||||
sourceBuffer: buffer,
|
||||
sourceFilename: options.filename || 'import.xlsx',
|
||||
createRelationshipFile: true,
|
||||
createMetadataFile: true
|
||||
}
|
||||
|
||||
const vfsResult = await this.vfsGenerator.generate(result.extraction, vfsOptions)
|
||||
|
||||
result.vfsStructure = {
|
||||
rootPath: vfsResult.rootPath,
|
||||
directories: vfsResult.directories,
|
||||
files: vfsResult.files.length
|
||||
}
|
||||
|
||||
result.stats.filesCreated = vfsResult.files.length
|
||||
}
|
||||
|
||||
// Complete
|
||||
result.success = result.errors.length === 0
|
||||
result.stats.totalTime = Date.now() - startTime
|
||||
|
||||
onProgress?.({
|
||||
phase: 'complete',
|
||||
message: `Import complete: ${result.stats.entitiesCreated} entities, ${result.stats.relationshipsCreated} relationships`,
|
||||
processed: result.extraction.rows.length,
|
||||
total: result.extraction.rows.length,
|
||||
entities: result.stats.entitiesCreated,
|
||||
relationships: result.stats.relationshipsCreated
|
||||
})
|
||||
|
||||
} catch (error: any) {
|
||||
result.errors.push(`Import failed: ${error.message}`)
|
||||
result.success = false
|
||||
}
|
||||
|
||||
return result
|
||||
}
|
||||
|
||||
/**
|
||||
* Import PDF file with full pipeline
|
||||
*/
|
||||
async importPDF(
|
||||
buffer: Buffer,
|
||||
options: SmartImportOptions & SmartPDFOptions = {},
|
||||
onProgress?: (progress: SmartImportProgress) => void
|
||||
): Promise<SmartImportResult> {
|
||||
const startTime = Date.now()
|
||||
const result: SmartImportResult = {
|
||||
success: false,
|
||||
// Typed boundary: populated after extraction (see importExcel).
|
||||
extraction: null as unknown as SmartExcelResult,
|
||||
entityIds: [],
|
||||
relationshipIds: [],
|
||||
stats: {
|
||||
rowsProcessed: 0,
|
||||
entitiesCreated: 0,
|
||||
relationshipsCreated: 0,
|
||||
filesCreated: 0,
|
||||
totalTime: 0
|
||||
},
|
||||
errors: []
|
||||
}
|
||||
|
||||
try {
|
||||
// Phase 1: Extract from PDF
|
||||
onProgress?.({ phase: 'extracting', message: 'Extracting from PDF...', processed: 0, total: 0, entities: 0, relationships: 0 })
|
||||
|
||||
const pdfResult = await this.pdfImporter.extract(buffer, options)
|
||||
|
||||
// Convert PDF result to Excel-like format for processing
|
||||
result.extraction = this.convertPDFToExcelFormat(pdfResult)
|
||||
result.stats.rowsProcessed = pdfResult.sectionsProcessed
|
||||
|
||||
// Phase 2 & 3: Create entities and relationships
|
||||
await this.createEntitiesAndRelationships(result, options, onProgress)
|
||||
|
||||
// Phase 4: Create VFS structure
|
||||
if (options.createVFSStructure !== false) {
|
||||
const vfsOptions: VFSStructureOptions = {
|
||||
rootPath: options.vfsRootPath || '/imports/' + (options.filename || 'import'),
|
||||
groupBy: options.vfsGroupBy || 'type',
|
||||
preserveSource: true,
|
||||
sourceBuffer: buffer,
|
||||
sourceFilename: options.filename || 'import.pdf',
|
||||
createRelationshipFile: true,
|
||||
createMetadataFile: true
|
||||
}
|
||||
const vfsResult = await this.vfsGenerator.generate(result.extraction, vfsOptions)
|
||||
result.vfsStructure = { rootPath: vfsResult.rootPath, directories: vfsResult.directories, files: vfsResult.files.length }
|
||||
result.stats.filesCreated = vfsResult.files.length
|
||||
}
|
||||
|
||||
result.success = result.errors.length === 0
|
||||
result.stats.totalTime = Date.now() - startTime
|
||||
onProgress?.({ phase: 'complete', message: `Import complete: ${result.stats.entitiesCreated} entities, ${result.stats.relationshipsCreated} relationships`, processed: result.stats.rowsProcessed, total: result.stats.rowsProcessed, entities: result.stats.entitiesCreated, relationships: result.stats.relationshipsCreated })
|
||||
|
||||
} catch (error: any) {
|
||||
result.errors.push(`PDF import failed: ${error.message}`)
|
||||
result.success = false
|
||||
}
|
||||
|
||||
return result
|
||||
}
|
||||
|
||||
/**
|
||||
* Import CSV file with full pipeline
|
||||
*/
|
||||
async importCSV(
|
||||
buffer: Buffer,
|
||||
options: SmartImportOptions & SmartCSVOptions = {},
|
||||
onProgress?: (progress: SmartImportProgress) => void
|
||||
): Promise<SmartImportResult> {
|
||||
// CSV is very similar to Excel, can reuse importExcel logic
|
||||
return this.importExcel(buffer, options, onProgress)
|
||||
}
|
||||
|
||||
/**
|
||||
* Import JSON data with full pipeline
|
||||
*/
|
||||
async importJSON(
|
||||
data: any,
|
||||
options: SmartImportOptions & SmartJSONOptions = {},
|
||||
onProgress?: (progress: SmartImportProgress) => void
|
||||
): Promise<SmartImportResult> {
|
||||
const startTime = Date.now()
|
||||
const result: SmartImportResult = {
|
||||
success: false,
|
||||
// Typed boundary: populated after extraction (see importExcel).
|
||||
extraction: null as unknown as SmartExcelResult,
|
||||
entityIds: [],
|
||||
relationshipIds: [],
|
||||
stats: {
|
||||
rowsProcessed: 0,
|
||||
entitiesCreated: 0,
|
||||
relationshipsCreated: 0,
|
||||
filesCreated: 0,
|
||||
totalTime: 0
|
||||
},
|
||||
errors: []
|
||||
}
|
||||
|
||||
try {
|
||||
onProgress?.({ phase: 'extracting', message: 'Extracting from JSON...', processed: 0, total: 0, entities: 0, relationships: 0 })
|
||||
|
||||
const jsonResult = await this.jsonImporter.extract(data, options)
|
||||
|
||||
result.extraction = this.convertJSONToExcelFormat(jsonResult)
|
||||
result.stats.rowsProcessed = jsonResult.nodesProcessed
|
||||
|
||||
await this.createEntitiesAndRelationships(result, options, onProgress)
|
||||
|
||||
if (options.createVFSStructure !== false) {
|
||||
const sourceBuffer = Buffer.from(typeof data === 'string' ? data : JSON.stringify(data, null, 2))
|
||||
const vfsOptions: VFSStructureOptions = {
|
||||
rootPath: options.vfsRootPath || '/imports/' + (options.filename || 'import'),
|
||||
groupBy: options.vfsGroupBy || 'type',
|
||||
preserveSource: true,
|
||||
sourceBuffer,
|
||||
sourceFilename: options.filename || 'import.json',
|
||||
createRelationshipFile: true,
|
||||
createMetadataFile: true
|
||||
}
|
||||
const vfsResult = await this.vfsGenerator.generate(result.extraction, vfsOptions)
|
||||
result.vfsStructure = { rootPath: vfsResult.rootPath, directories: vfsResult.directories, files: vfsResult.files.length }
|
||||
result.stats.filesCreated = vfsResult.files.length
|
||||
}
|
||||
|
||||
result.success = result.errors.length === 0
|
||||
result.stats.totalTime = Date.now() - startTime
|
||||
onProgress?.({ phase: 'complete', message: `Import complete: ${result.stats.entitiesCreated} entities, ${result.stats.relationshipsCreated} relationships`, processed: result.stats.rowsProcessed, total: result.stats.rowsProcessed, entities: result.stats.entitiesCreated, relationships: result.stats.relationshipsCreated })
|
||||
|
||||
} catch (error: any) {
|
||||
result.errors.push(`JSON import failed: ${error.message}`)
|
||||
result.success = false
|
||||
}
|
||||
|
||||
return result
|
||||
}
|
||||
|
||||
/**
|
||||
* Import Markdown content with full pipeline
|
||||
*/
|
||||
async importMarkdown(
|
||||
markdown: string,
|
||||
options: SmartImportOptions & SmartMarkdownOptions = {},
|
||||
onProgress?: (progress: SmartImportProgress) => void
|
||||
): Promise<SmartImportResult> {
|
||||
const startTime = Date.now()
|
||||
const result: SmartImportResult = {
|
||||
success: false,
|
||||
// Typed boundary: populated after extraction (see importExcel).
|
||||
extraction: null as unknown as SmartExcelResult,
|
||||
entityIds: [],
|
||||
relationshipIds: [],
|
||||
stats: {
|
||||
rowsProcessed: 0,
|
||||
entitiesCreated: 0,
|
||||
relationshipsCreated: 0,
|
||||
filesCreated: 0,
|
||||
totalTime: 0
|
||||
},
|
||||
errors: []
|
||||
}
|
||||
|
||||
try {
|
||||
onProgress?.({ phase: 'extracting', message: 'Extracting from Markdown...', processed: 0, total: 0, entities: 0, relationships: 0 })
|
||||
|
||||
const mdResult = await this.markdownImporter.extract(markdown, options)
|
||||
|
||||
result.extraction = this.convertMarkdownToExcelFormat(mdResult)
|
||||
result.stats.rowsProcessed = mdResult.sectionsProcessed
|
||||
|
||||
await this.createEntitiesAndRelationships(result, options, onProgress)
|
||||
|
||||
if (options.createVFSStructure !== false) {
|
||||
const sourceBuffer = Buffer.from(markdown, 'utf-8')
|
||||
const vfsOptions: VFSStructureOptions = {
|
||||
rootPath: options.vfsRootPath || '/imports/' + (options.filename || 'import'),
|
||||
groupBy: options.vfsGroupBy || 'type',
|
||||
preserveSource: true,
|
||||
sourceBuffer,
|
||||
sourceFilename: options.filename || 'import.md',
|
||||
createRelationshipFile: true,
|
||||
createMetadataFile: true
|
||||
}
|
||||
const vfsResult = await this.vfsGenerator.generate(result.extraction, vfsOptions)
|
||||
result.vfsStructure = { rootPath: vfsResult.rootPath, directories: vfsResult.directories, files: vfsResult.files.length }
|
||||
result.stats.filesCreated = vfsResult.files.length
|
||||
}
|
||||
|
||||
result.success = result.errors.length === 0
|
||||
result.stats.totalTime = Date.now() - startTime
|
||||
onProgress?.({ phase: 'complete', message: `Import complete: ${result.stats.entitiesCreated} entities, ${result.stats.relationshipsCreated} relationships`, processed: result.stats.rowsProcessed, total: result.stats.rowsProcessed, entities: result.stats.entitiesCreated, relationships: result.stats.relationshipsCreated })
|
||||
|
||||
} catch (error: any) {
|
||||
result.errors.push(`Markdown import failed: ${error.message}`)
|
||||
result.success = false
|
||||
}
|
||||
|
||||
return result
|
||||
}
|
||||
|
||||
/**
|
||||
* Helper: Create entities and relationships from extraction result
|
||||
*/
|
||||
private async createEntitiesAndRelationships(
|
||||
result: SmartImportResult,
|
||||
options: SmartImportOptions,
|
||||
onProgress?: (progress: SmartImportProgress) => void
|
||||
): Promise<void> {
|
||||
if (options.createEntities !== false) {
|
||||
onProgress?.({ phase: 'creating', message: 'Creating entities in knowledge graph...', processed: 0, total: result.extraction.rows.length, entities: 0, relationships: 0 })
|
||||
|
||||
for (let i = 0; i < result.extraction.rows.length; i++) {
|
||||
const extracted = result.extraction.rows[i]
|
||||
try {
|
||||
// Subtype precedence: extractor → caller default → `'imported'` (7.30.1).
|
||||
const entityId = await this.brain.add({
|
||||
data: extracted.entity.description,
|
||||
type: extracted.entity.type,
|
||||
subtype:
|
||||
(extracted.entity as typeof extracted.entity & { subtype?: string })
|
||||
.subtype ?? options.defaultSubtype ?? 'imported',
|
||||
confidence: extracted.entity.confidence, // reserved field — dedicated param, not metadata
|
||||
metadata: { ...splitNounMetadataRecord(extracted.entity.metadata).custom, name: extracted.entity.name, importedFrom: 'smart-import' }
|
||||
})
|
||||
result.entityIds.push(entityId)
|
||||
result.stats.entitiesCreated++
|
||||
extracted.entity.id = entityId
|
||||
} catch (error: any) {
|
||||
result.errors.push(`Failed to create entity ${extracted.entity.name}: ${error.message}`)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (options.createRelationships !== false && options.createEntities !== false) {
|
||||
onProgress?.({ phase: 'creating', message: 'Preparing relationships...', processed: 0, total: result.extraction.rows.length, entities: result.entityIds.length, relationships: 0 })
|
||||
|
||||
// Collect all relationship parameters
|
||||
const relationshipParams: Array<{from: string; to: string; type: VerbType; subtype?: string; confidence?: number; metadata?: any}> = []
|
||||
|
||||
for (const extracted of result.extraction.rows) {
|
||||
for (const rel of extracted.relationships) {
|
||||
try {
|
||||
let toEntityId: string | undefined
|
||||
for (const otherExtracted of result.extraction.rows) {
|
||||
if (rel.to.toLowerCase().includes(otherExtracted.entity.name.toLowerCase()) || otherExtracted.entity.name.toLowerCase().includes(rel.to.toLowerCase())) {
|
||||
toEntityId = otherExtracted.entity.id
|
||||
break
|
||||
}
|
||||
}
|
||||
if (!toEntityId) {
|
||||
// Subtype `import-placeholder` marks synthetic targets (7.30.1).
|
||||
toEntityId = await this.brain.add({ data: rel.to, type: NounType.Thing, subtype: 'import-placeholder', metadata: { name: rel.to, placeholder: true, extractedFrom: extracted.entity.name } })
|
||||
result.entityIds.push(toEntityId)
|
||||
}
|
||||
// Relationship subtype precedence: extractor → caller default → `'imported'` (7.30.1).
|
||||
// `confidence` is a reserved top-level field — dedicated relate() param, not metadata
|
||||
relationshipParams.push({ from: extracted.entity.id, to: toEntityId, type: rel.type, subtype: (rel as typeof rel & { subtype?: string }).subtype ?? options.defaultSubtype ?? 'imported', confidence: rel.confidence, metadata: { evidence: rel.evidence } })
|
||||
} catch (error: any) {
|
||||
result.errors.push(`Failed to prepare relationship: ${error.message}`)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Batch create all relationships with progress
|
||||
if (relationshipParams.length > 0) {
|
||||
onProgress?.({ phase: 'relationships', message: 'Building relationships...', processed: 0, total: relationshipParams.length, entities: result.entityIds.length, relationships: 0 })
|
||||
|
||||
try {
|
||||
const relationshipIds = await this.brain.relateMany({
|
||||
items: relationshipParams,
|
||||
parallel: true,
|
||||
chunkSize: 100,
|
||||
continueOnError: true,
|
||||
onProgress: (done, total) => {
|
||||
onProgress?.({ phase: 'relationships', message: `Building relationships: ${done}/${total}`, processed: done, total: total, entities: result.entityIds.length, relationships: done })
|
||||
}
|
||||
})
|
||||
|
||||
result.relationshipIds = relationshipIds
|
||||
result.stats.relationshipsCreated = relationshipIds.length
|
||||
} catch (error: any) {
|
||||
result.errors.push(`Failed to create relationships: ${error.message}`)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Helper: Convert PDF result to Excel-like format
|
||||
*/
|
||||
private convertPDFToExcelFormat(pdfResult: SmartPDFResult): Omit<SmartExcelResult, 'rows'> & { rows: any[] } {
|
||||
const rows = pdfResult.sections.flatMap(section =>
|
||||
section.entities.map(entity => ({
|
||||
entity,
|
||||
relatedEntities: [],
|
||||
relationships: section.relationships.filter(r => r.from === entity.id),
|
||||
concepts: section.concepts
|
||||
}))
|
||||
)
|
||||
|
||||
return {
|
||||
rowsProcessed: pdfResult.sectionsProcessed,
|
||||
entitiesExtracted: pdfResult.entitiesExtracted,
|
||||
relationshipsInferred: pdfResult.relationshipsInferred,
|
||||
rows,
|
||||
entityMap: pdfResult.entityMap,
|
||||
processingTime: pdfResult.processingTime,
|
||||
stats: pdfResult.stats
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Helper: Convert JSON result to Excel-like format
|
||||
*/
|
||||
private convertJSONToExcelFormat(jsonResult: SmartJSONResult): Omit<SmartExcelResult, 'rows'> & { rows: any[] } {
|
||||
const rows = jsonResult.entities.map(entity => ({
|
||||
entity,
|
||||
relatedEntities: [],
|
||||
relationships: jsonResult.relationships.filter(r => r.from === entity.id),
|
||||
concepts: entity.metadata.concepts || []
|
||||
}))
|
||||
|
||||
return {
|
||||
rowsProcessed: jsonResult.nodesProcessed,
|
||||
entitiesExtracted: jsonResult.entitiesExtracted,
|
||||
relationshipsInferred: jsonResult.relationshipsInferred,
|
||||
rows,
|
||||
entityMap: jsonResult.entityMap,
|
||||
processingTime: jsonResult.processingTime,
|
||||
stats: jsonResult.stats
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Helper: Convert Markdown result to Excel-like format
|
||||
*/
|
||||
private convertMarkdownToExcelFormat(mdResult: SmartMarkdownResult): Omit<SmartExcelResult, 'rows'> & { rows: any[] } {
|
||||
const rows = mdResult.sections.flatMap(section =>
|
||||
section.entities.map(entity => ({
|
||||
entity,
|
||||
relatedEntities: [],
|
||||
relationships: section.relationships.filter(r => r.from === entity.id),
|
||||
concepts: section.concepts
|
||||
}))
|
||||
)
|
||||
|
||||
return {
|
||||
rowsProcessed: mdResult.sectionsProcessed,
|
||||
entitiesExtracted: mdResult.entitiesExtracted,
|
||||
relationshipsInferred: mdResult.relationshipsInferred,
|
||||
rows,
|
||||
entityMap: mdResult.entityMap,
|
||||
processingTime: mdResult.processingTime,
|
||||
stats: mdResult.stats
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get import statistics
|
||||
*/
|
||||
async getImportStatistics(vfsRootPath: string): Promise<{
|
||||
entitiesInGraph: number
|
||||
relationshipsInGraph: number
|
||||
filesInVFS: number
|
||||
lastImport?: Date
|
||||
}> {
|
||||
// Read metadata file
|
||||
const vfs = new VirtualFileSystem(this.brain)
|
||||
await vfs.init()
|
||||
|
||||
const metadataPath = `${vfsRootPath}/_metadata.json`
|
||||
|
||||
try {
|
||||
const metadataBuffer = await vfs.readFile(metadataPath)
|
||||
const metadata = JSON.parse(metadataBuffer.toString('utf-8'))
|
||||
|
||||
return {
|
||||
entitiesInGraph: metadata.import.stats.entitiesExtracted,
|
||||
relationshipsInGraph: metadata.import.stats.relationshipsInferred,
|
||||
filesInVFS: metadata.structure.fileCount,
|
||||
lastImport: new Date(metadata.import.timestamp)
|
||||
}
|
||||
} catch (error) {
|
||||
return {
|
||||
entitiesInGraph: 0,
|
||||
relationshipsInGraph: 0,
|
||||
filesInVFS: 0
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -1,69 +0,0 @@
|
|||
/**
|
||||
* Smart Import System
|
||||
*
|
||||
* Production-ready entity and relationship extraction from multiple formats:
|
||||
* - Excel (.xlsx)
|
||||
* - PDF (.pdf)
|
||||
* - CSV (.csv)
|
||||
* - JSON (.json)
|
||||
* - Markdown (.md)
|
||||
*
|
||||
* Uses brainy's built-in NeuralEntityExtractor and NaturalLanguageProcessor
|
||||
*
|
||||
* NO MOCKS - Real working implementation
|
||||
*/
|
||||
|
||||
// Excel Importer
|
||||
export { SmartExcelImporter } from './SmartExcelImporter.js'
|
||||
export type {
|
||||
SmartExcelOptions,
|
||||
ExtractedRow,
|
||||
SmartExcelResult
|
||||
} from './SmartExcelImporter.js'
|
||||
|
||||
// PDF Importer
|
||||
export { SmartPDFImporter } from './SmartPDFImporter.js'
|
||||
export type {
|
||||
SmartPDFOptions,
|
||||
ExtractedSection,
|
||||
SmartPDFResult
|
||||
} from './SmartPDFImporter.js'
|
||||
|
||||
// CSV Importer
|
||||
export { SmartCSVImporter } from './SmartCSVImporter.js'
|
||||
export type {
|
||||
SmartCSVOptions,
|
||||
SmartCSVResult
|
||||
} from './SmartCSVImporter.js'
|
||||
|
||||
// JSON Importer
|
||||
export { SmartJSONImporter } from './SmartJSONImporter.js'
|
||||
export type {
|
||||
SmartJSONOptions,
|
||||
ExtractedJSONEntity,
|
||||
ExtractedJSONRelationship,
|
||||
SmartJSONResult
|
||||
} from './SmartJSONImporter.js'
|
||||
|
||||
// Markdown Importer
|
||||
export { SmartMarkdownImporter } from './SmartMarkdownImporter.js'
|
||||
export type {
|
||||
SmartMarkdownOptions,
|
||||
MarkdownSection,
|
||||
SmartMarkdownResult
|
||||
} from './SmartMarkdownImporter.js'
|
||||
|
||||
// VFS Structure Generator
|
||||
export { VFSStructureGenerator } from './VFSStructureGenerator.js'
|
||||
export type {
|
||||
VFSStructureOptions,
|
||||
VFSStructureResult
|
||||
} from './VFSStructureGenerator.js'
|
||||
|
||||
// Orchestrator (Main entry point)
|
||||
export { SmartImportOrchestrator } from './SmartImportOrchestrator.js'
|
||||
export type {
|
||||
SmartImportOptions,
|
||||
SmartImportProgress,
|
||||
SmartImportResult
|
||||
} from './SmartImportOrchestrator.js'
|
||||
Loading…
Add table
Add a link
Reference in a new issue