David Snelling
54c7c39669
An untyped (JS) caller that smuggles a Brainy-reserved field (confidence, weight, subtype, visibility, service, createdBy, noun/verb, data, createdAt, updatedAt, _rev) inside a write-path metadata bag previously got a silent remap-or-drop — a class of bug where confidence-evolution writes no-oped for weeks before being caught on read-back. 8.0 closes this with no silent failures. - New BrainyConfig.reservedFieldPolicy: 'throw' | 'warn' | 'remap' (default 'throw'). 'throw' rejects the write naming every offending key + its correct write path; 'warn' remaps with a one-shot per-key warning; 'remap' is the legacy silent path. - Central enforceReservedPolicy gate wired into all four remap methods (add, update, relate, updateRelation) so live calls AND their transact()/with() mirrors honor it. Single-source reservedWritePath guidance shared by throw+warn. - 'warn' now warns for EVERY reserved key (closes the gap where only system-managed fields warned). Dead warnDropped* helpers removed. - Import pipeline migrated to route reserved values (confidence/weight/subtype) through dedicated params and strip reserved keys from extractor/customMetadata bags via the canonical split*MetadataRecord helpers — imports no longer trip the default throw. - Tests: new reservedFieldPolicy matrix (throw/warn/remap across every write path + transact); remap-correctness suite reframed as opt-in 'remap'; shared test-factory no longer emits reserved keys in custom metadata.
789 lines
29 KiB
TypeScript
789 lines
29 KiB
TypeScript
/**
|
|
* Smart Import Orchestrator
|
|
*
|
|
* Coordinates the entire smart import pipeline:
|
|
* 1. Extract entities/relationships using SmartExcelImporter
|
|
* 2. Create entities and relationships in Brainy
|
|
* 3. Organize into VFS structure using VFSStructureGenerator
|
|
*
|
|
* NO MOCKS - Production-ready implementation
|
|
*/
|
|
|
|
import { Brainy } from '../brainy.js'
|
|
import { VirtualFileSystem } from '../vfs/VirtualFileSystem.js'
|
|
import { NounType, VerbType } from '../types/graphTypes.js'
|
|
import { splitNounMetadataRecord } from '../types/reservedFields.js'
|
|
import { SmartExcelImporter, SmartExcelOptions, SmartExcelResult } from './SmartExcelImporter.js'
|
|
import { SmartPDFImporter, SmartPDFOptions, SmartPDFResult } from './SmartPDFImporter.js'
|
|
import { SmartCSVImporter, SmartCSVOptions, SmartCSVResult } from './SmartCSVImporter.js'
|
|
import { SmartJSONImporter, SmartJSONOptions, SmartJSONResult } from './SmartJSONImporter.js'
|
|
import { SmartMarkdownImporter, SmartMarkdownOptions, SmartMarkdownResult } from './SmartMarkdownImporter.js'
|
|
import { VFSStructureGenerator, VFSStructureOptions } from './VFSStructureGenerator.js'
|
|
|
|
export interface SmartImportOptions extends SmartExcelOptions {
|
|
/** Create VFS structure */
|
|
createVFSStructure?: boolean
|
|
|
|
/** VFS root path */
|
|
vfsRootPath?: string
|
|
|
|
/** VFS grouping strategy */
|
|
vfsGroupBy?: 'type' | 'sheet' | 'flat' | 'custom'
|
|
|
|
/** Create entities in Brainy */
|
|
createEntities?: boolean
|
|
|
|
/** Create relationships in Brainy */
|
|
createRelationships?: boolean
|
|
|
|
/** Source filename */
|
|
filename?: string
|
|
|
|
/**
|
|
* Default subtype tag for entities + relationships this importer creates when
|
|
* the extractor doesn't set one. See `ValidImportOptions.defaultSubtype` —
|
|
* same semantics, same precedence (extractor > caller default > `'imported'`).
|
|
* Added 7.30.1.
|
|
*/
|
|
defaultSubtype?: string
|
|
}
|
|
|
|
export interface SmartImportProgress {
|
|
phase: 'parsing' | 'extracting' | 'creating' | 'relationships' | 'organizing' | 'complete'
|
|
message: string
|
|
processed: number
|
|
total: number
|
|
entities: number
|
|
relationships: number
|
|
}
|
|
|
|
export interface SmartImportResult {
|
|
success: boolean
|
|
|
|
/** Extraction results */
|
|
extraction: SmartExcelResult
|
|
|
|
/** Created entity IDs */
|
|
entityIds: string[]
|
|
|
|
/** Created relationship IDs */
|
|
relationshipIds: string[]
|
|
|
|
/** VFS structure created */
|
|
vfsStructure?: {
|
|
rootPath: string
|
|
directories: string[]
|
|
files: number
|
|
}
|
|
|
|
/** Overall statistics */
|
|
stats: {
|
|
rowsProcessed: number
|
|
entitiesCreated: number
|
|
relationshipsCreated: number
|
|
filesCreated: number
|
|
totalTime: number
|
|
}
|
|
|
|
/** Any errors encountered */
|
|
errors: string[]
|
|
}
|
|
|
|
/**
|
|
* SmartImportOrchestrator - Main entry point for smart imports
|
|
*/
|
|
export class SmartImportOrchestrator {
|
|
private brain: Brainy
|
|
private excelImporter: SmartExcelImporter
|
|
private pdfImporter: SmartPDFImporter
|
|
private csvImporter: SmartCSVImporter
|
|
private jsonImporter: SmartJSONImporter
|
|
private markdownImporter: SmartMarkdownImporter
|
|
private vfsGenerator: VFSStructureGenerator
|
|
|
|
constructor(brain: Brainy) {
|
|
this.brain = brain
|
|
this.excelImporter = new SmartExcelImporter(brain)
|
|
this.pdfImporter = new SmartPDFImporter(brain)
|
|
this.csvImporter = new SmartCSVImporter(brain)
|
|
this.jsonImporter = new SmartJSONImporter(brain)
|
|
this.markdownImporter = new SmartMarkdownImporter(brain)
|
|
this.vfsGenerator = new VFSStructureGenerator(brain)
|
|
}
|
|
|
|
/**
|
|
* Initialize the orchestrator
|
|
*/
|
|
async init(): Promise<void> {
|
|
await this.excelImporter.init()
|
|
await this.pdfImporter.init()
|
|
await this.csvImporter.init()
|
|
await this.jsonImporter.init()
|
|
await this.markdownImporter.init()
|
|
await this.vfsGenerator.init()
|
|
}
|
|
|
|
/**
|
|
* Import Excel file with full pipeline
|
|
*/
|
|
async importExcel(
|
|
buffer: Buffer,
|
|
options: SmartImportOptions = {},
|
|
onProgress?: (progress: SmartImportProgress) => void
|
|
): Promise<SmartImportResult> {
|
|
const startTime = Date.now()
|
|
const result: SmartImportResult = {
|
|
success: false,
|
|
// Typed boundary: populated in the extraction phase below. If extraction
|
|
// throws, the error path returns with this still null (pre-existing
|
|
// contract — consumers check `success`/`errors` before reading it).
|
|
extraction: null as unknown as SmartExcelResult,
|
|
entityIds: [],
|
|
relationshipIds: [],
|
|
stats: {
|
|
rowsProcessed: 0,
|
|
entitiesCreated: 0,
|
|
relationshipsCreated: 0,
|
|
filesCreated: 0,
|
|
totalTime: 0
|
|
},
|
|
errors: []
|
|
}
|
|
|
|
try {
|
|
// Phase 1: Extract entities and relationships
|
|
onProgress?.({
|
|
phase: 'extracting',
|
|
message: 'Extracting entities and relationships...',
|
|
processed: 0,
|
|
total: 0,
|
|
entities: 0,
|
|
relationships: 0
|
|
})
|
|
|
|
result.extraction = await this.excelImporter.extract(buffer, {
|
|
...options,
|
|
onProgress: (stats) => {
|
|
onProgress?.({
|
|
phase: 'extracting',
|
|
message: `Processing row ${stats.processed}/${stats.total}...`,
|
|
processed: stats.processed,
|
|
total: stats.total,
|
|
entities: stats.entities,
|
|
relationships: stats.relationships
|
|
})
|
|
}
|
|
})
|
|
|
|
result.stats.rowsProcessed = result.extraction.rowsProcessed
|
|
|
|
// Phase 2: Create entities in Brainy
|
|
if (options.createEntities !== false) {
|
|
onProgress?.({
|
|
phase: 'creating',
|
|
message: 'Creating entities in knowledge graph...',
|
|
processed: 0,
|
|
total: result.extraction.rows.length,
|
|
entities: 0,
|
|
relationships: 0
|
|
})
|
|
|
|
for (let i = 0; i < result.extraction.rows.length; i++) {
|
|
const extracted = result.extraction.rows[i]
|
|
|
|
try {
|
|
// Create main entity. Subtype precedence: extractor-set → caller default
|
|
// → Brainy default `'imported'` (added 7.30.1).
|
|
const entityId = await this.brain.add({
|
|
data: extracted.entity.description,
|
|
type: extracted.entity.type,
|
|
subtype:
|
|
(extracted.entity as typeof extracted.entity & { subtype?: string })
|
|
.subtype ?? options.defaultSubtype ?? 'imported',
|
|
confidence: extracted.entity.confidence, // reserved field — dedicated param, not metadata
|
|
metadata: {
|
|
// Strip reserved keys an extractor may have smuggled into the bag
|
|
// (8.0 reservedFieldPolicy defaults to 'throw').
|
|
...splitNounMetadataRecord(extracted.entity.metadata).custom,
|
|
name: extracted.entity.name,
|
|
importedFrom: 'smart-import'
|
|
}
|
|
})
|
|
|
|
result.entityIds.push(entityId)
|
|
result.stats.entitiesCreated++
|
|
|
|
// Update entity ID in extraction result
|
|
extracted.entity.id = entityId
|
|
|
|
onProgress?.({
|
|
phase: 'creating',
|
|
message: `Created entity: ${extracted.entity.name}`,
|
|
processed: i + 1,
|
|
total: result.extraction.rows.length,
|
|
entities: result.entityIds.length,
|
|
relationships: result.relationshipIds.length
|
|
})
|
|
} catch (error: any) {
|
|
result.errors.push(`Failed to create entity ${extracted.entity.name}: ${error.message}`)
|
|
}
|
|
}
|
|
}
|
|
|
|
// Phase 3: Create relationships
|
|
if (options.createRelationships !== false && options.createEntities !== false) {
|
|
onProgress?.({
|
|
phase: 'creating',
|
|
message: 'Preparing relationships...',
|
|
processed: 0,
|
|
total: result.extraction.rows.length,
|
|
entities: result.entityIds.length,
|
|
relationships: 0
|
|
})
|
|
|
|
// Build entity name -> ID map
|
|
const entityMap = new Map<string, string>()
|
|
for (const extracted of result.extraction.rows) {
|
|
entityMap.set(extracted.entity.name.toLowerCase(), extracted.entity.id)
|
|
}
|
|
|
|
// Collect all relationship parameters
|
|
const relationshipParams: Array<{from: string; to: string; type: VerbType; subtype?: string; confidence?: number; metadata?: any}> = []
|
|
|
|
for (const extracted of result.extraction.rows) {
|
|
for (const rel of extracted.relationships) {
|
|
try {
|
|
// Find target entity ID
|
|
let toEntityId: string | undefined
|
|
|
|
// Try to find by name in our extracted entities
|
|
for (const otherExtracted of result.extraction.rows) {
|
|
if (rel.to.toLowerCase().includes(otherExtracted.entity.name.toLowerCase()) ||
|
|
otherExtracted.entity.name.toLowerCase().includes(rel.to.toLowerCase())) {
|
|
toEntityId = otherExtracted.entity.id
|
|
break
|
|
}
|
|
}
|
|
|
|
// If not found, create a placeholder entity. `import-placeholder` marks
|
|
// these as synthetic targets so consumers can distinguish them from real
|
|
// imports and downstream dedup can consolidate (added 7.30.1).
|
|
if (!toEntityId) {
|
|
toEntityId = await this.brain.add({
|
|
data: rel.to,
|
|
type: NounType.Thing,
|
|
subtype: 'import-placeholder',
|
|
metadata: {
|
|
name: rel.to,
|
|
placeholder: true,
|
|
extractedFrom: extracted.entity.name
|
|
}
|
|
})
|
|
result.entityIds.push(toEntityId)
|
|
}
|
|
|
|
// Collect relationship parameter. Subtype precedence: extractor-set rel
|
|
// subtype → caller default → Brainy default `'imported'` (added 7.30.1).
|
|
relationshipParams.push({
|
|
from: extracted.entity.id,
|
|
to: toEntityId,
|
|
type: rel.type,
|
|
subtype:
|
|
(rel as typeof rel & { subtype?: string }).subtype ??
|
|
options.defaultSubtype ?? 'imported',
|
|
confidence: rel.confidence, // reserved field — dedicated param, not metadata
|
|
metadata: {
|
|
evidence: rel.evidence
|
|
}
|
|
})
|
|
} catch (error: any) {
|
|
result.errors.push(`Failed to prepare relationship: ${error.message}`)
|
|
}
|
|
}
|
|
}
|
|
|
|
// Batch create all relationships with progress
|
|
if (relationshipParams.length > 0) {
|
|
onProgress?.({
|
|
phase: 'relationships',
|
|
message: 'Building relationships...',
|
|
processed: 0,
|
|
total: relationshipParams.length,
|
|
entities: result.entityIds.length,
|
|
relationships: 0
|
|
})
|
|
|
|
try {
|
|
const relationshipIds = await this.brain.relateMany({
|
|
items: relationshipParams,
|
|
parallel: true,
|
|
chunkSize: 100,
|
|
continueOnError: true,
|
|
onProgress: (done, total) => {
|
|
onProgress?.({
|
|
phase: 'relationships',
|
|
message: `Building relationships: ${done}/${total}`,
|
|
processed: done,
|
|
total: total,
|
|
entities: result.entityIds.length,
|
|
relationships: done
|
|
})
|
|
}
|
|
})
|
|
|
|
result.relationshipIds = relationshipIds
|
|
result.stats.relationshipsCreated = relationshipIds.length
|
|
} catch (error: any) {
|
|
result.errors.push(`Failed to create relationships: ${error.message}`)
|
|
}
|
|
}
|
|
}
|
|
|
|
// Phase 4: Create VFS structure
|
|
if (options.createVFSStructure !== false) {
|
|
onProgress?.({
|
|
phase: 'organizing',
|
|
message: 'Organizing into file structure...',
|
|
processed: 0,
|
|
total: result.extraction.rows.length,
|
|
entities: result.entityIds.length,
|
|
relationships: result.relationshipIds.length
|
|
})
|
|
|
|
const vfsOptions: VFSStructureOptions = {
|
|
rootPath: options.vfsRootPath || '/imports/' + (options.filename || 'import'),
|
|
groupBy: options.vfsGroupBy || 'type',
|
|
preserveSource: true,
|
|
sourceBuffer: buffer,
|
|
sourceFilename: options.filename || 'import.xlsx',
|
|
createRelationshipFile: true,
|
|
createMetadataFile: true
|
|
}
|
|
|
|
const vfsResult = await this.vfsGenerator.generate(result.extraction, vfsOptions)
|
|
|
|
result.vfsStructure = {
|
|
rootPath: vfsResult.rootPath,
|
|
directories: vfsResult.directories,
|
|
files: vfsResult.files.length
|
|
}
|
|
|
|
result.stats.filesCreated = vfsResult.files.length
|
|
}
|
|
|
|
// Complete
|
|
result.success = result.errors.length === 0
|
|
result.stats.totalTime = Date.now() - startTime
|
|
|
|
onProgress?.({
|
|
phase: 'complete',
|
|
message: `Import complete: ${result.stats.entitiesCreated} entities, ${result.stats.relationshipsCreated} relationships`,
|
|
processed: result.extraction.rows.length,
|
|
total: result.extraction.rows.length,
|
|
entities: result.stats.entitiesCreated,
|
|
relationships: result.stats.relationshipsCreated
|
|
})
|
|
|
|
} catch (error: any) {
|
|
result.errors.push(`Import failed: ${error.message}`)
|
|
result.success = false
|
|
}
|
|
|
|
return result
|
|
}
|
|
|
|
/**
|
|
* Import PDF file with full pipeline
|
|
*/
|
|
async importPDF(
|
|
buffer: Buffer,
|
|
options: SmartImportOptions & SmartPDFOptions = {},
|
|
onProgress?: (progress: SmartImportProgress) => void
|
|
): Promise<SmartImportResult> {
|
|
const startTime = Date.now()
|
|
const result: SmartImportResult = {
|
|
success: false,
|
|
// Typed boundary: populated after extraction (see importExcel).
|
|
extraction: null as unknown as SmartExcelResult,
|
|
entityIds: [],
|
|
relationshipIds: [],
|
|
stats: {
|
|
rowsProcessed: 0,
|
|
entitiesCreated: 0,
|
|
relationshipsCreated: 0,
|
|
filesCreated: 0,
|
|
totalTime: 0
|
|
},
|
|
errors: []
|
|
}
|
|
|
|
try {
|
|
// Phase 1: Extract from PDF
|
|
onProgress?.({ phase: 'extracting', message: 'Extracting from PDF...', processed: 0, total: 0, entities: 0, relationships: 0 })
|
|
|
|
const pdfResult = await this.pdfImporter.extract(buffer, options)
|
|
|
|
// Convert PDF result to Excel-like format for processing
|
|
result.extraction = this.convertPDFToExcelFormat(pdfResult)
|
|
result.stats.rowsProcessed = pdfResult.sectionsProcessed
|
|
|
|
// Phase 2 & 3: Create entities and relationships
|
|
await this.createEntitiesAndRelationships(result, options, onProgress)
|
|
|
|
// Phase 4: Create VFS structure
|
|
if (options.createVFSStructure !== false) {
|
|
const vfsOptions: VFSStructureOptions = {
|
|
rootPath: options.vfsRootPath || '/imports/' + (options.filename || 'import'),
|
|
groupBy: options.vfsGroupBy || 'type',
|
|
preserveSource: true,
|
|
sourceBuffer: buffer,
|
|
sourceFilename: options.filename || 'import.pdf',
|
|
createRelationshipFile: true,
|
|
createMetadataFile: true
|
|
}
|
|
const vfsResult = await this.vfsGenerator.generate(result.extraction, vfsOptions)
|
|
result.vfsStructure = { rootPath: vfsResult.rootPath, directories: vfsResult.directories, files: vfsResult.files.length }
|
|
result.stats.filesCreated = vfsResult.files.length
|
|
}
|
|
|
|
result.success = result.errors.length === 0
|
|
result.stats.totalTime = Date.now() - startTime
|
|
onProgress?.({ phase: 'complete', message: `Import complete: ${result.stats.entitiesCreated} entities, ${result.stats.relationshipsCreated} relationships`, processed: result.stats.rowsProcessed, total: result.stats.rowsProcessed, entities: result.stats.entitiesCreated, relationships: result.stats.relationshipsCreated })
|
|
|
|
} catch (error: any) {
|
|
result.errors.push(`PDF import failed: ${error.message}`)
|
|
result.success = false
|
|
}
|
|
|
|
return result
|
|
}
|
|
|
|
/**
|
|
* Import CSV file with full pipeline
|
|
*/
|
|
async importCSV(
|
|
buffer: Buffer,
|
|
options: SmartImportOptions & SmartCSVOptions = {},
|
|
onProgress?: (progress: SmartImportProgress) => void
|
|
): Promise<SmartImportResult> {
|
|
// CSV is very similar to Excel, can reuse importExcel logic
|
|
return this.importExcel(buffer, options, onProgress)
|
|
}
|
|
|
|
/**
|
|
* Import JSON data with full pipeline
|
|
*/
|
|
async importJSON(
|
|
data: any,
|
|
options: SmartImportOptions & SmartJSONOptions = {},
|
|
onProgress?: (progress: SmartImportProgress) => void
|
|
): Promise<SmartImportResult> {
|
|
const startTime = Date.now()
|
|
const result: SmartImportResult = {
|
|
success: false,
|
|
// Typed boundary: populated after extraction (see importExcel).
|
|
extraction: null as unknown as SmartExcelResult,
|
|
entityIds: [],
|
|
relationshipIds: [],
|
|
stats: {
|
|
rowsProcessed: 0,
|
|
entitiesCreated: 0,
|
|
relationshipsCreated: 0,
|
|
filesCreated: 0,
|
|
totalTime: 0
|
|
},
|
|
errors: []
|
|
}
|
|
|
|
try {
|
|
onProgress?.({ phase: 'extracting', message: 'Extracting from JSON...', processed: 0, total: 0, entities: 0, relationships: 0 })
|
|
|
|
const jsonResult = await this.jsonImporter.extract(data, options)
|
|
|
|
result.extraction = this.convertJSONToExcelFormat(jsonResult)
|
|
result.stats.rowsProcessed = jsonResult.nodesProcessed
|
|
|
|
await this.createEntitiesAndRelationships(result, options, onProgress)
|
|
|
|
if (options.createVFSStructure !== false) {
|
|
const sourceBuffer = Buffer.from(typeof data === 'string' ? data : JSON.stringify(data, null, 2))
|
|
const vfsOptions: VFSStructureOptions = {
|
|
rootPath: options.vfsRootPath || '/imports/' + (options.filename || 'import'),
|
|
groupBy: options.vfsGroupBy || 'type',
|
|
preserveSource: true,
|
|
sourceBuffer,
|
|
sourceFilename: options.filename || 'import.json',
|
|
createRelationshipFile: true,
|
|
createMetadataFile: true
|
|
}
|
|
const vfsResult = await this.vfsGenerator.generate(result.extraction, vfsOptions)
|
|
result.vfsStructure = { rootPath: vfsResult.rootPath, directories: vfsResult.directories, files: vfsResult.files.length }
|
|
result.stats.filesCreated = vfsResult.files.length
|
|
}
|
|
|
|
result.success = result.errors.length === 0
|
|
result.stats.totalTime = Date.now() - startTime
|
|
onProgress?.({ phase: 'complete', message: `Import complete: ${result.stats.entitiesCreated} entities, ${result.stats.relationshipsCreated} relationships`, processed: result.stats.rowsProcessed, total: result.stats.rowsProcessed, entities: result.stats.entitiesCreated, relationships: result.stats.relationshipsCreated })
|
|
|
|
} catch (error: any) {
|
|
result.errors.push(`JSON import failed: ${error.message}`)
|
|
result.success = false
|
|
}
|
|
|
|
return result
|
|
}
|
|
|
|
/**
|
|
* Import Markdown content with full pipeline
|
|
*/
|
|
async importMarkdown(
|
|
markdown: string,
|
|
options: SmartImportOptions & SmartMarkdownOptions = {},
|
|
onProgress?: (progress: SmartImportProgress) => void
|
|
): Promise<SmartImportResult> {
|
|
const startTime = Date.now()
|
|
const result: SmartImportResult = {
|
|
success: false,
|
|
// Typed boundary: populated after extraction (see importExcel).
|
|
extraction: null as unknown as SmartExcelResult,
|
|
entityIds: [],
|
|
relationshipIds: [],
|
|
stats: {
|
|
rowsProcessed: 0,
|
|
entitiesCreated: 0,
|
|
relationshipsCreated: 0,
|
|
filesCreated: 0,
|
|
totalTime: 0
|
|
},
|
|
errors: []
|
|
}
|
|
|
|
try {
|
|
onProgress?.({ phase: 'extracting', message: 'Extracting from Markdown...', processed: 0, total: 0, entities: 0, relationships: 0 })
|
|
|
|
const mdResult = await this.markdownImporter.extract(markdown, options)
|
|
|
|
result.extraction = this.convertMarkdownToExcelFormat(mdResult)
|
|
result.stats.rowsProcessed = mdResult.sectionsProcessed
|
|
|
|
await this.createEntitiesAndRelationships(result, options, onProgress)
|
|
|
|
if (options.createVFSStructure !== false) {
|
|
const sourceBuffer = Buffer.from(markdown, 'utf-8')
|
|
const vfsOptions: VFSStructureOptions = {
|
|
rootPath: options.vfsRootPath || '/imports/' + (options.filename || 'import'),
|
|
groupBy: options.vfsGroupBy || 'type',
|
|
preserveSource: true,
|
|
sourceBuffer,
|
|
sourceFilename: options.filename || 'import.md',
|
|
createRelationshipFile: true,
|
|
createMetadataFile: true
|
|
}
|
|
const vfsResult = await this.vfsGenerator.generate(result.extraction, vfsOptions)
|
|
result.vfsStructure = { rootPath: vfsResult.rootPath, directories: vfsResult.directories, files: vfsResult.files.length }
|
|
result.stats.filesCreated = vfsResult.files.length
|
|
}
|
|
|
|
result.success = result.errors.length === 0
|
|
result.stats.totalTime = Date.now() - startTime
|
|
onProgress?.({ phase: 'complete', message: `Import complete: ${result.stats.entitiesCreated} entities, ${result.stats.relationshipsCreated} relationships`, processed: result.stats.rowsProcessed, total: result.stats.rowsProcessed, entities: result.stats.entitiesCreated, relationships: result.stats.relationshipsCreated })
|
|
|
|
} catch (error: any) {
|
|
result.errors.push(`Markdown import failed: ${error.message}`)
|
|
result.success = false
|
|
}
|
|
|
|
return result
|
|
}
|
|
|
|
/**
|
|
* Helper: Create entities and relationships from extraction result
|
|
*/
|
|
private async createEntitiesAndRelationships(
|
|
result: SmartImportResult,
|
|
options: SmartImportOptions,
|
|
onProgress?: (progress: SmartImportProgress) => void
|
|
): Promise<void> {
|
|
if (options.createEntities !== false) {
|
|
onProgress?.({ phase: 'creating', message: 'Creating entities in knowledge graph...', processed: 0, total: result.extraction.rows.length, entities: 0, relationships: 0 })
|
|
|
|
for (let i = 0; i < result.extraction.rows.length; i++) {
|
|
const extracted = result.extraction.rows[i]
|
|
try {
|
|
// Subtype precedence: extractor → caller default → `'imported'` (7.30.1).
|
|
const entityId = await this.brain.add({
|
|
data: extracted.entity.description,
|
|
type: extracted.entity.type,
|
|
subtype:
|
|
(extracted.entity as typeof extracted.entity & { subtype?: string })
|
|
.subtype ?? options.defaultSubtype ?? 'imported',
|
|
confidence: extracted.entity.confidence, // reserved field — dedicated param, not metadata
|
|
metadata: { ...splitNounMetadataRecord(extracted.entity.metadata).custom, name: extracted.entity.name, importedFrom: 'smart-import' }
|
|
})
|
|
result.entityIds.push(entityId)
|
|
result.stats.entitiesCreated++
|
|
extracted.entity.id = entityId
|
|
} catch (error: any) {
|
|
result.errors.push(`Failed to create entity ${extracted.entity.name}: ${error.message}`)
|
|
}
|
|
}
|
|
}
|
|
|
|
if (options.createRelationships !== false && options.createEntities !== false) {
|
|
onProgress?.({ phase: 'creating', message: 'Preparing relationships...', processed: 0, total: result.extraction.rows.length, entities: result.entityIds.length, relationships: 0 })
|
|
|
|
// Collect all relationship parameters
|
|
const relationshipParams: Array<{from: string; to: string; type: VerbType; subtype?: string; confidence?: number; metadata?: any}> = []
|
|
|
|
for (const extracted of result.extraction.rows) {
|
|
for (const rel of extracted.relationships) {
|
|
try {
|
|
let toEntityId: string | undefined
|
|
for (const otherExtracted of result.extraction.rows) {
|
|
if (rel.to.toLowerCase().includes(otherExtracted.entity.name.toLowerCase()) || otherExtracted.entity.name.toLowerCase().includes(rel.to.toLowerCase())) {
|
|
toEntityId = otherExtracted.entity.id
|
|
break
|
|
}
|
|
}
|
|
if (!toEntityId) {
|
|
// Subtype `import-placeholder` marks synthetic targets (7.30.1).
|
|
toEntityId = await this.brain.add({ data: rel.to, type: NounType.Thing, subtype: 'import-placeholder', metadata: { name: rel.to, placeholder: true, extractedFrom: extracted.entity.name } })
|
|
result.entityIds.push(toEntityId)
|
|
}
|
|
// Relationship subtype precedence: extractor → caller default → `'imported'` (7.30.1).
|
|
// `confidence` is a reserved top-level field — dedicated relate() param, not metadata
|
|
relationshipParams.push({ from: extracted.entity.id, to: toEntityId, type: rel.type, subtype: (rel as typeof rel & { subtype?: string }).subtype ?? options.defaultSubtype ?? 'imported', confidence: rel.confidence, metadata: { evidence: rel.evidence } })
|
|
} catch (error: any) {
|
|
result.errors.push(`Failed to prepare relationship: ${error.message}`)
|
|
}
|
|
}
|
|
}
|
|
|
|
// Batch create all relationships with progress
|
|
if (relationshipParams.length > 0) {
|
|
onProgress?.({ phase: 'relationships', message: 'Building relationships...', processed: 0, total: relationshipParams.length, entities: result.entityIds.length, relationships: 0 })
|
|
|
|
try {
|
|
const relationshipIds = await this.brain.relateMany({
|
|
items: relationshipParams,
|
|
parallel: true,
|
|
chunkSize: 100,
|
|
continueOnError: true,
|
|
onProgress: (done, total) => {
|
|
onProgress?.({ phase: 'relationships', message: `Building relationships: ${done}/${total}`, processed: done, total: total, entities: result.entityIds.length, relationships: done })
|
|
}
|
|
})
|
|
|
|
result.relationshipIds = relationshipIds
|
|
result.stats.relationshipsCreated = relationshipIds.length
|
|
} catch (error: any) {
|
|
result.errors.push(`Failed to create relationships: ${error.message}`)
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Helper: Convert PDF result to Excel-like format
|
|
*/
|
|
private convertPDFToExcelFormat(pdfResult: SmartPDFResult): Omit<SmartExcelResult, 'rows'> & { rows: any[] } {
|
|
const rows = pdfResult.sections.flatMap(section =>
|
|
section.entities.map(entity => ({
|
|
entity,
|
|
relatedEntities: [],
|
|
relationships: section.relationships.filter(r => r.from === entity.id),
|
|
concepts: section.concepts
|
|
}))
|
|
)
|
|
|
|
return {
|
|
rowsProcessed: pdfResult.sectionsProcessed,
|
|
entitiesExtracted: pdfResult.entitiesExtracted,
|
|
relationshipsInferred: pdfResult.relationshipsInferred,
|
|
rows,
|
|
entityMap: pdfResult.entityMap,
|
|
processingTime: pdfResult.processingTime,
|
|
stats: pdfResult.stats
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Helper: Convert JSON result to Excel-like format
|
|
*/
|
|
private convertJSONToExcelFormat(jsonResult: SmartJSONResult): Omit<SmartExcelResult, 'rows'> & { rows: any[] } {
|
|
const rows = jsonResult.entities.map(entity => ({
|
|
entity,
|
|
relatedEntities: [],
|
|
relationships: jsonResult.relationships.filter(r => r.from === entity.id),
|
|
concepts: entity.metadata.concepts || []
|
|
}))
|
|
|
|
return {
|
|
rowsProcessed: jsonResult.nodesProcessed,
|
|
entitiesExtracted: jsonResult.entitiesExtracted,
|
|
relationshipsInferred: jsonResult.relationshipsInferred,
|
|
rows,
|
|
entityMap: jsonResult.entityMap,
|
|
processingTime: jsonResult.processingTime,
|
|
stats: jsonResult.stats
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Helper: Convert Markdown result to Excel-like format
|
|
*/
|
|
private convertMarkdownToExcelFormat(mdResult: SmartMarkdownResult): Omit<SmartExcelResult, 'rows'> & { rows: any[] } {
|
|
const rows = mdResult.sections.flatMap(section =>
|
|
section.entities.map(entity => ({
|
|
entity,
|
|
relatedEntities: [],
|
|
relationships: section.relationships.filter(r => r.from === entity.id),
|
|
concepts: section.concepts
|
|
}))
|
|
)
|
|
|
|
return {
|
|
rowsProcessed: mdResult.sectionsProcessed,
|
|
entitiesExtracted: mdResult.entitiesExtracted,
|
|
relationshipsInferred: mdResult.relationshipsInferred,
|
|
rows,
|
|
entityMap: mdResult.entityMap,
|
|
processingTime: mdResult.processingTime,
|
|
stats: mdResult.stats
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Get import statistics
|
|
*/
|
|
async getImportStatistics(vfsRootPath: string): Promise<{
|
|
entitiesInGraph: number
|
|
relationshipsInGraph: number
|
|
filesInVFS: number
|
|
lastImport?: Date
|
|
}> {
|
|
// Read metadata file
|
|
const vfs = new VirtualFileSystem(this.brain)
|
|
await vfs.init()
|
|
|
|
const metadataPath = `${vfsRootPath}/_metadata.json`
|
|
|
|
try {
|
|
const metadataBuffer = await vfs.readFile(metadataPath)
|
|
const metadata = JSON.parse(metadataBuffer.toString('utf-8'))
|
|
|
|
return {
|
|
entitiesInGraph: metadata.import.stats.entitiesExtracted,
|
|
relationshipsInGraph: metadata.import.stats.relationshipsInferred,
|
|
filesInVFS: metadata.structure.fileCount,
|
|
lastImport: new Date(metadata.import.timestamp)
|
|
}
|
|
} catch (error) {
|
|
return {
|
|
entitiesInGraph: 0,
|
|
relationshipsInGraph: 0,
|
|
filesInVFS: 0
|
|
}
|
|
}
|
|
}
|
|
}
|