/** * VFS Structure Generator * * Organizes imported entities into structured VFS directories * - Type-based grouping (Place/, Character/, Concept/) * - Metadata files (_metadata.json, _relationships.json) * - Source file preservation * * NO MOCKS - Production-ready implementation */ import { Brainy } from '../brainy.js' import { VirtualFileSystem } from '../vfs/VirtualFileSystem.js' import { NounType, VerbType } from '../types/graphTypes.js' import type { SmartExcelResult } from './SmartExcelImporter.js' import type { TrackingContext } from '../import/ImportCoordinator.js' export interface VFSStructureOptions { /** Root path in VFS for import */ rootPath: string /** Grouping strategy */ groupBy: 'type' | 'sheet' | 'flat' | 'custom' /** Custom grouping function */ customGrouping?: (entity: any) => string /** Preserve source file */ preserveSource?: boolean /** Source file buffer (if preserving) */ sourceBuffer?: Buffer /** Source filename */ sourceFilename?: string /** Create relationship file */ createRelationshipFile?: boolean /** Create metadata file */ createMetadataFile?: boolean /** Import tracking context */ trackingContext?: TrackingContext /** Progress callback - Reports VFS creation progress */ onProgress?: (progress: { stage: 'directories' | 'entities' | 'metadata' message: string processed: number total: number }) => void } export interface VFSStructureResult { /** Root path created */ rootPath: string /** Directories created */ directories: string[] /** Files created */ files: Array<{ path: string entityId?: string type: 'entity' | 'metadata' | 'source' | 'relationships' }> /** Total operations */ operations: number /** Time taken in ms */ duration: number } /** * VFSStructureGenerator - Organizes imported data into VFS */ export class VFSStructureGenerator { private brain: Brainy private vfs!: VirtualFileSystem // Non-null assertion - will be set in init() constructor(brain: Brainy) { this.brain = brain // CRITICAL FIX: Use brain.vfs instead of creating separate instance // This ensures VFSStructureGenerator and user code share the same VFS instance // Before: Created separate instance that wasn't accessible to users // After: Uses brain's cached instance, making VFS queryable after import } /** * Initialize the generator * * CRITICAL: Gets brain's VFS instance and initializes it if needed. * This ensures that after import, brain.vfs returns an initialized instance. */ async init(): Promise { // Get brain's cached VFS instance (creates if doesn't exist) this.vfs = this.brain.vfs // CRITICAL FIX: Always call vfs.init() explicitly // The previous code tried to check if initialized via stat('/') but this was unreliable // vfs.init() is idempotent, so calling it multiple times is safe await this.vfs.init() } /** * Generate VFS structure from import result */ async generate( importResult: SmartExcelResult, options: VFSStructureOptions ): Promise { const startTime = Date.now() const result: VFSStructureResult = { rootPath: options.rootPath, directories: [], files: [], operations: 0, duration: 0 } // Ensure VFS is initialized await this.init() // Calculate total operations for progress tracking const groups = this.groupEntities(importResult, options) const totalEntities = Array.from(groups.values()).reduce((sum, entities) => sum + entities.length, 0) const totalOperations = 1 + // root directory (options.preserveSource ? 1 : 0) + // source file groups.size + // group directories totalEntities + // entity files (options.createRelationshipFile !== false ? 1 : 0) + // relationships file (options.createMetadataFile !== false ? 1 : 0) // metadata file let completedOperations = 0 // Helper to report progress const reportProgress = (stage: 'directories' | 'entities' | 'metadata', message: string) => { completedOperations++ options.onProgress?.({ stage, message, processed: completedOperations, total: totalOperations }) } // Extract tracking metadata if provided const trackingMetadata = options.trackingContext ? { importIds: [options.trackingContext.importId], projectId: options.trackingContext.projectId, importedAt: options.trackingContext.importedAt, importFormat: options.trackingContext.importFormat, importSource: options.trackingContext.importSource, ...options.trackingContext.customMetadata } : {} // Create root directory try { await this.vfs.mkdir(options.rootPath, { recursive: true, metadata: trackingMetadata // Add tracking metadata }) result.directories.push(options.rootPath) result.operations++ reportProgress('directories', `Created root directory: ${options.rootPath}`) } catch (error: any) { // Directory might already exist, that's fine if (error.code !== 'EEXIST') { throw error } result.directories.push(options.rootPath) reportProgress('directories', `Root directory exists: ${options.rootPath}`) } // Preserve source file if requested if (options.preserveSource && options.sourceBuffer && options.sourceFilename) { const sourcePath = `${options.rootPath}/_source${this.getExtension(options.sourceFilename)}` await this.vfs.writeFile(sourcePath, options.sourceBuffer, { metadata: trackingMetadata // Add tracking metadata }) result.files.push({ path: sourcePath, type: 'source' }) result.operations++ reportProgress('metadata', `Preserved source file: ${options.sourceFilename}`) } // Note: groups already calculated above for progress tracking // const groups = this.groupEntities(importResult, options) // Create directories and files for each group for (const [groupName, entities] of groups.entries()) { const groupPath = `${options.rootPath}/${groupName}` // Create group directory try { await this.vfs.mkdir(groupPath, { recursive: true, metadata: trackingMetadata // Add tracking metadata }) result.directories.push(groupPath) result.operations++ reportProgress('directories', `Created directory: ${groupName} (${entities.length} entities)`) } catch (error: any) { // Directory might already exist if (error.code !== 'EEXIST') { throw error } result.directories.push(groupPath) reportProgress('directories', `Directory exists: ${groupName}`) } // Create entity files let entityCount = 0 for (const extracted of entities) { entityCount++ const sanitizedName = this.sanitizeFilename(extracted.entity.name) const entityPath = `${groupPath}/${sanitizedName}.json` // Create entity JSON const entityJson = { id: extracted.entity.id, name: extracted.entity.name, type: extracted.entity.type, description: extracted.entity.description, confidence: extracted.entity.confidence, metadata: extracted.entity.metadata, concepts: extracted.concepts || [], relatedEntities: extracted.relatedEntities, relationships: extracted.relationships.map(rel => ({ from: rel.from, to: rel.to, type: rel.type, confidence: rel.confidence, evidence: rel.evidence })) } await this.vfs.writeFile(entityPath, JSON.stringify(entityJson, null, 2), { metadata: { ...trackingMetadata, // Add tracking metadata entityId: extracted.entity.id } }) result.files.push({ path: entityPath, entityId: extracted.entity.id, type: 'entity' }) result.operations++ // Report progress every 10 entities (or on last entity) if (entityCount % 10 === 0 || entityCount === entities.length) { reportProgress('entities', `Created ${entityCount}/${entities.length} ${groupName} files`) } } } // Create relationships file if (options.createRelationshipFile !== false) { const relationshipsPath = `${options.rootPath}/_relationships.json` const allRelationships = importResult.rows.flatMap(row => row.relationships) const relationshipsJson = { source: options.sourceFilename || 'unknown', count: allRelationships.length, relationships: allRelationships, stats: { byType: this.groupByType(allRelationships, 'type'), byConfidence: { high: allRelationships.filter(r => r.confidence > 0.8).length, medium: allRelationships.filter(r => r.confidence >= 0.6 && r.confidence <= 0.8).length, low: allRelationships.filter(r => r.confidence < 0.6).length } } } await this.vfs.writeFile(relationshipsPath, JSON.stringify(relationshipsJson, null, 2), { metadata: trackingMetadata // Add tracking metadata }) result.files.push({ path: relationshipsPath, type: 'relationships' }) result.operations++ reportProgress('metadata', `Created relationships file (${allRelationships.length} relationships)`) } // Create metadata file if (options.createMetadataFile !== false) { const metadataPath = `${options.rootPath}/_metadata.json` const metadataJson = { import: { timestamp: new Date().toISOString(), source: { filename: options.sourceFilename || 'unknown', format: 'excel' }, options: { groupBy: options.groupBy, preserveSource: options.preserveSource }, stats: { rowsProcessed: importResult.rowsProcessed, entitiesExtracted: importResult.entitiesExtracted, relationshipsInferred: importResult.relationshipsInferred, processingTime: importResult.processingTime, byType: importResult.stats.byType, byConfidence: importResult.stats.byConfidence } }, structure: { rootPath: options.rootPath, groupingStrategy: options.groupBy, directories: result.directories, fileCount: result.files.length } } await this.vfs.writeFile(metadataPath, JSON.stringify(metadataJson, null, 2), { metadata: trackingMetadata // Add tracking metadata }) result.files.push({ path: metadataPath, type: 'metadata' }) result.operations++ reportProgress('metadata', 'Created metadata file') } // Final progress update if (options.onProgress) { options.onProgress({ stage: 'metadata', message: `VFS structure created successfully (${result.files.length} files, ${result.directories.length} directories)`, processed: totalOperations, total: totalOperations }) } result.duration = Date.now() - startTime return result } /** * Group entities by strategy */ private groupEntities( importResult: SmartExcelResult, options: VFSStructureOptions ): Map { const groups = new Map() // Handle sheet-based grouping if (options.groupBy === 'sheet' && importResult.sheets && importResult.sheets.length > 0) { for (const sheet of importResult.sheets) { groups.set(sheet.name, sheet.rows) } return groups } // Handle other grouping strategies for (const extracted of importResult.rows) { let groupName: string switch (options.groupBy) { case 'type': groupName = this.getTypeGroupName(extracted.entity.type) break case 'flat': groupName = 'entities' break case 'custom': groupName = options.customGrouping ? options.customGrouping(extracted.entity) : 'entities' break case 'sheet': // Fallback if sheets data not available groupName = 'entities' break default: groupName = 'entities' } if (!groups.has(groupName)) { groups.set(groupName, []) } groups.get(groupName)!.push(extracted) } return groups } /** * Get directory name for entity type */ private getTypeGroupName(type: NounType): string { const typeMap: Record = { [NounType.Person]: 'Characters', [NounType.Location]: 'Places', [NounType.Organization]: 'Organizations', [NounType.Concept]: 'Concepts', [NounType.Event]: 'Events', [NounType.Product]: 'Items', [NounType.Document]: 'Documents', [NounType.Project]: 'Projects', [NounType.Thing]: 'Other' } return typeMap[type as string] || 'Other' } /** * Sanitize filename */ private sanitizeFilename(name: string): string { return name .replace(/[<>:"/\\|?*]/g, '_') // Replace invalid chars .replace(/\s+/g, '_') // Replace spaces .replace(/_{2,}/g, '_') // Collapse multiple underscores .substring(0, 200) // Limit length } /** * Get file extension */ private getExtension(filename: string): string { const lastDot = filename.lastIndexOf('.') return lastDot !== -1 ? filename.substring(lastDot) : '.bin' } /** * Group items by property */ private groupByType>( items: T[], property: keyof T ): Record { const groups: Record = {} for (const item of items) { const key = String(item[property]) groups[key] = (groups[key] || 0) + 1 } return groups } }