From 549d7736507b7f21484e161fc586ef5341b42cc5 Mon Sep 17 00:00:00 2001 From: David Snelling Date: Thu, 30 Oct 2025 13:19:08 -0700 Subject: [PATCH] fix: prevent orphaned relationships in restore() and add VFS progress tracking MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - DataAPI.restore() now filters relationships to prevent orphaned references when some entities fail to restore - Added relationshipsSkipped tracking to restore() return type - VFSStructureGenerator now reports progress during VFS creation (directories, entities, metadata stages) - ImportCoordinator wires VFS progress callback to main import progress - Fixes P0 "Entity not found" errors after restore - Fixes P1 "import appears frozen" during 3-5 minute VFS creation 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude --- CHANGELOG.md | 25 +++++++++++ src/api/DataAPI.ts | 35 ++++++++++++++- src/import/ImportCoordinator.ts | 11 ++++- src/importers/VFSStructureGenerator.ts | 60 +++++++++++++++++++++++++- 4 files changed, 126 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 57aa25f9..4dbb80f5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,31 @@ All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines. +## [4.11.1](https://github.com/soulcraftlabs/brainy/compare/v4.11.0...v4.11.1) (2025-10-30) + +### 🐛 Bug Fixes + +* **fix(api)**: DataAPI.restore() now filters orphaned relationships (P0 Critical) + - **Issue**: restore() created relationships to entities that failed to restore, causing "Entity not found" errors + - **Root Cause**: Relationships were not filtered based on successfully restored entities + - **Fix**: Now builds Set of successful entity IDs and filters relationships accordingly + - **New Tracking**: Added `relationshipsSkipped` to return type for visibility + - **Impact**: Prevents complete data corruption when some entities fail to restore + +* **fix(import)**: VFS creation now reports progress during import (P1 High) + - **Issue**: 3-5 minute VFS creation showed no progress (stuck at 0%), causing users to think import froze + - **Root Cause**: VFSStructureGenerator.generate() had no progress callback parameter + - **Fix**: Added onProgress callback to VFSStructureOptions interface + - **Progress Stages**: Reports 'directories', 'entities', 'metadata' with detailed messages + - **Frequency**: Reports every 10 entity files to avoid excessive updates + - **Integration**: Wired through ImportCoordinator to main progress callback + +### 📝 Files Modified + +* `src/api/DataAPI.ts` (lines 173-350) - Added orphaned relationship filtering +* `src/importers/VFSStructureGenerator.ts` (lines 18-53, 110-347) - Added progress callback +* `src/import/ImportCoordinator.ts` (lines 438-459) - Wired progress callback + ## [4.11.0](https://github.com/soulcraftlabs/brainy/compare/v4.10.4...v4.11.0) (2025-10-30) ### 🚨 CRITICAL BUG FIX diff --git a/src/api/DataAPI.ts b/src/api/DataAPI.ts index 4de6e6ea..04a882ce 100644 --- a/src/api/DataAPI.ts +++ b/src/api/DataAPI.ts @@ -179,6 +179,7 @@ export class DataAPI { }): Promise<{ entitiesRestored: number relationshipsRestored: number + relationshipsSkipped: number errors: Array<{ type: 'entity' | 'relation'; id: string; error: string }> }> { const { backup, merge = false, overwrite = false, validate = true, onProgress } = params @@ -186,6 +187,7 @@ export class DataAPI { const result = { entitiesRestored: 0, relationshipsRestored: 0, + relationshipsSkipped: 0, errors: [] as Array<{ type: 'entity' | 'relation'; id: string; error: string }> } @@ -244,6 +246,9 @@ export class DataAPI { }) // Restore entities in batches using storage-aware batching (v4.11.0) + // v4.11.1: Track successful entity IDs to prevent orphaned relationships + const successfulEntityIds = new Set() + if (entityParams.length > 0) { try { const addResult = await this.brain.addMany({ @@ -256,6 +261,11 @@ export class DataAPI { result.entitiesRestored = addResult.successful.length + // Build Set of successfully restored entity IDs (v4.11.1 Bug Fix) + addResult.successful.forEach((entityId: string) => { + successfulEntityIds.add(entityId) + }) + // Track errors addResult.failed.forEach((failure: any) => { result.errors.push({ @@ -271,10 +281,10 @@ export class DataAPI { // ============================================ // Phase 2: Restore relationships using relateMany() - // v4.11.1: Uses proper persistence path through brain.relateMany() + // v4.11.1: CRITICAL FIX - Filter orphaned relationships // ============================================ - // Prepare relationship parameters for relateMany() + // Prepare relationship parameters - filter out orphaned references const relationParams = backup.relations .filter(relation => { // Skip existing relations when merging without overwrite @@ -282,6 +292,27 @@ export class DataAPI { // Note: We'll rely on relateMany's internal duplicate handling return true } + + // v4.11.1 CRITICAL BUG FIX: Skip relationships where source or target entity failed to restore + // This prevents creating orphaned references that cause "Entity not found" errors + const sourceExists = successfulEntityIds.has(relation.from) + const targetExists = successfulEntityIds.has(relation.to) + + if (!sourceExists || !targetExists) { + // Track skipped relationship + result.relationshipsSkipped++ + + // Optionally log for debugging (can be removed in production) + if (process.env.NODE_ENV !== 'production') { + console.debug( + `Skipping orphaned relationship: ${relation.from} -> ${relation.to} ` + + `(${!sourceExists ? 'missing source' : 'missing target'})` + ) + } + + return false + } + return true }) .map(relation => ({ diff --git a/src/import/ImportCoordinator.ts b/src/import/ImportCoordinator.ts index 3e0acc0c..3ffc70df 100644 --- a/src/import/ImportCoordinator.ts +++ b/src/import/ImportCoordinator.ts @@ -445,7 +445,16 @@ export class ImportCoordinator { sourceFilename: normalizedSource.filename || `import.${detection.format}`, createRelationshipFile: true, createMetadataFile: true, - trackingContext // v4.10.0: Pass tracking metadata to VFS + trackingContext, // v4.10.0: Pass tracking metadata to VFS + // v4.11.1: Pass progress callback for VFS creation updates + onProgress: (vfsProgress) => { + options.onProgress?.({ + stage: 'storing-vfs', + message: vfsProgress.message, + processed: vfsProgress.processed, + total: vfsProgress.total + }) + } }) // Report graph storage stage diff --git a/src/importers/VFSStructureGenerator.ts b/src/importers/VFSStructureGenerator.ts index ca4e37f4..72e87b17 100644 --- a/src/importers/VFSStructureGenerator.ts +++ b/src/importers/VFSStructureGenerator.ts @@ -42,6 +42,14 @@ export interface VFSStructureOptions { /** Import tracking context (v4.10.0) */ trackingContext?: TrackingContext + + /** Progress callback (v4.11.1) - Reports VFS creation progress */ + onProgress?: (progress: { + stage: 'directories' | 'entities' | 'metadata' + message: string + processed: number + total: number + }) => void } export interface VFSStructureResult { @@ -115,6 +123,30 @@ export class VFSStructureGenerator { // Ensure VFS is initialized await this.init() + // v4.11.1: Calculate total operations for progress tracking + const groups = this.groupEntities(importResult, options) + const totalEntities = Array.from(groups.values()).reduce((sum, entities) => sum + entities.length, 0) + const totalOperations = + 1 + // root directory + (options.preserveSource ? 1 : 0) + // source file + groups.size + // group directories + totalEntities + // entity files + (options.createRelationshipFile !== false ? 1 : 0) + // relationships file + (options.createMetadataFile !== false ? 1 : 0) // metadata file + + let completedOperations = 0 + + // Helper to report progress + const reportProgress = (stage: 'directories' | 'entities' | 'metadata', message: string) => { + completedOperations++ + options.onProgress?.({ + stage, + message, + processed: completedOperations, + total: totalOperations + }) + } + // Extract tracking metadata if provided const trackingMetadata = options.trackingContext ? { importIds: [options.trackingContext.importId], @@ -133,12 +165,14 @@ export class VFSStructureGenerator { }) result.directories.push(options.rootPath) result.operations++ + reportProgress('directories', `Created root directory: ${options.rootPath}`) } catch (error: any) { // Directory might already exist, that's fine if (error.code !== 'EEXIST') { throw error } result.directories.push(options.rootPath) + reportProgress('directories', `Root directory exists: ${options.rootPath}`) } // Preserve source file if requested @@ -152,10 +186,11 @@ export class VFSStructureGenerator { type: 'source' }) result.operations++ + reportProgress('metadata', `Preserved source file: ${options.sourceFilename}`) } - // Group entities - const groups = this.groupEntities(importResult, options) + // Note: groups already calculated above for progress tracking + // const groups = this.groupEntities(importResult, options) // Create directories and files for each group for (const [groupName, entities] of groups.entries()) { @@ -169,16 +204,20 @@ export class VFSStructureGenerator { }) result.directories.push(groupPath) result.operations++ + reportProgress('directories', `Created directory: ${groupName} (${entities.length} entities)`) } catch (error: any) { // Directory might already exist if (error.code !== 'EEXIST') { throw error } result.directories.push(groupPath) + reportProgress('directories', `Directory exists: ${groupName}`) } // Create entity files + let entityCount = 0 for (const extracted of entities) { + entityCount++ const sanitizedName = this.sanitizeFilename(extracted.entity.name) const entityPath = `${groupPath}/${sanitizedName}.json` @@ -213,6 +252,11 @@ export class VFSStructureGenerator { type: 'entity' }) result.operations++ + + // v4.11.1: Report progress every 10 entities (or on last entity) + if (entityCount % 10 === 0 || entityCount === entities.length) { + reportProgress('entities', `Created ${entityCount}/${entities.length} ${groupName} files`) + } } } @@ -243,6 +287,7 @@ export class VFSStructureGenerator { type: 'relationships' }) result.operations++ + reportProgress('metadata', `Created relationships file (${allRelationships.length} relationships)`) } // Create metadata file @@ -284,6 +329,17 @@ export class VFSStructureGenerator { type: 'metadata' }) result.operations++ + reportProgress('metadata', 'Created metadata file') + } + + // v4.11.1: Final progress update + if (options.onProgress) { + options.onProgress({ + stage: 'metadata', + message: `VFS structure created successfully (${result.files.length} files, ${result.directories.length} directories)`, + processed: totalOperations, + total: totalOperations + }) } result.duration = Date.now() - startTime