fix: resolve import hang and index rebuild data loss bugs

Critical Bug Fixes (v3.43.2):

Bug #1 - Import Infinite Loop:
- Fix placeholder entity infinite loop in ImportCoordinator
- Use exact matching instead of fuzzy .includes() for entity names
- Search entities array (not rows) for existing placeholders
- Add duplicate relationship prevention in brain.relate()

Bug #2 - Index Rebuild File Discovery:
- Fix fileSystemStorage to scan sharded subdirectories
- Update getAllNodes() to use getAllShardedFiles()
- Update getAllEdges() to use getAllShardedFiles()
- Update getNodesByNounType() to use getAllShardedFiles()
- Fix getStorageStatus() to use O(1) persisted counts

Additional Improvements:
- Add brain.flush() API for explicit index persistence
- Make GraphAdjacencyIndex.flush() public
- Add auto-flush at end of import pipeline
- Update duplicate relationship test to expect deduplication

Files Modified:
- src/storage/adapters/fileSystemStorage.ts
- src/import/ImportCoordinator.ts
- src/brainy.ts
- src/graph/graphAdjacencyIndex.ts
- tests/unit/brainy/relate.test.ts
This commit is contained in:
David Snelling 2025-10-14 13:06:32 -07:00
parent 165def11a9
commit dcf20ffa1d
6 changed files with 184 additions and 83 deletions

View file

@ -320,6 +320,16 @@ export class ImportCoordinator {
)
}
// CRITICAL FIX (v3.43.2): Auto-flush all indexes before returning
// Ensures imported data survives server restarts
// Bug #5: Import data was only in memory, lost on restart
options.onProgress?.({
stage: 'complete',
message: 'Flushing indexes to disk...'
})
await this.brain.flush()
return result
}
@ -590,10 +600,12 @@ export class ImportCoordinator {
if (options.createRelationships && row.relationships) {
for (const rel of row.relationships) {
try {
// Find or create target entity
// CRITICAL FIX (v3.43.2): Prevent infinite placeholder creation loop
// Find or create target entity using EXACT matching only
let targetEntityId: string | undefined
// Check if target already exists in our entities list
// STEP 1: Check if target already exists in entities list (includes placeholders)
// This prevents creating duplicate placeholders - the root cause of Bug #1
const existingTarget = entities.find(e =>
e.name.toLowerCase() === rel.to.toLowerCase()
)
@ -601,17 +613,19 @@ export class ImportCoordinator {
if (existingTarget) {
targetEntityId = existingTarget.id
} else {
// Try to find in other extracted entities
// STEP 2: Try to find in extraction results (rows)
// FIX: Use EXACT matching instead of fuzzy .includes()
// Fuzzy matching caused false matches (e.g., "Entity_29" matching "Entity_297")
for (const otherRow of rows) {
const otherEntity = otherRow.entity || otherRow
if (rel.to.toLowerCase().includes(otherEntity.name.toLowerCase()) ||
otherEntity.name.toLowerCase().includes(rel.to.toLowerCase())) {
if (otherEntity.name.toLowerCase() === rel.to.toLowerCase()) {
targetEntityId = otherEntity.id
break
}
}
// If still not found, create placeholder entity
// STEP 3: If still not found, create placeholder entity ONCE
// The placeholder is added to entities array, so future searches will find it
if (!targetEntityId) {
targetEntityId = await this.brain.add({
data: rel.to,
@ -624,6 +638,7 @@ export class ImportCoordinator {
}
})
// CRITICAL: Add to entities array so future searches find it
entities.push({
id: targetEntityId,
name: rel.to,