fix: resolve import hang and index rebuild data loss bugs
Critical Bug Fixes (v3.43.2): Bug #1 - Import Infinite Loop: - Fix placeholder entity infinite loop in ImportCoordinator - Use exact matching instead of fuzzy .includes() for entity names - Search entities array (not rows) for existing placeholders - Add duplicate relationship prevention in brain.relate() Bug #2 - Index Rebuild File Discovery: - Fix fileSystemStorage to scan sharded subdirectories - Update getAllNodes() to use getAllShardedFiles() - Update getAllEdges() to use getAllShardedFiles() - Update getNodesByNounType() to use getAllShardedFiles() - Fix getStorageStatus() to use O(1) persisted counts Additional Improvements: - Add brain.flush() API for explicit index persistence - Make GraphAdjacencyIndex.flush() public - Add auto-flush at end of import pipeline - Update duplicate relationship test to expect deduplication Files Modified: - src/storage/adapters/fileSystemStorage.ts - src/import/ImportCoordinator.ts - src/brainy.ts - src/graph/graphAdjacencyIndex.ts - tests/unit/brainy/relate.test.ts
This commit is contained in:
parent
165def11a9
commit
dcf20ffa1d
6 changed files with 184 additions and 83 deletions
|
|
@ -320,6 +320,16 @@ export class ImportCoordinator {
|
|||
)
|
||||
}
|
||||
|
||||
// CRITICAL FIX (v3.43.2): Auto-flush all indexes before returning
|
||||
// Ensures imported data survives server restarts
|
||||
// Bug #5: Import data was only in memory, lost on restart
|
||||
options.onProgress?.({
|
||||
stage: 'complete',
|
||||
message: 'Flushing indexes to disk...'
|
||||
})
|
||||
|
||||
await this.brain.flush()
|
||||
|
||||
return result
|
||||
}
|
||||
|
||||
|
|
@ -590,10 +600,12 @@ export class ImportCoordinator {
|
|||
if (options.createRelationships && row.relationships) {
|
||||
for (const rel of row.relationships) {
|
||||
try {
|
||||
// Find or create target entity
|
||||
// CRITICAL FIX (v3.43.2): Prevent infinite placeholder creation loop
|
||||
// Find or create target entity using EXACT matching only
|
||||
let targetEntityId: string | undefined
|
||||
|
||||
// Check if target already exists in our entities list
|
||||
// STEP 1: Check if target already exists in entities list (includes placeholders)
|
||||
// This prevents creating duplicate placeholders - the root cause of Bug #1
|
||||
const existingTarget = entities.find(e =>
|
||||
e.name.toLowerCase() === rel.to.toLowerCase()
|
||||
)
|
||||
|
|
@ -601,17 +613,19 @@ export class ImportCoordinator {
|
|||
if (existingTarget) {
|
||||
targetEntityId = existingTarget.id
|
||||
} else {
|
||||
// Try to find in other extracted entities
|
||||
// STEP 2: Try to find in extraction results (rows)
|
||||
// FIX: Use EXACT matching instead of fuzzy .includes()
|
||||
// Fuzzy matching caused false matches (e.g., "Entity_29" matching "Entity_297")
|
||||
for (const otherRow of rows) {
|
||||
const otherEntity = otherRow.entity || otherRow
|
||||
if (rel.to.toLowerCase().includes(otherEntity.name.toLowerCase()) ||
|
||||
otherEntity.name.toLowerCase().includes(rel.to.toLowerCase())) {
|
||||
if (otherEntity.name.toLowerCase() === rel.to.toLowerCase()) {
|
||||
targetEntityId = otherEntity.id
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
// If still not found, create placeholder entity
|
||||
// STEP 3: If still not found, create placeholder entity ONCE
|
||||
// The placeholder is added to entities array, so future searches will find it
|
||||
if (!targetEntityId) {
|
||||
targetEntityId = await this.brain.add({
|
||||
data: rel.to,
|
||||
|
|
@ -624,6 +638,7 @@ export class ImportCoordinator {
|
|||
}
|
||||
})
|
||||
|
||||
// CRITICAL: Add to entities array so future searches find it
|
||||
entities.push({
|
||||
id: targetEntityId,
|
||||
name: rel.to,
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue