perf: optimize imports with background deduplication (12-24x speedup)

- Remove O(n²) deduplication from import path for 12-24x faster imports
- Implement BackgroundDeduplicator with 3-tier strategy (ID/Name/Similarity)
- Sequential tier processing reduces entity set after each pass
- Auto-schedules 5 minutes after imports (debounced, zero config)
- Import-scoped deduplication prevents cross-contamination

GraphAdjacencyIndex improvements:
- Fix concurrent rebuild race condition with promise-based locking
- Fix removeVerb() by filtering deleted IDs in query methods
- Replace console.* with prodLog for silent mode compatibility

Performance impact:
- Import speed: O(n²) → O(n) complexity
- 400 entities: 24 min → 2 min (12x faster)
- 1000 entities: >2 hours → 5 min (24x faster)
- Background dedup uses existing indexes (TypeAware HNSW, MetadataIndexManager)

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
David Snelling 2025-11-11 14:10:14 -08:00
parent 804319ecaf
commit 02c80a045b
5 changed files with 743 additions and 177 deletions

View file

@ -12,8 +12,8 @@
import { Brainy } from '../brainy.js'
import { FormatDetector, SupportedFormat } from './FormatDetector.js'
import { EntityDeduplicator } from './EntityDeduplicator.js'
import { ImportHistory } from './ImportHistory.js'
import { BackgroundDeduplicator } from './BackgroundDeduplicator.js'
import { SmartExcelImporter } from '../importers/SmartExcelImporter.js'
import { SmartPDFImporter } from '../importers/SmartPDFImporter.js'
import { SmartCSVImporter } from '../importers/SmartCSVImporter.js'
@ -307,8 +307,8 @@ export interface ImportResult {
export class ImportCoordinator {
private brain: Brainy
private detector: FormatDetector
private deduplicator: EntityDeduplicator
private history: ImportHistory
private backgroundDedup: BackgroundDeduplicator
private excelImporter: SmartExcelImporter
private pdfImporter: SmartPDFImporter
private csvImporter: SmartCSVImporter
@ -321,8 +321,8 @@ export class ImportCoordinator {
constructor(brain: Brainy) {
this.brain = brain
this.detector = new FormatDetector()
this.deduplicator = new EntityDeduplicator(brain)
this.history = new ImportHistory(brain)
this.backgroundDedup = new BackgroundDeduplicator(brain)
this.excelImporter = new SmartExcelImporter(brain)
this.pdfImporter = new SmartPDFImporter(brain)
this.csvImporter = new SmartCSVImporter(brain)
@ -1089,49 +1089,34 @@ export class ImportCoordinator {
try {
const importSource = vfsResult.rootPath
let entityId: string
let wasMerged = false
// Use deduplicator to check for existing entities
const mergeResult = await this.deduplicator.createOrMerge(
{
id: entity.id,
// v5.7.0: No deduplication during import (12-24x speedup)
// Background deduplication runs 5 minutes after import completes
entityId = await this.brain.add({
data: entity.description || entity.name,
type: entity.type,
metadata: {
...entity.metadata,
name: entity.name,
type: entity.type,
description: entity.description || entity.name,
confidence: entity.confidence,
metadata: {
...entity.metadata,
vfsPath: vfsFile?.path,
importedFrom: 'import-coordinator',
// v4.10.0: Import tracking metadata
...(trackingContext && {
importIds: [trackingContext.importId],
projectId: trackingContext.projectId,
importedAt: trackingContext.importedAt,
importFormat: trackingContext.importFormat,
importSource: trackingContext.importSource,
sourceRow: row.rowNumber,
sourceSheet: row.sheet,
...trackingContext.customMetadata
})
}
},
importSource,
{
similarityThreshold: options.deduplicationThreshold || 0.85,
strictTypeMatching: true,
enableFuzzyMatching: true
vfsPath: vfsFile?.path,
importedFrom: 'import-coordinator',
// v4.10.0: Import tracking metadata
...(trackingContext && {
importId: trackingContext.importId, // Used for background dedup
importIds: [trackingContext.importId],
projectId: trackingContext.projectId,
importedAt: trackingContext.importedAt,
importFormat: trackingContext.importFormat,
importSource: trackingContext.importSource,
sourceRow: row.rowNumber,
sourceSheet: row.sheet,
...trackingContext.customMetadata
})
}
)
})
entityId = mergeResult.mergedEntityId
wasMerged = mergeResult.wasMerged
if (wasMerged) {
mergedCount++
} else {
newCount++
}
newCount++
// Update entity ID in extraction result
entity.id = entityId
@ -1384,6 +1369,11 @@ export class ImportCoordinator {
}
}
// v5.7.0: Schedule background deduplication (debounced 5 minutes)
if (trackingContext && trackingContext.importId) {
this.backgroundDedup.scheduleDedup(trackingContext.importId)
}
return {
entities,
relationships,