fix: import dedup off-switch honesty + brain-owned lifecycle for the background pass

The post-import background deduplication pass (a merge-DELETE writer,
debounced ~5 minutes after import) had four lifecycle defects:

- enableDeduplication: false did not gate the background schedule — an
  import that explicitly opted out could still have entities auto-removed
  minutes later. The flag now gates both the inline and background passes.
- Each import() constructed its own coordinator-owned deduplicator, so the
  debounce never spanned imports (N imports = N delete timers). The brain
  now owns a single lazy instance (getBackgroundDeduplicator).
- close() never cancelled pending dedup; a delete pass could fire against
  a closed brain. close() now cancels it first.
- The 5-minute timer held the process open (exit-hang class); now unref'd.

Four regression tests pin the contract (background-dedup-lifecycle);
import guides document that false disables both passes.
This commit is contained in:
David Snelling 2026-07-18 10:40:45 -07:00
parent 120205b69c
commit 4fcef7b8ed
7 changed files with 179 additions and 10 deletions

View file

@ -13,7 +13,6 @@
import { Brainy } from '../brainy.js'
import { FormatDetector, SupportedFormat } from './FormatDetector.js'
import { ImportHistory, type ImportHistoryEntry } from './ImportHistory.js'
import { BackgroundDeduplicator } from './BackgroundDeduplicator.js'
import { SmartExcelImporter } from '../importers/SmartExcelImporter.js'
import { SmartPDFImporter } from '../importers/SmartPDFImporter.js'
import { SmartCSVImporter } from '../importers/SmartCSVImporter.js'
@ -112,7 +111,12 @@ export interface ValidImportOptions {
/** Confidence threshold for entities */
confidenceThreshold?: number
/** Enable entity deduplication across imports */
/**
* Enable entity deduplication (default: true). Gates BOTH passes: the
* inline merge during import AND the debounced background pass that runs
* ~5 minutes after the last import (which merge-DELETES duplicate entities).
* Set false for deployments that must never auto-remove records.
*/
enableDeduplication?: boolean
/** Similarity threshold for deduplication (0-1) */
@ -286,7 +290,6 @@ export class ImportCoordinator {
private brain: Brainy
private detector: FormatDetector
private history: ImportHistory
private backgroundDedup: BackgroundDeduplicator
private excelImporter: SmartExcelImporter
private pdfImporter: SmartPDFImporter
private csvImporter: SmartCSVImporter
@ -300,7 +303,6 @@ export class ImportCoordinator {
this.brain = brain
this.detector = new FormatDetector()
this.history = new ImportHistory(brain)
this.backgroundDedup = new BackgroundDeduplicator(brain)
this.excelImporter = new SmartExcelImporter(brain)
this.pdfImporter = new SmartPDFImporter(brain)
this.csvImporter = new SmartCSVImporter(brain)
@ -1459,9 +1461,16 @@ export class ImportCoordinator {
}
}
// Schedule background deduplication (debounced 5 minutes)
if (trackingContext && trackingContext.importId) {
this.backgroundDedup.scheduleDedup(trackingContext.importId)
// Schedule background deduplication (debounced 5 minutes, brain-owned so
// close() can cancel it). Honors the same enableDeduplication gate as the
// inline pass — false means NO dedup, inline or background.
if (
trackingContext &&
trackingContext.importId &&
options.enableDeduplication !== false
) {
const backgroundDedup = await this.brain.getBackgroundDeduplicator()
backgroundDedup.scheduleDedup(trackingContext.importId)
}
return {