fix: import dedup off-switch honesty + brain-owned lifecycle for the background pass
The post-import background deduplication pass (a merge-DELETE writer, debounced ~5 minutes after import) had four lifecycle defects: - enableDeduplication: false did not gate the background schedule — an import that explicitly opted out could still have entities auto-removed minutes later. The flag now gates both the inline and background passes. - Each import() constructed its own coordinator-owned deduplicator, so the debounce never spanned imports (N imports = N delete timers). The brain now owns a single lazy instance (getBackgroundDeduplicator). - close() never cancelled pending dedup; a delete pass could fire against a closed brain. close() now cancels it first. - The 5-minute timer held the process open (exit-hang class); now unref'd. Four regression tests pin the contract (background-dedup-lifecycle); import guides document that false disables both passes.
This commit is contained in:
parent
120205b69c
commit
4fcef7b8ed
7 changed files with 179 additions and 10 deletions
|
|
@ -10128,6 +10128,30 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
|||
return await coordinator.import(source as Buffer | string | object, options)
|
||||
}
|
||||
|
||||
/** Brain-owned background deduplicator (lazy; see getBackgroundDeduplicator). */
|
||||
private _backgroundDedup?: import('./import/BackgroundDeduplicator.js').BackgroundDeduplicator
|
||||
|
||||
/**
|
||||
* The single brain-owned BackgroundDeduplicator, lazily constructed.
|
||||
*
|
||||
* Ownership matters here: the post-import dedup timer must outlive the
|
||||
* per-call ImportCoordinator but never the brain. One instance per brain
|
||||
* restores the intended cross-import debounce (per-coordinator instances
|
||||
* each armed their own timer, so the "debounce" never spanned imports) and
|
||||
* gives close() a handle to cancel pending work — a delete pass must never
|
||||
* fire against a closed brain.
|
||||
* @internal
|
||||
*/
|
||||
async getBackgroundDeduplicator(): Promise<
|
||||
import('./import/BackgroundDeduplicator.js').BackgroundDeduplicator
|
||||
> {
|
||||
if (!this._backgroundDedup) {
|
||||
const { BackgroundDeduplicator } = await import('./import/BackgroundDeduplicator.js')
|
||||
this._backgroundDedup = new BackgroundDeduplicator(this)
|
||||
}
|
||||
return this._backgroundDedup
|
||||
}
|
||||
|
||||
/**
|
||||
* Virtual File System API - Knowledge Operating System
|
||||
*
|
||||
|
|
@ -16070,6 +16094,10 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
|||
* This ensures deferred persistence mode data is saved
|
||||
*/
|
||||
async close(): Promise<void> {
|
||||
// Cancel any pending post-import background deduplication FIRST — it is a
|
||||
// writer (merge-deletes), and no delete pass may start mid- or post-close.
|
||||
this._backgroundDedup?.cancelPending()
|
||||
|
||||
// Change-feed teardown: no events are delivered for or after close().
|
||||
this._changeFeed.close()
|
||||
|
||||
|
|
|
|||
|
|
@ -41,6 +41,14 @@ export interface DeduplicationStats {
|
|||
* - Import-scoped deduplication (no cross-contamination)
|
||||
* - 3-tier strategy (ID → Name → Similarity)
|
||||
* - Uses existing indexes (EntityIdMapper, MetadataIndexManager, TypeAware HNSW)
|
||||
*
|
||||
* Lifecycle: ONE instance per brain, owned by Brainy (getBackgroundDeduplicator)
|
||||
* so the debounce genuinely spans imports and brain.close() cancels pending
|
||||
* work via cancelPending() — this pass merge-DELETES duplicate entities, so it
|
||||
* must never fire against a closed brain. The enableDeduplication gate lives
|
||||
* at the scheduling call site (ImportCoordinator); scheduleDedup itself is
|
||||
* unconditional. The timer is unref'd — a pending pass never holds the
|
||||
* process open.
|
||||
*/
|
||||
export class BackgroundDeduplicator {
|
||||
private brain: Brainy
|
||||
|
|
@ -67,12 +75,15 @@ export class BackgroundDeduplicator {
|
|||
clearTimeout(this.debounceTimer)
|
||||
}
|
||||
|
||||
// Schedule for 5 minutes from now
|
||||
// Schedule for 5 minutes from now. unref'd: a pending dedup pass must
|
||||
// never hold the process open (exit-hang class) — if the process exits
|
||||
// first, the pass simply never runs; imports are already durable.
|
||||
this.debounceTimer = setTimeout(() => {
|
||||
this.runBatchDedup().catch(error => {
|
||||
prodLog.error('[BackgroundDedup] Batch dedup failed:', error)
|
||||
})
|
||||
}, 5 * 60 * 1000)
|
||||
this.debounceTimer.unref?.()
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
|
|||
|
|
@ -13,7 +13,6 @@
|
|||
import { Brainy } from '../brainy.js'
|
||||
import { FormatDetector, SupportedFormat } from './FormatDetector.js'
|
||||
import { ImportHistory, type ImportHistoryEntry } from './ImportHistory.js'
|
||||
import { BackgroundDeduplicator } from './BackgroundDeduplicator.js'
|
||||
import { SmartExcelImporter } from '../importers/SmartExcelImporter.js'
|
||||
import { SmartPDFImporter } from '../importers/SmartPDFImporter.js'
|
||||
import { SmartCSVImporter } from '../importers/SmartCSVImporter.js'
|
||||
|
|
@ -112,7 +111,12 @@ export interface ValidImportOptions {
|
|||
/** Confidence threshold for entities */
|
||||
confidenceThreshold?: number
|
||||
|
||||
/** Enable entity deduplication across imports */
|
||||
/**
|
||||
* Enable entity deduplication (default: true). Gates BOTH passes: the
|
||||
* inline merge during import AND the debounced background pass that runs
|
||||
* ~5 minutes after the last import (which merge-DELETES duplicate entities).
|
||||
* Set false for deployments that must never auto-remove records.
|
||||
*/
|
||||
enableDeduplication?: boolean
|
||||
|
||||
/** Similarity threshold for deduplication (0-1) */
|
||||
|
|
@ -286,7 +290,6 @@ export class ImportCoordinator {
|
|||
private brain: Brainy
|
||||
private detector: FormatDetector
|
||||
private history: ImportHistory
|
||||
private backgroundDedup: BackgroundDeduplicator
|
||||
private excelImporter: SmartExcelImporter
|
||||
private pdfImporter: SmartPDFImporter
|
||||
private csvImporter: SmartCSVImporter
|
||||
|
|
@ -300,7 +303,6 @@ export class ImportCoordinator {
|
|||
this.brain = brain
|
||||
this.detector = new FormatDetector()
|
||||
this.history = new ImportHistory(brain)
|
||||
this.backgroundDedup = new BackgroundDeduplicator(brain)
|
||||
this.excelImporter = new SmartExcelImporter(brain)
|
||||
this.pdfImporter = new SmartPDFImporter(brain)
|
||||
this.csvImporter = new SmartCSVImporter(brain)
|
||||
|
|
@ -1459,9 +1461,16 @@ export class ImportCoordinator {
|
|||
}
|
||||
}
|
||||
|
||||
// Schedule background deduplication (debounced 5 minutes)
|
||||
if (trackingContext && trackingContext.importId) {
|
||||
this.backgroundDedup.scheduleDedup(trackingContext.importId)
|
||||
// Schedule background deduplication (debounced 5 minutes, brain-owned so
|
||||
// close() can cancel it). Honors the same enableDeduplication gate as the
|
||||
// inline pass — false means NO dedup, inline or background.
|
||||
if (
|
||||
trackingContext &&
|
||||
trackingContext.importId &&
|
||||
options.enableDeduplication !== false
|
||||
) {
|
||||
const backgroundDedup = await this.brain.getBackgroundDeduplicator()
|
||||
backgroundDedup.scheduleDedup(trackingContext.importId)
|
||||
}
|
||||
|
||||
return {
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue