refactor(8.0): remove dead, unreachable, and unwired modules

Pre-GA dead-code sweep. A deterministic import-reachability walk from the
package's real entry points (the exports map, the CLI bin, the conversation
surface) found 34 source modules that nothing reachable imports — they
compile and ship as dist output that no consumer or internal path can ever
reach. All were superseded duplicates, abandoned parallel implementations,
or built-but-never-wired features:

  - superseded duplicates of live modules: an older "unified" entry, a
    standalone neural-import variant, a static NLP processor + its matcher,
    a parallel API-types module, a duplicate progress-types module
  - an abandoned import path (orchestrator + entity deduplicator + barrels)
    left behind when ingestion moved to the coordinator
  - unwired feature modules (instance pool, import presets, cached
    embeddings, relationship-confidence scorer) reachable only from tests
  - dead leaf utilities (write buffer, deleted-items index, bounded
    registry, two crypto shims, a cache manager, a structured logger, a
    stale v5 type-migration helper, a browser-only FS type shim) and
    several dead re-export barrels
  - a CLI catalog command wired into no command

Also removed two tests that only exercised deleted modules, repointed two
VFS tests off a deleted barrel onto the implementation module, and deleted
one example that demoed removed features.

Verified: clean build (both tsc passes), unit 1505 + integration 607 green,
and a re-run of the reachability walk reports zero remaining orphans.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
David Snelling 2026-06-24 15:18:40 -07:00
parent 03d654061f
commit bf0afe8563
39 changed files with 2 additions and 10497 deletions

View file

@ -1,354 +0,0 @@
/**
* Entity Deduplicator
*
* Finds and merges duplicate entities across imports using:
* - Embedding-based similarity matching
* - Type-aware comparison
* - Confidence-weighted merging
* - Provenance tracking
*
* NO MOCKS - Production-ready implementation
*/
import { Brainy } from '../brainy.js'
import { NounType } from '../types/graphTypes.js'
export interface EntityCandidate {
id?: string
name: string
type: NounType
description: string
confidence: number
metadata: Record<string, any>
}
export interface DuplicateMatch {
existingId: string
existingName: string
similarity: number
shouldMerge: boolean
reason: string
}
export interface EntityDeduplicationOptions {
/** Similarity threshold for considering entities as duplicates (0-1) */
similarityThreshold?: number
/** Only match entities of the same type */
strictTypeMatching?: boolean
/** Enable fuzzy name matching */
enableFuzzyMatching?: boolean
/** Minimum confidence to consider for merging */
minConfidence?: number
}
export interface MergeResult {
mergedEntityId: string
wasMerged: boolean
mergedWith?: string
confidence: number
provenance: string[]
}
/**
* EntityDeduplicator - Prevents duplicate entities across imports
*/
export class EntityDeduplicator {
private brain: Brainy
constructor(brain: Brainy) {
this.brain = brain
}
/**
* Find duplicate entities in the knowledge graph
*/
async findDuplicates(
candidate: EntityCandidate,
options: EntityDeduplicationOptions = {}
): Promise<DuplicateMatch | null> {
const opts = {
similarityThreshold: options.similarityThreshold || 0.85,
strictTypeMatching: options.strictTypeMatching !== false,
enableFuzzyMatching: options.enableFuzzyMatching !== false,
minConfidence: options.minConfidence || 0.6
}
// Skip low-confidence candidates
if (candidate.confidence < opts.minConfidence) {
return null
}
// Search for similar entities by name and description
const searchText = `${candidate.name} ${candidate.description}`.trim()
try {
const results = await this.brain.find({
query: searchText,
limit: 5,
where: opts.strictTypeMatching ? { type: candidate.type } : undefined
})
// Check each result for potential duplicates
for (const result of results) {
const similarity = result.score || 0
const existingName = result.entity.metadata?.name || result.id
const existingType = result.entity.metadata?.type || result.entity.metadata?.nounType || result.entity.type
// Skip if below similarity threshold
if (similarity < opts.similarityThreshold) {
continue
}
// Type matching check
if (opts.strictTypeMatching && existingType !== candidate.type) {
continue
}
// Exact name match (case-insensitive)
if (this.normalizeString(candidate.name) === this.normalizeString(existingName)) {
return {
existingId: result.id,
existingName,
similarity: 1.0,
shouldMerge: true,
reason: 'Exact name match'
}
}
// High similarity match
if (similarity >= opts.similarityThreshold) {
// Additional validation for fuzzy matching
if (opts.enableFuzzyMatching && this.areSimilarNames(candidate.name, existingName)) {
return {
existingId: result.id,
existingName,
similarity,
shouldMerge: true,
reason: `High similarity (${(similarity * 100).toFixed(1)}%)`
}
}
}
}
} catch (error) {
// If search fails, assume no duplicates
return null
}
return null
}
/**
* Merge entity data with existing entity
*/
async mergeEntity(
existingId: string,
candidate: EntityCandidate,
importSource: string
): Promise<MergeResult> {
try {
// Get existing entity
const existing = await this.brain.get(existingId)
if (!existing) {
throw new Error(`Entity ${existingId} not found`)
}
// Update confidence (weighted average) — `confidence` is a reserved
// top-level field, so it travels via the dedicated update() param, not
// the metadata bag.
const mergedConfidence = this.mergeConfidence(
existing.confidence ?? 0.5,
candidate.confidence
)
// Merge metadata (custom fields only — reserved fields are top-level)
const mergedMetadata = {
...existing.metadata,
// Track provenance
imports: [
...(existing.metadata?.imports || []),
importSource
],
// Merge VFS paths
vfsPaths: [
...(existing.metadata?.vfsPaths || [existing.metadata?.vfsPath]).filter(Boolean),
candidate.metadata?.vfsPath
].filter(Boolean),
// Merge other metadata
...this.mergeMetadataFields(existing.metadata, candidate.metadata),
// Track last update
lastUpdated: Date.now(),
mergeCount: (existing.metadata?.mergeCount || 0) + 1
}
// Update entity
await this.brain.update({
id: existingId,
confidence: mergedConfidence,
metadata: mergedMetadata,
merge: true
})
return {
mergedEntityId: existingId,
wasMerged: true,
mergedWith: existing.metadata?.name || existingId,
confidence: mergedConfidence,
provenance: mergedMetadata.imports
}
} catch (error) {
throw new Error(`Failed to merge entity: ${error instanceof Error ? error.message : String(error)}`)
}
}
/**
* Create or merge entity with deduplication
*/
async createOrMerge(
candidate: EntityCandidate,
importSource: string,
options: EntityDeduplicationOptions = {}
): Promise<MergeResult> {
// Check for duplicates
const duplicate = await this.findDuplicates(candidate, options)
if (duplicate && duplicate.shouldMerge) {
// Merge with existing entity
return await this.mergeEntity(duplicate.existingId, candidate, importSource)
}
// No duplicate found, create new entity. Preserve any subtype the candidate
// carried (set by the extractor or upstream importer), else fall back to
// `'imported'` so enforcement doesn't fire (added 7.30.1).
// `confidence` is a reserved top-level field (dedicated add() param);
// creation time is system-managed — neither belongs in the metadata bag.
const entityId = await this.brain.add({
data: candidate.description || candidate.name,
type: candidate.type,
subtype:
(candidate as EntityCandidate & { subtype?: string }).subtype ??
'imported',
confidence: candidate.confidence,
metadata: {
...candidate.metadata,
name: candidate.name,
imports: [importSource],
vfsPaths: [candidate.metadata?.vfsPath].filter(Boolean),
mergeCount: 0
}
})
// Update candidate with new ID
candidate.id = entityId
return {
mergedEntityId: entityId,
wasMerged: false,
confidence: candidate.confidence,
provenance: [importSource]
}
}
/**
* Normalize string for comparison
*/
private normalizeString(str: string): string {
return str
.toLowerCase()
.trim()
.replace(/[^a-z0-9]/g, '')
}
/**
* Check if two names are similar (fuzzy matching)
*/
private areSimilarNames(name1: string, name2: string): boolean {
const n1 = this.normalizeString(name1)
const n2 = this.normalizeString(name2)
// Exact match
if (n1 === n2) return true
// Length difference check
const lengthDiff = Math.abs(n1.length - n2.length)
if (lengthDiff > 3) return false
// Levenshtein distance
const distance = this.levenshteinDistance(n1, n2)
const maxLength = Math.max(n1.length, n2.length)
const similarity = 1 - (distance / maxLength)
return similarity >= 0.85
}
/**
* Calculate Levenshtein distance between two strings
*/
private levenshteinDistance(str1: string, str2: string): number {
const m = str1.length
const n = str2.length
const dp: number[][] = Array(m + 1).fill(null).map(() => Array(n + 1).fill(0))
for (let i = 0; i <= m; i++) dp[i][0] = i
for (let j = 0; j <= n; j++) dp[0][j] = j
for (let i = 1; i <= m; i++) {
for (let j = 1; j <= n; j++) {
if (str1[i - 1] === str2[j - 1]) {
dp[i][j] = dp[i - 1][j - 1]
} else {
dp[i][j] = Math.min(
dp[i - 1][j] + 1, // deletion
dp[i][j - 1] + 1, // insertion
dp[i - 1][j - 1] + 1 // substitution
)
}
}
}
return dp[m][n]
}
/**
* Merge confidence scores (weighted average favoring higher confidence)
*/
private mergeConfidence(existing: number, incoming: number): number {
// Weight higher confidence more heavily
const weights = existing > incoming ? [0.6, 0.4] : [0.4, 0.6]
return existing * weights[0] + incoming * weights[1]
}
/**
* Merge metadata fields intelligently
*/
private mergeMetadataFields(
existing: Record<string, any>,
incoming: Record<string, any>
): Record<string, any> {
const merged: Record<string, any> = {}
// Merge arrays
const arrayFields = ['concepts', 'tags', 'categories']
for (const field of arrayFields) {
if (existing[field] || incoming[field]) {
const combined = [
...(existing[field] || []),
...(incoming[field] || [])
]
// Deduplicate
merged[field] = [...new Set(combined)]
}
}
// Prefer longer descriptions
if (existing.description || incoming.description) {
merged.description = (existing.description || '').length > (incoming.description || '').length
? existing.description
: incoming.description
}
return merged
}
}

View file

@ -1,269 +0,0 @@
/**
* InstancePool - Shared instance management for memory efficiency
*
* Production-grade instance pooling to prevent memory leaks during imports.
* Critical for scaling to billions of entities.
*
* Problem: Creating new NLP/Extractor instances in loops memory leak
* Solution: Reuse shared instances across entire import session
*
* Memory savings:
* - Without pooling: 100K rows × 50MB per instance = 5TB RAM (OOM!)
* - With pooling: 50MB total (shared across all rows)
*/
import { Brainy } from '../brainy.js'
import { NaturalLanguageProcessor } from '../neural/naturalLanguageProcessor.js'
import { NeuralEntityExtractor } from '../neural/entityExtractor.js'
/**
* InstancePool - Manages shared instances for memory efficiency
*
* Lifecycle:
* 1. Create pool at import start
* 2. Reuse instances across all rows
* 3. Pool is garbage collected when import completes
*
* Thread safety: Not thread-safe (single import session per pool)
*/
export class InstancePool {
private brain: Brainy
// Shared instances (created lazily)
private nlpInstance: NaturalLanguageProcessor | null = null
private extractorInstance: NeuralEntityExtractor | null = null
// Initialization state
private nlpInitialized = false
private initializationPromise: Promise<void> | null = null
// Statistics
private stats = {
nlpReuses: 0,
extractorReuses: 0,
creationTime: 0
}
constructor(brain: Brainy) {
this.brain = brain
}
/**
* Get shared NaturalLanguageProcessor instance
*
* Lazy initialization - created on first access
* All subsequent calls return same instance
*
* @returns Shared NLP instance
*/
async getNLP(): Promise<NaturalLanguageProcessor> {
if (!this.nlpInstance) {
const startTime = Date.now()
this.nlpInstance = new NaturalLanguageProcessor(this.brain)
this.stats.creationTime += Date.now() - startTime
}
// Ensure initialized before returning
if (!this.nlpInitialized) {
await this.ensureNLPInitialized()
}
this.stats.nlpReuses++
return this.nlpInstance
}
/**
* Get shared NeuralEntityExtractor instance
*
* Lazy initialization - created on first access
* All subsequent calls return same instance
*
* @returns Shared extractor instance
*/
getExtractor(): NeuralEntityExtractor {
if (!this.extractorInstance) {
const startTime = Date.now()
this.extractorInstance = new NeuralEntityExtractor(this.brain)
this.stats.creationTime += Date.now() - startTime
}
this.stats.extractorReuses++
return this.extractorInstance
}
/**
* Get shared NLP instance (synchronous, may return uninitialized)
*
* Use when you need NLP synchronously and will handle initialization yourself.
* Prefer getNLP() for async code.
*
* @returns Shared NLP instance (possibly uninitialized)
*/
getNLPSync(): NaturalLanguageProcessor {
if (!this.nlpInstance) {
this.nlpInstance = new NaturalLanguageProcessor(this.brain)
}
this.stats.nlpReuses++
return this.nlpInstance
}
/**
* Initialize all instances upfront
*
* Call at start of import to avoid lazy initialization overhead
* during processing. Improves predictability and first-row performance.
*
* @returns Promise that resolves when all instances are ready
*/
async init(): Promise<void> {
// Prevent duplicate initialization
if (this.initializationPromise) {
return this.initializationPromise
}
this.initializationPromise = this.initializeInternal()
return this.initializationPromise
}
/**
* Internal initialization implementation
*/
private async initializeInternal(): Promise<void> {
const startTime = Date.now()
// Create instances
if (!this.nlpInstance) {
this.nlpInstance = new NaturalLanguageProcessor(this.brain)
}
if (!this.extractorInstance) {
this.extractorInstance = new NeuralEntityExtractor(this.brain)
}
// Initialize NLP (loads pattern library)
await this.ensureNLPInitialized()
this.stats.creationTime = Date.now() - startTime
}
/**
* Ensure NLP is initialized (loads 220 patterns)
*
* Handles concurrent initialization requests safely
*/
private async ensureNLPInitialized(): Promise<void> {
if (this.nlpInitialized) {
return
}
if (!this.nlpInstance) {
throw new Error('NLP instance not created yet')
}
await this.nlpInstance.init()
this.nlpInitialized = true
}
/**
* Check if instances are initialized
*
* @returns True if NLP is initialized and ready to use
*/
isInitialized(): boolean {
return this.nlpInitialized && this.nlpInstance !== null
}
/**
* Get pool statistics
*
* Useful for performance monitoring and memory leak detection
*
* @returns Statistics about instance reuse
*/
getStats() {
return {
...this.stats,
nlpCreated: this.nlpInstance !== null,
extractorCreated: this.extractorInstance !== null,
initialized: this.isInitialized(),
// Memory savings estimate
memorySaved: this.calculateMemorySaved()
}
}
/**
* Calculate estimated memory saved by pooling
*
* Assumes ~50MB per NLP instance, ~10MB per extractor instance
*
* @returns Estimated memory saved in bytes
*/
private calculateMemorySaved(): number {
const nlpSize = 50 * 1024 * 1024 // 50MB per instance
const extractorSize = 10 * 1024 * 1024 // 10MB per instance
// Without pooling: size × reuses
// With pooling: size × 1
// Saved: size × (reuses - 1)
const nlpSaved = nlpSize * Math.max(0, this.stats.nlpReuses - 1)
const extractorSaved = extractorSize * Math.max(0, this.stats.extractorReuses - 1)
return nlpSaved + extractorSaved
}
/**
* Reset statistics (useful for testing)
*/
resetStats(): void {
this.stats = {
nlpReuses: 0,
extractorReuses: 0,
creationTime: 0
}
}
/**
* Get string representation (for debugging)
*/
toString(): string {
const stats = this.getStats()
return `InstancePool(nlp=${stats.nlpCreated}, extractor=${stats.extractorCreated}, initialized=${stats.initialized}, nlpReuses=${stats.nlpReuses}, extractorReuses=${stats.extractorReuses})`
}
/**
* Cleanup method (for explicit resource management)
*
* Note: Usually not needed - pool is garbage collected when import completes.
* Use only if you need explicit cleanup for some reason.
*/
cleanup(): void {
// Clear references to allow garbage collection
this.nlpInstance = null
this.extractorInstance = null
this.nlpInitialized = false
this.initializationPromise = null
}
}
/**
* Create a new instance pool
*
* Convenience factory function
*
* @param brain Brainy instance
* @param autoInit Whether to initialize instances immediately
* @returns Instance pool
*/
export async function createInstancePool(
brain: Brainy,
autoInit = true
): Promise<InstancePool> {
const pool = new InstancePool(brain)
if (autoInit) {
await pool.init()
}
return pool
}

View file

@ -1,38 +0,0 @@
/**
* Unified Import System
*
* Single entry point for importing any file format into Brainy with:
* - Auto-detection of formats
* - Dual storage (VFS + Knowledge Graph)
* - Shared entities across imports (deduplication)
* - Simple, powerful API
*/
export { ImportCoordinator } from './ImportCoordinator.js'
export { FormatDetector, SupportedFormat, DetectionResult } from './FormatDetector.js'
export { EntityDeduplicator } from './EntityDeduplicator.js'
export { BackgroundDeduplicator } from './BackgroundDeduplicator.js'
export { ImportHistory } from './ImportHistory.js'
export type {
ImportSource,
ImportOptions,
ImportProgress,
ImportResult
} from './ImportCoordinator.js'
export type {
EntityCandidate,
DuplicateMatch,
EntityDeduplicationOptions,
MergeResult
} from './EntityDeduplicator.js'
export type {
DeduplicationStats
} from './BackgroundDeduplicator.js'
export type {
ImportHistoryEntry,
RollbackResult
} from './ImportHistory.js'