refactor: remove augmentation system and semantic type matching

Remove the entire augmentation pipeline infrastructure (52 files,
~15,000 lines) and the semantic type matching system. These were
unused middleware layers adding complexity without value.

What was removed:
- src/augmentations/ directory (all augmentation implementations)
- src/augmentationManager.ts (pipeline orchestrator)
- src/types/augmentations.ts, src/types/pipelineTypes.ts
- src/shared/default-augmentations.ts
- Semantic type suggestion (BrainyTypes.suggestNoun/suggestVerb)
- src/utils/typeMatching/ (embedding-based type matcher)

What was preserved by relocating:
- Import handlers (CSV, PDF, Excel) -> src/importers/handlers/
- NeuralImportAugmentation -> src/cortex/neuralImportAugmentation.ts
- Type matching utilities -> heuristic inference in consumers

What was simplified:
- brainy.ts: operations call storage directly (no execute() wrapper)
- IntegrationBase: standalone class (no BaseAugmentation parent)
- BrainyTypes: validation-only (nouns, verbs, isValid*, get*)
- Pipeline: direct execution (no augmentation interception)
- index.ts: removed TypeSuggestion, suggestType exports
- package.json: removed stale types/augmentations export

Build passes, 1176 tests pass, 0 failures.
This commit is contained in:
David Snelling 2026-02-01 10:48:56 -08:00
parent ac7a1f772c
commit d1db3510be
97 changed files with 349 additions and 19705 deletions

View file

@ -60,8 +60,7 @@ async function runBrainyBenchmark() {
const brain = new Brainy({
storage: { type: 'memory' },
augmentations: {},
model: {
model: {
type: 'fast', // Using real transformer model
precision: 'Q8' // Quantized for speed
}

View file

@ -158,12 +158,6 @@ async function runBrainyBenchmark() {
const brain = new Brainy({
storage: { type: 'memory' },
augmentations: {
cache: false,
metrics: false,
display: false,
index: false
},
embedder: mockEmbedder,
warmup: false
})

View file

@ -14,15 +14,8 @@ async function runBenchmark() {
console.log('🧠 Brainy v3 Performance Benchmark')
console.log('═'.repeat(60))
// Disable all augmentations for raw performance
const brain = new Brainy({
storage: { type: 'memory' },
augmentations: {
cache: false,
metrics: false,
display: false,
index: false
},
embedder: mockEmbedder,
warmup: false
})

View file

@ -16,7 +16,6 @@ async function benchmark() {
const brain = new Brainy({
storage: { type: 'memory' },
augmentations: {},
embedder: mockEmbedder
})

View file

@ -22,7 +22,7 @@ async function runV2Benchmark() {
const brain = new Brainy({
storage: { type: 'memory' },
embeddingFunction: mockEmbedder,
augmentations: false // Disable for raw performance
// Raw performance test
})
await brain.init()
@ -111,7 +111,6 @@ async function runV3Benchmark() {
const brain = new Brainy({
storage: { type: 'memory' },
augmentations: {}, // Minimal augmentations
embedder: mockEmbedder
})
@ -216,7 +215,6 @@ async function runScaleTest() {
const brain = new Brainy({
storage: { type: 'memory' },
augmentations: {},
embedder: mockEmbedder
})

View file

@ -77,30 +77,21 @@ async function profilePerformance() {
// Initialize Brainy with different configurations
console.log('Initializing Brainy configurations...')
// 1. Minimal config (no augmentations)
// 1. Minimal config
const minimalBrain = new Brainy({
storage: new MemoryStorage(),
augmentations: false // Disable all augmentations
storage: new MemoryStorage()
})
await minimalBrain.init()
// 2. Default config (with augmentations)
// 2. Default config
const defaultBrain = new Brainy({
storage: new MemoryStorage()
})
await defaultBrain.init()
// 3. Full augmentations config
// 3. Full config
const fullBrain = new Brainy({
storage: new MemoryStorage(),
augmentations: {
batchProcessing: { enabled: true, maxBatchSize: 1000 },
connectionPool: { enabled: true },
cache: { enabled: true },
index: { enabled: true },
entityRegistry: { enabled: true },
monitoring: { enabled: true }
}
storage: new MemoryStorage()
})
await fullBrain.init()
@ -188,11 +179,7 @@ async function profilePerformance() {
verbCounter++
}, 100)
console.log('\n🔧 Testing Augmentation Overhead\n')
// Test individual augmentations
const augmentations = defaultBrain.augmentations.getAugmentationTypes()
console.log(`Active augmentations: ${augmentations.join(', ')}\n`)
console.log('\n🔧 Testing Operation Overhead\n')
// Measure raw storage performance
const storage = new MemoryStorage()
@ -261,8 +248,8 @@ async function profilePerformance() {
console.log('Overhead breakdown:')
console.log(` Base operation: ${minimalPerf.perOpMs.toFixed(2)}ms`)
console.log(` Default augmentations: +${(defaultPerf.perOpMs - minimalPerf.perOpMs).toFixed(2)}ms`)
console.log(` Full augmentations: +${(fullPerf.perOpMs - defaultPerf.perOpMs).toFixed(2)}ms`)
console.log(` Default config: +${(defaultPerf.perOpMs - minimalPerf.perOpMs).toFixed(2)}ms`)
console.log(` Full config: +${(fullPerf.perOpMs - defaultPerf.perOpMs).toFixed(2)}ms`)
console.log(` Embedding generation: ${embedPerf.perOpMs.toFixed(2)}ms`)
console.log(` Without embeddings: ${vectorPerf.perOpMs.toFixed(2)}ms`)
@ -290,7 +277,7 @@ async function profilePerformance() {
console.log(' 1. Fake/stub operations that returned immediately')
console.log(' 2. No actual embedding generation')
console.log(' 3. No real storage operations')
console.log(' 4. No augmentation processing')
console.log(' 4. Direct operation calls')
console.log('\n With real implementations:')
console.log(` - Raw storage: ${profiler.results['Raw storage.saveNoun']?.opsPerSec || 'N/A'} ops/sec`)
console.log(` - With embeddings: ${defaultPerf.opsPerSec} ops/sec`)

View file

@ -17,7 +17,6 @@ async function benchmarkV2() {
const brain = new Brainy({
storage: { type: 'memory' },
augmentations: false,
embeddingFunction: async () => new Array(384).fill(0).map(() => Math.random())
})
await brain.init()
@ -92,7 +91,6 @@ async function benchmarkV3() {
const brain = new Brainy({
storage: { type: 'memory' },
augmentations: {},
warmup: false,
embedder: async () => new Array(384).fill(0).map(() => Math.random())
})

View file

@ -13,7 +13,6 @@ async function testV3() {
const brain = new Brainy({
storage: { type: 'memory' },
augmentations: {},
warmup: false,
embedder: async () => new Array(384).fill(0).map(() => Math.random())
})

View file

@ -37,7 +37,7 @@ async function quickPerf() {
const brain = new Brainy({
storage: new MemoryStorage(),
embeddingFunction: mockEmbed,
augmentations: false // Disable augmentations
// Minimal config
})
await brain.init()
@ -55,12 +55,11 @@ async function quickPerf() {
const brainyOps = Math.round(1000 / ((end2 - start2) / 1000))
console.log(` ✅ Brainy: ${brainyOps.toLocaleString()} ops/sec\n`)
// Test 3: With augmentations
console.log('3⃣ Brainy with Augmentations')
// Test 3: Default config
console.log('3⃣ Brainy with Default Config')
const brain2 = new Brainy({
storage: new MemoryStorage(),
embeddingFunction: mockEmbed
// Default augmentations enabled
})
await brain2.init()
@ -74,13 +73,12 @@ async function quickPerf() {
}
const end3 = performance.now()
const augOps = Math.round(1000 / ((end3 - start3) / 1000))
console.log(`With Augmentations: ${augOps.toLocaleString()} ops/sec\n`)
console.log(`Default Config: ${augOps.toLocaleString()} ops/sec\n`)
// Test 4: Real embeddings (the killer)
console.log('4⃣ With Real Embeddings (10 samples)')
const brain3 = new Brainy({
storage: new MemoryStorage(),
augmentations: false
// Uses real embedding function
})
await brain3.init()
@ -119,15 +117,10 @@ async function quickPerf() {
console.log(' ❌ Embeddings are the primary bottleneck')
console.log(' Each embedding takes ~' + Math.round((end4 - start4) / 10) + 'ms')
}
if (augOverhead > 50) {
console.log(' ⚠️ Augmentations add significant overhead')
}
console.log('\n🎯 The 500,000 ops/sec claim was achievable with:')
console.log(' 1. Pre-computed vectors (no embedding)')
console.log(' 2. Minimal augmentations')
console.log(' 3. In-memory storage')
console.log(' 4. Batch operations')
console.log(' 2. In-memory storage')
console.log(' 3. Batch operations')
await brain.close()
await brain2.close()

View file

@ -69,10 +69,6 @@ async function quickVerify() {
console.log(`❌ Verb storage: Expected 1 verb, got ${stats.totalVerbs}`)
}
// Check augmentations
const augTypes = brain.augmentations.getAugmentationTypes()
console.log(`✅ Active augmentations: ${augTypes.join(', ')}`)
// Close
await brain.close()
console.log('\n✅ All tests passed - implementations are REAL!')

View file

@ -33,34 +33,14 @@ console.log(`🔧 Mode: ${CURRENT_SCALE}\n`)
async function runScaleTest() {
const startTime = Date.now()
// Initialize Brainy with real augmentations
// Initialize Brainy
const brain = new Brainy({
storage: new MemoryStorage(),
augmentations: {
// These should all be REAL implementations now
batchProcessing: {
enabled: true,
maxBatchSize: BATCH_SIZE,
adaptiveBatching: true
},
connectionPool: {
enabled: true,
maxConnections: 50,
minConnections: 5
},
cache: {
enabled: true,
maxSize: 10000
},
index: {
enabled: true
}
}
storage: new MemoryStorage()
})
await brain.init()
console.log('✅ Brainy initialized with real augmentations\n')
console.log('✅ Brainy initialized\n')
// Test 1: Batch Insert Performance
console.log('📝 Test 1: Batch Insert Performance')
@ -160,35 +140,6 @@ async function runScaleTest() {
console.log(`✅ Created ${verbCount.toLocaleString()} relationships in ${verbTime}ms`)
console.log(`📈 Rate: ${verbRate.toLocaleString()} relationships/second\n`)
// Test 4: Verify Augmentations Are Real
console.log('🔧 Test 4: Verify Real Implementations')
// Check BatchProcessing stats
const batchAug = brain.augmentations.getAugmentation('BatchProcessing')
if (batchAug && typeof batchAug.getStats === 'function') {
const stats = batchAug.getStats()
console.log(` BatchProcessing: ${stats.batchesProcessed} batches processed`)
console.log(` Average batch size: ${Math.round(stats.averageBatchSize)}`)
console.log(` Throughput: ${stats.throughputPerSecond} ops/sec`)
}
// Check ConnectionPool stats
const poolAug = brain.augmentations.getAugmentation('ConnectionPool')
if (poolAug && typeof poolAug.getStats === 'function') {
const stats = poolAug.getStats()
console.log(` ConnectionPool: ${stats.totalConnections} connections`)
console.log(` Pool utilization: ${stats.poolUtilization}`)
console.log(` Total requests: ${stats.totalRequests}`)
}
// Check Cache stats
const cacheAug = brain.augmentations.getAugmentation('cache')
if (cacheAug && typeof cacheAug.getStats === 'function') {
const stats = cacheAug.getStats()
console.log(` Cache: ${stats.hits} hits, ${stats.misses} misses`)
console.log(` Hit rate: ${Math.round((stats.hits / (stats.hits + stats.misses)) * 100)}%`)
}
// Final Statistics
const totalTime = Date.now() - startTime
const stats = brain.getStats()