feat: implement clean embedding architecture with Q8/FP32 precision control

- Unified embedding system with single EmbeddingManager
- Q8 model support with 75% smaller footprint (23MB vs 90MB)
- Intelligent precision auto-selection based on environment
- Clean cached embeddings with TTL and memory management
- Zero-config setup with smart defaults
- Complete storage structure documentation
- Removed legacy worker and hybrid managers
- Streamlined model configuration and precision management
This commit is contained in:
David Snelling 2025-09-02 10:00:52 -07:00
parent 3227ad907c
commit 184d5dcf34
23 changed files with 1575 additions and 1369 deletions

View file

@ -1678,21 +1678,10 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
}
}
// CRITICAL: Initialize universal memory manager ONLY for default embedding function
// This preserves custom embedding functions (like test mocks)
if (typeof this.embeddingFunction === 'function' && this.embeddingFunction === defaultEmbeddingFunction) {
try {
const { universalMemoryManager } = await import('./embeddings/universal-memory-manager.js')
this.embeddingFunction = await universalMemoryManager.getEmbeddingFunction()
console.log('✅ UNIVERSAL: Memory-safe embedding system initialized')
} catch (error) {
console.error('🚨 CRITICAL: Universal memory manager initialization failed!')
console.error('Falling back to standard embedding with potential memory issues.')
console.warn('Consider reducing usage or restarting process periodically.')
// Continue with default function - better than crashing
}
} else if (this.embeddingFunction !== defaultEmbeddingFunction) {
console.log('✅ CUSTOM: Using custom embedding function (test or production override)')
// The embedding function is already set (either custom or default)
// EmbeddingManager handles all initialization internally
if (this.embeddingFunction !== defaultEmbeddingFunction) {
console.log('✅ Using custom embedding function')
}
try {