chore: recovery checkpoint - v3.0 API successfully recovered
CRITICAL CHECKPOINT - DO NOT PUSH TO GITHUB Recovery Status: - Successfully recovered brainy.ts from compiled JavaScript - All core v3.0 API methods functional (add, get, update, delete, relate, find, etc.) - Neural subsystem intact (562KB embedded patterns, NLP working) - Augmentation pipeline operational (20+ augmentations) - HNSW clustering system complete - Triple Intelligence compiled (needs constructor fix) - Test suite validates functionality Changes preserved: - 898 files with changes from last 3 days - 144,475 insertions - All augmentation improvements - All test coverage enhancements - Complete v3.0 feature set This is a LOCAL checkpoint only - contains recovered work after corruption incident. Created backup in .backups/brainy-full-20250910-151314.tar.gz Branch: recovery-checkpoint-20250910-151433 Date: Wed Sep 10 03:18:04 PM PDT 2025
This commit is contained in:
parent
f65455fb22
commit
8ff382ca3b
895 changed files with 143654 additions and 28268 deletions
|
|
@ -0,0 +1,73 @@
|
|||
/**
|
||||
* Worker process for embeddings - Workaround for transformers.js memory leak
|
||||
*
|
||||
* This worker can be killed and restarted to release memory completely.
|
||||
* Based on 2024 research: dispose() doesn't fully free memory in transformers.js
|
||||
*/
|
||||
import { TransformerEmbedding } from '../utils/embedding.js';
|
||||
import { parentPort } from 'worker_threads';
|
||||
import { getModelPrecision } from '../config/modelPrecisionManager.js';
|
||||
let model = null;
|
||||
let requestCount = 0;
|
||||
const MAX_REQUESTS = 100; // Restart worker after 100 requests to prevent memory leak
|
||||
async function initModel() {
|
||||
if (!model) {
|
||||
model = new TransformerEmbedding({
|
||||
verbose: false,
|
||||
precision: getModelPrecision(), // Use centrally managed precision
|
||||
localFilesOnly: process.env.BRAINY_ALLOW_REMOTE_MODELS !== 'true'
|
||||
});
|
||||
await model.init();
|
||||
console.log('🔧 Worker: Model initialized');
|
||||
}
|
||||
}
|
||||
if (parentPort) {
|
||||
parentPort.on('message', async (message) => {
|
||||
try {
|
||||
const { id, type, data } = message;
|
||||
switch (type) {
|
||||
case 'embed':
|
||||
await initModel();
|
||||
const embeddings = await model.embed(data);
|
||||
parentPort.postMessage({ id, success: true, result: embeddings });
|
||||
requestCount++;
|
||||
// Proactively restart worker to prevent memory leak
|
||||
if (requestCount >= MAX_REQUESTS) {
|
||||
console.log(`🔄 Worker: Restarting after ${requestCount} requests (memory leak prevention)`);
|
||||
process.exit(0); // Parent will restart us
|
||||
}
|
||||
break;
|
||||
case 'dispose':
|
||||
// SingletonModelManager persists - just acknowledge
|
||||
console.log('ℹ️ Worker: Singleton model persists');
|
||||
parentPort.postMessage({ id, success: true });
|
||||
break;
|
||||
case 'restart':
|
||||
// Force restart to clear memory
|
||||
console.log('🔄 Worker: Force restart requested');
|
||||
process.exit(0);
|
||||
break;
|
||||
default:
|
||||
parentPort.postMessage({
|
||||
id,
|
||||
success: false,
|
||||
error: `Unknown message type: ${type}`
|
||||
});
|
||||
}
|
||||
}
|
||||
catch (error) {
|
||||
parentPort.postMessage({
|
||||
id: message.id,
|
||||
success: false,
|
||||
error: error instanceof Error ? error.message : String(error)
|
||||
});
|
||||
}
|
||||
});
|
||||
console.log('🚀 Embedding worker started');
|
||||
parentPort.postMessage({ type: 'ready' });
|
||||
}
|
||||
else {
|
||||
console.error('❌ Worker: parentPort is null, cannot communicate with main thread');
|
||||
process.exit(1);
|
||||
}
|
||||
//# sourceMappingURL=worker-embedding.js.map
|
||||
Loading…
Add table
Add a link
Reference in a new issue