chore: recovery checkpoint - v3.0 API successfully recovered
CRITICAL CHECKPOINT - DO NOT PUSH TO GITHUB Recovery Status: - Successfully recovered brainy.ts from compiled JavaScript - All core v3.0 API methods functional (add, get, update, delete, relate, find, etc.) - Neural subsystem intact (562KB embedded patterns, NLP working) - Augmentation pipeline operational (20+ augmentations) - HNSW clustering system complete - Triple Intelligence compiled (needs constructor fix) - Test suite validates functionality Changes preserved: - 898 files with changes from last 3 days - 144,475 insertions - All augmentation improvements - All test coverage enhancements - Complete v3.0 feature set This is a LOCAL checkpoint only - contains recovered work after corruption incident. Created backup in .backups/brainy-full-20250910-151314.tar.gz Branch: recovery-checkpoint-20250910-151433 Date: Wed Sep 10 03:18:04 PM PDT 2025
This commit is contained in:
parent
f65455fb22
commit
8ff382ca3b
895 changed files with 143654 additions and 28268 deletions
40
.recovery-workspace/dist-backup-20250910-141917/embeddings/CachedEmbeddings.d.ts
vendored
Normal file
40
.recovery-workspace/dist-backup-20250910-141917/embeddings/CachedEmbeddings.d.ts
vendored
Normal file
|
|
@ -0,0 +1,40 @@
|
|||
/**
|
||||
* Cached Embeddings - Performance Optimization Layer
|
||||
*
|
||||
* Provides pre-computed embeddings for common terms to avoid
|
||||
* unnecessary model calls. Falls back to EmbeddingManager for
|
||||
* unknown terms.
|
||||
*
|
||||
* This is purely a performance optimization - it doesn't affect
|
||||
* the consistency or accuracy of embeddings.
|
||||
*/
|
||||
import { Vector } from '../coreTypes.js';
|
||||
/**
|
||||
* Cached Embeddings with fallback to EmbeddingManager
|
||||
*/
|
||||
export declare class CachedEmbeddings {
|
||||
private stats;
|
||||
/**
|
||||
* Generate embedding with caching
|
||||
*/
|
||||
embed(text: string | string[]): Promise<Vector | Vector[]>;
|
||||
/**
|
||||
* Embed single text with cache lookup
|
||||
*/
|
||||
private embedSingle;
|
||||
/**
|
||||
* Get cache statistics
|
||||
*/
|
||||
getStats(): {
|
||||
totalEmbeddings: number;
|
||||
cacheHitRate: number;
|
||||
cacheHits: number;
|
||||
simpleComputes: number;
|
||||
modelCalls: number;
|
||||
};
|
||||
/**
|
||||
* Add custom pre-computed embeddings
|
||||
*/
|
||||
addPrecomputed(term: string, embedding: Vector): void;
|
||||
}
|
||||
export declare const cachedEmbeddings: CachedEmbeddings;
|
||||
|
|
@ -0,0 +1,146 @@
|
|||
/**
|
||||
* Cached Embeddings - Performance Optimization Layer
|
||||
*
|
||||
* Provides pre-computed embeddings for common terms to avoid
|
||||
* unnecessary model calls. Falls back to EmbeddingManager for
|
||||
* unknown terms.
|
||||
*
|
||||
* This is purely a performance optimization - it doesn't affect
|
||||
* the consistency or accuracy of embeddings.
|
||||
*/
|
||||
import { embeddingManager } from './EmbeddingManager.js';
|
||||
// Pre-computed embeddings for top common terms
|
||||
// In production, this could be loaded from a file or expanded significantly
|
||||
const PRECOMPUTED_EMBEDDINGS = {
|
||||
// Programming languages
|
||||
'javascript': new Array(384).fill(0).map((_, i) => Math.sin(i * 0.1)),
|
||||
'python': new Array(384).fill(0).map((_, i) => Math.cos(i * 0.1)),
|
||||
'typescript': new Array(384).fill(0).map((_, i) => Math.sin(i * 0.15)),
|
||||
'java': new Array(384).fill(0).map((_, i) => Math.cos(i * 0.15)),
|
||||
'rust': new Array(384).fill(0).map((_, i) => Math.sin(i * 0.2)),
|
||||
'go': new Array(384).fill(0).map((_, i) => Math.cos(i * 0.2)),
|
||||
'c++': new Array(384).fill(0).map((_, i) => Math.sin(i * 0.22)),
|
||||
'c#': new Array(384).fill(0).map((_, i) => Math.cos(i * 0.22)),
|
||||
// Web frameworks
|
||||
'react': new Array(384).fill(0).map((_, i) => Math.sin(i * 0.25)),
|
||||
'vue': new Array(384).fill(0).map((_, i) => Math.cos(i * 0.25)),
|
||||
'angular': new Array(384).fill(0).map((_, i) => Math.sin(i * 0.3)),
|
||||
'svelte': new Array(384).fill(0).map((_, i) => Math.cos(i * 0.3)),
|
||||
'nextjs': new Array(384).fill(0).map((_, i) => Math.sin(i * 0.32)),
|
||||
'nuxt': new Array(384).fill(0).map((_, i) => Math.cos(i * 0.32)),
|
||||
// Databases
|
||||
'postgresql': new Array(384).fill(0).map((_, i) => Math.sin(i * 0.35)),
|
||||
'mysql': new Array(384).fill(0).map((_, i) => Math.cos(i * 0.35)),
|
||||
'mongodb': new Array(384).fill(0).map((_, i) => Math.sin(i * 0.4)),
|
||||
'redis': new Array(384).fill(0).map((_, i) => Math.cos(i * 0.4)),
|
||||
'elasticsearch': new Array(384).fill(0).map((_, i) => Math.sin(i * 0.42)),
|
||||
// Common tech terms
|
||||
'database': new Array(384).fill(0).map((_, i) => Math.sin(i * 0.45)),
|
||||
'api': new Array(384).fill(0).map((_, i) => Math.cos(i * 0.45)),
|
||||
'server': new Array(384).fill(0).map((_, i) => Math.sin(i * 0.5)),
|
||||
'client': new Array(384).fill(0).map((_, i) => Math.cos(i * 0.5)),
|
||||
'frontend': new Array(384).fill(0).map((_, i) => Math.sin(i * 0.55)),
|
||||
'backend': new Array(384).fill(0).map((_, i) => Math.cos(i * 0.55)),
|
||||
'fullstack': new Array(384).fill(0).map((_, i) => Math.sin(i * 0.57)),
|
||||
'devops': new Array(384).fill(0).map((_, i) => Math.cos(i * 0.57)),
|
||||
'cloud': new Array(384).fill(0).map((_, i) => Math.sin(i * 0.6)),
|
||||
'docker': new Array(384).fill(0).map((_, i) => Math.cos(i * 0.6)),
|
||||
'kubernetes': new Array(384).fill(0).map((_, i) => Math.sin(i * 0.62)),
|
||||
'microservices': new Array(384).fill(0).map((_, i) => Math.cos(i * 0.62)),
|
||||
};
|
||||
/**
|
||||
* Simple character n-gram based embedding for short text
|
||||
* This is much faster than using the model for simple terms
|
||||
*/
|
||||
function computeSimpleEmbedding(text) {
|
||||
const normalized = text.toLowerCase().trim();
|
||||
const vector = new Array(384).fill(0);
|
||||
// Character trigrams for simple semantic similarity
|
||||
for (let i = 0; i < normalized.length - 2; i++) {
|
||||
const trigram = normalized.slice(i, i + 3);
|
||||
const hash = trigram.charCodeAt(0) * 31 +
|
||||
trigram.charCodeAt(1) * 7 +
|
||||
trigram.charCodeAt(2);
|
||||
const index = Math.abs(hash) % 384;
|
||||
vector[index] += 1 / (normalized.length - 2);
|
||||
}
|
||||
// Normalize vector
|
||||
const magnitude = Math.sqrt(vector.reduce((sum, val) => sum + val * val, 0));
|
||||
if (magnitude > 0) {
|
||||
for (let i = 0; i < vector.length; i++) {
|
||||
vector[i] /= magnitude;
|
||||
}
|
||||
}
|
||||
return vector;
|
||||
}
|
||||
/**
|
||||
* Cached Embeddings with fallback to EmbeddingManager
|
||||
*/
|
||||
export class CachedEmbeddings {
|
||||
constructor() {
|
||||
this.stats = {
|
||||
cacheHits: 0,
|
||||
simpleComputes: 0,
|
||||
modelCalls: 0
|
||||
};
|
||||
}
|
||||
/**
|
||||
* Generate embedding with caching
|
||||
*/
|
||||
async embed(text) {
|
||||
if (Array.isArray(text)) {
|
||||
return Promise.all(text.map(t => this.embedSingle(t)));
|
||||
}
|
||||
return this.embedSingle(text);
|
||||
}
|
||||
/**
|
||||
* Embed single text with cache lookup
|
||||
*/
|
||||
async embedSingle(text) {
|
||||
const normalized = text.toLowerCase().trim();
|
||||
// 1. Check pre-computed cache (instant, zero cost)
|
||||
if (PRECOMPUTED_EMBEDDINGS[normalized]) {
|
||||
this.stats.cacheHits++;
|
||||
return PRECOMPUTED_EMBEDDINGS[normalized];
|
||||
}
|
||||
// 2. Check for partial matches in cache
|
||||
for (const [term, embedding] of Object.entries(PRECOMPUTED_EMBEDDINGS)) {
|
||||
if (normalized.includes(term) || term.includes(normalized)) {
|
||||
this.stats.cacheHits++;
|
||||
// Return slightly modified version to maintain uniqueness
|
||||
return embedding.map(v => v * 0.95);
|
||||
}
|
||||
}
|
||||
// 3. For short text, use simple embedding (fast, low cost)
|
||||
if (normalized.length < 50 && normalized.split(' ').length < 5) {
|
||||
this.stats.simpleComputes++;
|
||||
return computeSimpleEmbedding(normalized);
|
||||
}
|
||||
// 4. Fall back to EmbeddingManager for complex text
|
||||
this.stats.modelCalls++;
|
||||
return await embeddingManager.embed(text);
|
||||
}
|
||||
/**
|
||||
* Get cache statistics
|
||||
*/
|
||||
getStats() {
|
||||
return {
|
||||
...this.stats,
|
||||
totalEmbeddings: this.stats.cacheHits + this.stats.simpleComputes + this.stats.modelCalls,
|
||||
cacheHitRate: this.stats.cacheHits /
|
||||
(this.stats.cacheHits + this.stats.simpleComputes + this.stats.modelCalls) || 0
|
||||
};
|
||||
}
|
||||
/**
|
||||
* Add custom pre-computed embeddings
|
||||
*/
|
||||
addPrecomputed(term, embedding) {
|
||||
if (embedding.length !== 384) {
|
||||
throw new Error('Embedding must have 384 dimensions');
|
||||
}
|
||||
PRECOMPUTED_EMBEDDINGS[term.toLowerCase()] = embedding;
|
||||
}
|
||||
}
|
||||
// Export singleton instance
|
||||
export const cachedEmbeddings = new CachedEmbeddings();
|
||||
//# sourceMappingURL=CachedEmbeddings.js.map
|
||||
File diff suppressed because one or more lines are too long
106
.recovery-workspace/dist-backup-20250910-141917/embeddings/EmbeddingManager.d.ts
vendored
Normal file
106
.recovery-workspace/dist-backup-20250910-141917/embeddings/EmbeddingManager.d.ts
vendored
Normal file
|
|
@ -0,0 +1,106 @@
|
|||
/**
|
||||
* Unified Embedding Manager
|
||||
*
|
||||
* THE single source of truth for all embedding operations in Brainy.
|
||||
* Combines model management, precision configuration, and embedding generation
|
||||
* into one clean, maintainable class.
|
||||
*
|
||||
* Features:
|
||||
* - Singleton pattern ensures ONE model instance
|
||||
* - Automatic Q8 (default) or FP32 precision
|
||||
* - Model downloading and caching
|
||||
* - Thread-safe initialization
|
||||
* - Memory monitoring
|
||||
*
|
||||
* This replaces: SingletonModelManager, TransformerEmbedding, ModelPrecisionManager,
|
||||
* hybridModelManager, universalMemoryManager, and more.
|
||||
*/
|
||||
import { Vector, EmbeddingFunction } from '../coreTypes.js';
|
||||
export type ModelPrecision = 'q8' | 'fp32';
|
||||
interface EmbeddingStats {
|
||||
initialized: boolean;
|
||||
precision: ModelPrecision;
|
||||
modelName: string;
|
||||
embedCount: number;
|
||||
initTime: number | null;
|
||||
memoryMB: number | null;
|
||||
}
|
||||
/**
|
||||
* Unified Embedding Manager - Clean, simple, reliable
|
||||
*/
|
||||
export declare class EmbeddingManager {
|
||||
private model;
|
||||
private precision;
|
||||
private modelName;
|
||||
private initialized;
|
||||
private initTime;
|
||||
private embedCount;
|
||||
private locked;
|
||||
private constructor();
|
||||
/**
|
||||
* Get the singleton instance
|
||||
*/
|
||||
static getInstance(): EmbeddingManager;
|
||||
/**
|
||||
* Initialize the model (happens once)
|
||||
*/
|
||||
init(): Promise<void>;
|
||||
/**
|
||||
* Perform actual initialization
|
||||
*/
|
||||
private performInit;
|
||||
/**
|
||||
* Generate embeddings
|
||||
*/
|
||||
embed(text: string | string[]): Promise<Vector>;
|
||||
/**
|
||||
* Generate mock embeddings for unit tests
|
||||
*/
|
||||
private getMockEmbedding;
|
||||
/**
|
||||
* Get embedding function for compatibility
|
||||
*/
|
||||
getEmbeddingFunction(): EmbeddingFunction;
|
||||
/**
|
||||
* Determine model precision
|
||||
*/
|
||||
private determinePrecision;
|
||||
/**
|
||||
* Get models directory path
|
||||
*/
|
||||
private getModelsPath;
|
||||
/**
|
||||
* Get memory usage in MB
|
||||
*/
|
||||
private getMemoryUsage;
|
||||
/**
|
||||
* Get current statistics
|
||||
*/
|
||||
getStats(): EmbeddingStats;
|
||||
/**
|
||||
* Check if initialized
|
||||
*/
|
||||
isInitialized(): boolean;
|
||||
/**
|
||||
* Get current precision
|
||||
*/
|
||||
getPrecision(): ModelPrecision;
|
||||
/**
|
||||
* Validate precision matches expected
|
||||
*/
|
||||
validatePrecision(expected: ModelPrecision): void;
|
||||
}
|
||||
export declare const embeddingManager: EmbeddingManager;
|
||||
/**
|
||||
* Direct embed function
|
||||
*/
|
||||
export declare function embed(text: string | string[]): Promise<Vector>;
|
||||
/**
|
||||
* Get embedding function for compatibility
|
||||
*/
|
||||
export declare function getEmbeddingFunction(): EmbeddingFunction;
|
||||
/**
|
||||
* Get statistics
|
||||
*/
|
||||
export declare function getEmbeddingStats(): EmbeddingStats;
|
||||
export {};
|
||||
|
|
@ -0,0 +1,296 @@
|
|||
/**
|
||||
* Unified Embedding Manager
|
||||
*
|
||||
* THE single source of truth for all embedding operations in Brainy.
|
||||
* Combines model management, precision configuration, and embedding generation
|
||||
* into one clean, maintainable class.
|
||||
*
|
||||
* Features:
|
||||
* - Singleton pattern ensures ONE model instance
|
||||
* - Automatic Q8 (default) or FP32 precision
|
||||
* - Model downloading and caching
|
||||
* - Thread-safe initialization
|
||||
* - Memory monitoring
|
||||
*
|
||||
* This replaces: SingletonModelManager, TransformerEmbedding, ModelPrecisionManager,
|
||||
* hybridModelManager, universalMemoryManager, and more.
|
||||
*/
|
||||
import { pipeline, env } from '@huggingface/transformers';
|
||||
import { existsSync } from 'fs';
|
||||
import { join } from 'path';
|
||||
// Global state for true singleton across entire process
|
||||
let globalInstance = null;
|
||||
let globalInitPromise = null;
|
||||
/**
|
||||
* Unified Embedding Manager - Clean, simple, reliable
|
||||
*/
|
||||
export class EmbeddingManager {
|
||||
constructor() {
|
||||
this.model = null;
|
||||
this.modelName = 'Xenova/all-MiniLM-L6-v2';
|
||||
this.initialized = false;
|
||||
this.initTime = null;
|
||||
this.embedCount = 0;
|
||||
this.locked = false;
|
||||
// Determine precision - Q8 by default
|
||||
this.precision = this.determinePrecision();
|
||||
console.log(`🎯 EmbeddingManager: Using ${this.precision.toUpperCase()} precision`);
|
||||
}
|
||||
/**
|
||||
* Get the singleton instance
|
||||
*/
|
||||
static getInstance() {
|
||||
if (!globalInstance) {
|
||||
globalInstance = new EmbeddingManager();
|
||||
}
|
||||
return globalInstance;
|
||||
}
|
||||
/**
|
||||
* Initialize the model (happens once)
|
||||
*/
|
||||
async init() {
|
||||
// In unit test mode, skip real model initialization
|
||||
if (process.env.BRAINY_UNIT_TEST === 'true' || globalThis.__BRAINY_UNIT_TEST__) {
|
||||
if (!this.initialized) {
|
||||
this.initialized = true;
|
||||
this.initTime = 1; // Mock init time
|
||||
console.log('🧪 EmbeddingManager: Using mocked embeddings for unit tests');
|
||||
}
|
||||
return;
|
||||
}
|
||||
// Already initialized
|
||||
if (this.initialized && this.model) {
|
||||
return;
|
||||
}
|
||||
// Initialization in progress
|
||||
if (globalInitPromise) {
|
||||
await globalInitPromise;
|
||||
return;
|
||||
}
|
||||
// Start initialization
|
||||
globalInitPromise = this.performInit();
|
||||
try {
|
||||
await globalInitPromise;
|
||||
}
|
||||
finally {
|
||||
globalInitPromise = null;
|
||||
}
|
||||
}
|
||||
/**
|
||||
* Perform actual initialization
|
||||
*/
|
||||
async performInit() {
|
||||
const startTime = Date.now();
|
||||
console.log(`🚀 Initializing embedding model (${this.precision.toUpperCase()})...`);
|
||||
try {
|
||||
// Configure transformers.js environment
|
||||
const modelsPath = this.getModelsPath();
|
||||
env.cacheDir = modelsPath;
|
||||
env.allowLocalModels = true;
|
||||
env.useFSCache = true;
|
||||
// Check if models exist locally
|
||||
const modelPath = join(modelsPath, ...this.modelName.split('/'));
|
||||
const hasLocalModels = existsSync(modelPath);
|
||||
if (hasLocalModels) {
|
||||
console.log('✅ Using cached models from:', modelPath);
|
||||
}
|
||||
// Configure pipeline options for the selected precision
|
||||
const pipelineOptions = {
|
||||
cache_dir: modelsPath,
|
||||
local_files_only: false,
|
||||
// Specify precision
|
||||
dtype: this.precision,
|
||||
quantized: this.precision === 'q8',
|
||||
// Memory optimizations
|
||||
session_options: {
|
||||
enableCpuMemArena: false,
|
||||
enableMemPattern: false,
|
||||
interOpNumThreads: 1,
|
||||
intraOpNumThreads: 1,
|
||||
graphOptimizationLevel: 'disabled'
|
||||
}
|
||||
};
|
||||
// Load the model
|
||||
this.model = await pipeline('feature-extraction', this.modelName, pipelineOptions);
|
||||
// Lock precision after successful initialization
|
||||
this.locked = true;
|
||||
this.initialized = true;
|
||||
this.initTime = Date.now() - startTime;
|
||||
// Log success
|
||||
const memoryMB = this.getMemoryUsage();
|
||||
console.log(`✅ Model loaded in ${this.initTime}ms`);
|
||||
console.log(`📊 Precision: ${this.precision.toUpperCase()} | Memory: ${memoryMB}MB`);
|
||||
console.log(`🔒 Configuration locked`);
|
||||
}
|
||||
catch (error) {
|
||||
this.initialized = false;
|
||||
this.model = null;
|
||||
throw new Error(`Failed to initialize embedding model: ${error instanceof Error ? error.message : String(error)}`);
|
||||
}
|
||||
}
|
||||
/**
|
||||
* Generate embeddings
|
||||
*/
|
||||
async embed(text) {
|
||||
// Check for unit test environment - use mocks to prevent ONNX conflicts
|
||||
if (process.env.BRAINY_UNIT_TEST === 'true' || globalThis.__BRAINY_UNIT_TEST__) {
|
||||
return this.getMockEmbedding(text);
|
||||
}
|
||||
// Ensure initialized
|
||||
await this.init();
|
||||
if (!this.model) {
|
||||
throw new Error('Model not initialized');
|
||||
}
|
||||
// Handle array input
|
||||
const input = Array.isArray(text) ? text.join(' ') : text;
|
||||
// Generate embedding
|
||||
const output = await this.model(input, {
|
||||
pooling: 'mean',
|
||||
normalize: true
|
||||
});
|
||||
// Extract embedding vector
|
||||
const embedding = Array.from(output.data);
|
||||
// Validate dimensions
|
||||
if (embedding.length !== 384) {
|
||||
console.warn(`Unexpected embedding dimension: ${embedding.length}`);
|
||||
// Pad or truncate
|
||||
if (embedding.length < 384) {
|
||||
return [...embedding, ...new Array(384 - embedding.length).fill(0)];
|
||||
}
|
||||
else {
|
||||
return embedding.slice(0, 384);
|
||||
}
|
||||
}
|
||||
this.embedCount++;
|
||||
return embedding;
|
||||
}
|
||||
/**
|
||||
* Generate mock embeddings for unit tests
|
||||
*/
|
||||
getMockEmbedding(text) {
|
||||
// Use the same mock logic as setup-unit.ts for consistency
|
||||
const input = Array.isArray(text) ? text.join(' ') : text;
|
||||
const str = typeof input === 'string' ? input : JSON.stringify(input);
|
||||
const vector = new Array(384).fill(0);
|
||||
// Create semi-realistic embeddings based on text content
|
||||
for (let i = 0; i < Math.min(str.length, 384); i++) {
|
||||
vector[i] = (str.charCodeAt(i % str.length) % 256) / 256;
|
||||
}
|
||||
// Add position-based variation
|
||||
for (let i = 0; i < 384; i++) {
|
||||
vector[i] += Math.sin(i * 0.1 + str.length) * 0.1;
|
||||
}
|
||||
// Track mock embedding count
|
||||
this.embedCount++;
|
||||
return vector;
|
||||
}
|
||||
/**
|
||||
* Get embedding function for compatibility
|
||||
*/
|
||||
getEmbeddingFunction() {
|
||||
return async (data) => {
|
||||
return await this.embed(data);
|
||||
};
|
||||
}
|
||||
/**
|
||||
* Determine model precision
|
||||
*/
|
||||
determinePrecision() {
|
||||
// Check environment variable overrides
|
||||
if (process.env.BRAINY_MODEL_PRECISION === 'fp32') {
|
||||
return 'fp32';
|
||||
}
|
||||
if (process.env.BRAINY_MODEL_PRECISION === 'q8') {
|
||||
return 'q8';
|
||||
}
|
||||
if (process.env.BRAINY_FORCE_FP32 === 'true') {
|
||||
return 'fp32';
|
||||
}
|
||||
// Default to Q8 - optimal for most use cases
|
||||
return 'q8';
|
||||
}
|
||||
/**
|
||||
* Get models directory path
|
||||
*/
|
||||
getModelsPath() {
|
||||
// Check various possible locations
|
||||
const paths = [
|
||||
process.env.BRAINY_MODELS_PATH,
|
||||
'./models',
|
||||
join(process.cwd(), 'models'),
|
||||
join(process.env.HOME || '', '.brainy', 'models')
|
||||
];
|
||||
for (const path of paths) {
|
||||
if (path && existsSync(path)) {
|
||||
return path;
|
||||
}
|
||||
}
|
||||
// Default
|
||||
return join(process.cwd(), 'models');
|
||||
}
|
||||
/**
|
||||
* Get memory usage in MB
|
||||
*/
|
||||
getMemoryUsage() {
|
||||
if (typeof process !== 'undefined' && process.memoryUsage) {
|
||||
const usage = process.memoryUsage();
|
||||
return Math.round(usage.heapUsed / 1024 / 1024);
|
||||
}
|
||||
return null;
|
||||
}
|
||||
/**
|
||||
* Get current statistics
|
||||
*/
|
||||
getStats() {
|
||||
return {
|
||||
initialized: this.initialized,
|
||||
precision: this.precision,
|
||||
modelName: this.modelName,
|
||||
embedCount: this.embedCount,
|
||||
initTime: this.initTime,
|
||||
memoryMB: this.getMemoryUsage()
|
||||
};
|
||||
}
|
||||
/**
|
||||
* Check if initialized
|
||||
*/
|
||||
isInitialized() {
|
||||
return this.initialized;
|
||||
}
|
||||
/**
|
||||
* Get current precision
|
||||
*/
|
||||
getPrecision() {
|
||||
return this.precision;
|
||||
}
|
||||
/**
|
||||
* Validate precision matches expected
|
||||
*/
|
||||
validatePrecision(expected) {
|
||||
if (this.locked && expected !== this.precision) {
|
||||
throw new Error(`Precision mismatch! System using ${this.precision.toUpperCase()} ` +
|
||||
`but ${expected.toUpperCase()} was requested. Cannot mix precisions.`);
|
||||
}
|
||||
}
|
||||
}
|
||||
// Export singleton instance and convenience functions
|
||||
export const embeddingManager = EmbeddingManager.getInstance();
|
||||
/**
|
||||
* Direct embed function
|
||||
*/
|
||||
export async function embed(text) {
|
||||
return await embeddingManager.embed(text);
|
||||
}
|
||||
/**
|
||||
* Get embedding function for compatibility
|
||||
*/
|
||||
export function getEmbeddingFunction() {
|
||||
return embeddingManager.getEmbeddingFunction();
|
||||
}
|
||||
/**
|
||||
* Get statistics
|
||||
*/
|
||||
export function getEmbeddingStats() {
|
||||
return embeddingManager.getStats();
|
||||
}
|
||||
//# sourceMappingURL=EmbeddingManager.js.map
|
||||
File diff suppressed because one or more lines are too long
95
.recovery-workspace/dist-backup-20250910-141917/embeddings/SingletonModelManager.d.ts
vendored
Normal file
95
.recovery-workspace/dist-backup-20250910-141917/embeddings/SingletonModelManager.d.ts
vendored
Normal file
|
|
@ -0,0 +1,95 @@
|
|||
/**
|
||||
* Singleton Model Manager - THE ONLY SOURCE OF EMBEDDING MODELS
|
||||
*
|
||||
* This is the SINGLE, UNIFIED model initialization system that ensures:
|
||||
* - Only ONE model instance exists across the entire system
|
||||
* - Precision is configured once and locked
|
||||
* - All components share the same model
|
||||
* - No possibility of mixed precisions
|
||||
*
|
||||
* CRITICAL: This manager is used by EVERYTHING:
|
||||
* - Storage operations (add, update)
|
||||
* - Search operations (search, find)
|
||||
* - Public API (embed, cluster)
|
||||
* - Neural API (all neural.* methods)
|
||||
* - Internal operations (deduplication, indexing)
|
||||
*/
|
||||
import { TransformerEmbedding } from '../utils/embedding.js';
|
||||
import { EmbeddingFunction, Vector } from '../coreTypes.js';
|
||||
/**
|
||||
* Statistics for monitoring
|
||||
*/
|
||||
interface ModelStats {
|
||||
initialized: boolean;
|
||||
precision: string;
|
||||
initCount: number;
|
||||
embedCount: number;
|
||||
lastUsed: Date | null;
|
||||
memoryFootprint?: number;
|
||||
}
|
||||
/**
|
||||
* The ONE TRUE model manager
|
||||
*/
|
||||
export declare class SingletonModelManager {
|
||||
private static instance;
|
||||
private stats;
|
||||
private constructor();
|
||||
/**
|
||||
* Get the singleton instance
|
||||
*/
|
||||
static getInstance(): SingletonModelManager;
|
||||
/**
|
||||
* Get the model instance - creates if needed, reuses if exists
|
||||
* This is THE ONLY way to get a model in the entire system
|
||||
*/
|
||||
getModel(): Promise<TransformerEmbedding>;
|
||||
/**
|
||||
* Initialize the model - happens exactly once
|
||||
*/
|
||||
private initializeModel;
|
||||
/**
|
||||
* Get embedding function that uses the singleton model
|
||||
*/
|
||||
getEmbeddingFunction(): Promise<EmbeddingFunction>;
|
||||
/**
|
||||
* Direct embed method for convenience
|
||||
*/
|
||||
embed(data: string | string[]): Promise<Vector>;
|
||||
/**
|
||||
* Check if model is initialized
|
||||
*/
|
||||
isInitialized(): boolean;
|
||||
/**
|
||||
* Get current statistics
|
||||
*/
|
||||
getStats(): ModelStats;
|
||||
/**
|
||||
* Validate precision consistency
|
||||
* Throws error if attempting to use different precision
|
||||
*/
|
||||
validatePrecision(requestedPrecision?: string): void;
|
||||
/**
|
||||
* Force cleanup (for testing only)
|
||||
* WARNING: This will break consistency - use only in tests
|
||||
*/
|
||||
_testOnlyCleanup(): Promise<void>;
|
||||
}
|
||||
export declare const singletonModelManager: SingletonModelManager;
|
||||
/**
|
||||
* THE ONLY embedding function that should be used anywhere
|
||||
* This ensures all operations use the same model instance
|
||||
*/
|
||||
export declare function getUnifiedEmbeddingFunction(): Promise<EmbeddingFunction>;
|
||||
/**
|
||||
* Direct embed function for convenience
|
||||
*/
|
||||
export declare function unifiedEmbed(data: string | string[]): Promise<Vector>;
|
||||
/**
|
||||
* Check if model is ready
|
||||
*/
|
||||
export declare function isModelReady(): boolean;
|
||||
/**
|
||||
* Get model statistics
|
||||
*/
|
||||
export declare function getModelStats(): ModelStats;
|
||||
export {};
|
||||
|
|
@ -0,0 +1,220 @@
|
|||
/**
|
||||
* Singleton Model Manager - THE ONLY SOURCE OF EMBEDDING MODELS
|
||||
*
|
||||
* This is the SINGLE, UNIFIED model initialization system that ensures:
|
||||
* - Only ONE model instance exists across the entire system
|
||||
* - Precision is configured once and locked
|
||||
* - All components share the same model
|
||||
* - No possibility of mixed precisions
|
||||
*
|
||||
* CRITICAL: This manager is used by EVERYTHING:
|
||||
* - Storage operations (add, update)
|
||||
* - Search operations (search, find)
|
||||
* - Public API (embed, cluster)
|
||||
* - Neural API (all neural.* methods)
|
||||
* - Internal operations (deduplication, indexing)
|
||||
*/
|
||||
import { TransformerEmbedding } from '../utils/embedding.js';
|
||||
import { getModelPrecision, lockModelPrecision } from '../config/modelPrecisionManager.js';
|
||||
// Global state - ensures true singleton across entire process
|
||||
let globalModelInstance = null;
|
||||
let globalInitPromise = null;
|
||||
let globalInitialized = false;
|
||||
/**
|
||||
* The ONE TRUE model manager
|
||||
*/
|
||||
export class SingletonModelManager {
|
||||
constructor() {
|
||||
this.stats = {
|
||||
initialized: false,
|
||||
precision: 'unknown',
|
||||
initCount: 0,
|
||||
embedCount: 0,
|
||||
lastUsed: null
|
||||
};
|
||||
// Private constructor enforces singleton
|
||||
this.stats.precision = getModelPrecision();
|
||||
console.log(`🔐 SingletonModelManager initialized with ${this.stats.precision.toUpperCase()} precision`);
|
||||
}
|
||||
/**
|
||||
* Get the singleton instance
|
||||
*/
|
||||
static getInstance() {
|
||||
if (!SingletonModelManager.instance) {
|
||||
SingletonModelManager.instance = new SingletonModelManager();
|
||||
}
|
||||
return SingletonModelManager.instance;
|
||||
}
|
||||
/**
|
||||
* Get the model instance - creates if needed, reuses if exists
|
||||
* This is THE ONLY way to get a model in the entire system
|
||||
*/
|
||||
async getModel() {
|
||||
// If already initialized, return immediately
|
||||
if (globalModelInstance && globalInitialized) {
|
||||
this.stats.lastUsed = new Date();
|
||||
return globalModelInstance;
|
||||
}
|
||||
// If initialization is in progress, wait for it
|
||||
if (globalInitPromise) {
|
||||
console.log('⏳ Model initialization already in progress, waiting...');
|
||||
return await globalInitPromise;
|
||||
}
|
||||
// Start initialization (only happens once ever)
|
||||
globalInitPromise = this.initializeModel();
|
||||
try {
|
||||
const model = await globalInitPromise;
|
||||
globalInitialized = true;
|
||||
return model;
|
||||
}
|
||||
catch (error) {
|
||||
// Reset on error to allow retry
|
||||
globalInitPromise = null;
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
/**
|
||||
* Initialize the model - happens exactly once
|
||||
*/
|
||||
async initializeModel() {
|
||||
console.log('🚀 Initializing singleton model instance...');
|
||||
// Get precision from central manager
|
||||
const precision = getModelPrecision();
|
||||
console.log(`📊 Using ${precision.toUpperCase()} precision (${precision === 'q8' ? '23MB, 99% accuracy' : '90MB, 100% accuracy'})`);
|
||||
// Detect environment for optimal settings
|
||||
const isNode = typeof process !== 'undefined' && process.versions?.node;
|
||||
const isBrowser = typeof window !== 'undefined' && typeof document !== 'undefined';
|
||||
const isServerless = typeof process !== 'undefined' && (process.env.VERCEL ||
|
||||
process.env.NETLIFY ||
|
||||
process.env.AWS_LAMBDA_FUNCTION_NAME ||
|
||||
process.env.FUNCTIONS_WORKER_RUNTIME);
|
||||
const isTest = globalThis.__BRAINY_TEST_ENV__ || process.env.NODE_ENV === 'test';
|
||||
// Create optimized options based on environment
|
||||
const options = {
|
||||
precision: precision,
|
||||
verbose: !isTest && !isServerless && !isBrowser,
|
||||
device: 'cpu', // CPU is most compatible
|
||||
localFilesOnly: process.env.BRAINY_ALLOW_REMOTE_MODELS === 'false',
|
||||
model: 'Xenova/all-MiniLM-L6-v2'
|
||||
};
|
||||
try {
|
||||
// Create the ONE model instance
|
||||
globalModelInstance = new TransformerEmbedding(options);
|
||||
// Initialize it
|
||||
await globalModelInstance.init();
|
||||
// CRITICAL: Lock the precision after successful initialization
|
||||
// This prevents any future changes to precision
|
||||
lockModelPrecision();
|
||||
console.log('🔒 Model precision locked at:', precision.toUpperCase());
|
||||
// Update stats
|
||||
this.stats.initialized = true;
|
||||
this.stats.initCount++;
|
||||
this.stats.lastUsed = new Date();
|
||||
// Log memory usage if available
|
||||
if (isNode && process.memoryUsage) {
|
||||
const usage = process.memoryUsage();
|
||||
this.stats.memoryFootprint = Math.round(usage.heapUsed / 1024 / 1024);
|
||||
console.log(`💾 Model loaded, memory usage: ${this.stats.memoryFootprint}MB`);
|
||||
}
|
||||
console.log('✅ Singleton model initialized successfully');
|
||||
return globalModelInstance;
|
||||
}
|
||||
catch (error) {
|
||||
console.error('❌ Failed to initialize singleton model:', error);
|
||||
globalModelInstance = null;
|
||||
throw new Error(`Singleton model initialization failed: ${error instanceof Error ? error.message : String(error)}`);
|
||||
}
|
||||
}
|
||||
/**
|
||||
* Get embedding function that uses the singleton model
|
||||
*/
|
||||
async getEmbeddingFunction() {
|
||||
const model = await this.getModel();
|
||||
return async (data) => {
|
||||
this.stats.embedCount++;
|
||||
this.stats.lastUsed = new Date();
|
||||
return await model.embed(data);
|
||||
};
|
||||
}
|
||||
/**
|
||||
* Direct embed method for convenience
|
||||
*/
|
||||
async embed(data) {
|
||||
const model = await this.getModel();
|
||||
this.stats.embedCount++;
|
||||
this.stats.lastUsed = new Date();
|
||||
return await model.embed(data);
|
||||
}
|
||||
/**
|
||||
* Check if model is initialized
|
||||
*/
|
||||
isInitialized() {
|
||||
return globalInitialized && globalModelInstance !== null;
|
||||
}
|
||||
/**
|
||||
* Get current statistics
|
||||
*/
|
||||
getStats() {
|
||||
return {
|
||||
...this.stats,
|
||||
precision: getModelPrecision()
|
||||
};
|
||||
}
|
||||
/**
|
||||
* Validate precision consistency
|
||||
* Throws error if attempting to use different precision
|
||||
*/
|
||||
validatePrecision(requestedPrecision) {
|
||||
const currentPrecision = getModelPrecision();
|
||||
if (requestedPrecision && requestedPrecision !== currentPrecision) {
|
||||
throw new Error(`❌ Precision mismatch! System is using ${currentPrecision.toUpperCase()} ` +
|
||||
`but ${requestedPrecision.toUpperCase()} was requested. ` +
|
||||
`All operations must use the same precision.`);
|
||||
}
|
||||
}
|
||||
/**
|
||||
* Force cleanup (for testing only)
|
||||
* WARNING: This will break consistency - use only in tests
|
||||
*/
|
||||
async _testOnlyCleanup() {
|
||||
if (process.env.NODE_ENV !== 'test') {
|
||||
throw new Error('Cleanup only allowed in test environment');
|
||||
}
|
||||
if (globalModelInstance && 'dispose' in globalModelInstance) {
|
||||
await globalModelInstance.dispose();
|
||||
}
|
||||
globalModelInstance = null;
|
||||
globalInitPromise = null;
|
||||
globalInitialized = false;
|
||||
this.stats.initialized = false;
|
||||
console.log('🧹 Singleton model cleaned up (test only)');
|
||||
}
|
||||
}
|
||||
// Export the singleton instance getter
|
||||
export const singletonModelManager = SingletonModelManager.getInstance();
|
||||
/**
|
||||
* THE ONLY embedding function that should be used anywhere
|
||||
* This ensures all operations use the same model instance
|
||||
*/
|
||||
export async function getUnifiedEmbeddingFunction() {
|
||||
return await singletonModelManager.getEmbeddingFunction();
|
||||
}
|
||||
/**
|
||||
* Direct embed function for convenience
|
||||
*/
|
||||
export async function unifiedEmbed(data) {
|
||||
return await singletonModelManager.embed(data);
|
||||
}
|
||||
/**
|
||||
* Check if model is ready
|
||||
*/
|
||||
export function isModelReady() {
|
||||
return singletonModelManager.isInitialized();
|
||||
}
|
||||
/**
|
||||
* Get model statistics
|
||||
*/
|
||||
export function getModelStats() {
|
||||
return singletonModelManager.getStats();
|
||||
}
|
||||
//# sourceMappingURL=SingletonModelManager.js.map
|
||||
File diff suppressed because one or more lines are too long
12
.recovery-workspace/dist-backup-20250910-141917/embeddings/index.d.ts
vendored
Normal file
12
.recovery-workspace/dist-backup-20250910-141917/embeddings/index.d.ts
vendored
Normal file
|
|
@ -0,0 +1,12 @@
|
|||
/**
|
||||
* Embeddings Module - Clean, Unified Architecture
|
||||
*
|
||||
* This module provides all embedding functionality for Brainy.
|
||||
*
|
||||
* Main Components:
|
||||
* - EmbeddingManager: Core embedding generation with Q8/FP32 support
|
||||
* - CachedEmbeddings: Performance optimization layer with pre-computed embeddings
|
||||
*/
|
||||
export { EmbeddingManager, embeddingManager, embed, getEmbeddingFunction, getEmbeddingStats, type ModelPrecision } from './EmbeddingManager.js';
|
||||
export { CachedEmbeddings, cachedEmbeddings } from './CachedEmbeddings.js';
|
||||
export { embeddingManager as default } from './EmbeddingManager.js';
|
||||
|
|
@ -0,0 +1,16 @@
|
|||
/**
|
||||
* Embeddings Module - Clean, Unified Architecture
|
||||
*
|
||||
* This module provides all embedding functionality for Brainy.
|
||||
*
|
||||
* Main Components:
|
||||
* - EmbeddingManager: Core embedding generation with Q8/FP32 support
|
||||
* - CachedEmbeddings: Performance optimization layer with pre-computed embeddings
|
||||
*/
|
||||
// Core embedding functionality
|
||||
export { EmbeddingManager, embeddingManager, embed, getEmbeddingFunction, getEmbeddingStats } from './EmbeddingManager.js';
|
||||
// Cached embeddings for performance
|
||||
export { CachedEmbeddings, cachedEmbeddings } from './CachedEmbeddings.js';
|
||||
// Default export is the singleton manager
|
||||
export { embeddingManager as default } from './EmbeddingManager.js';
|
||||
//# sourceMappingURL=index.js.map
|
||||
|
|
@ -0,0 +1 @@
|
|||
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/embeddings/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAEH,+BAA+B;AAC/B,OAAO,EACL,gBAAgB,EAChB,gBAAgB,EAChB,KAAK,EACL,oBAAoB,EACpB,iBAAiB,EAElB,MAAM,uBAAuB,CAAA;AAE9B,oCAAoC;AACpC,OAAO,EACL,gBAAgB,EAChB,gBAAgB,EACjB,MAAM,uBAAuB,CAAA;AAE9B,0CAA0C;AAC1C,OAAO,EAAE,gBAAgB,IAAI,OAAO,EAAE,MAAM,uBAAuB,CAAA"}
|
||||
22
.recovery-workspace/dist-backup-20250910-141917/embeddings/lightweight-embedder.d.ts
vendored
Normal file
22
.recovery-workspace/dist-backup-20250910-141917/embeddings/lightweight-embedder.d.ts
vendored
Normal file
|
|
@ -0,0 +1,22 @@
|
|||
/**
|
||||
* Lightweight Embedding Alternative
|
||||
*
|
||||
* Uses pre-computed embeddings for common terms
|
||||
* Falls back to ONNX for unknown terms
|
||||
*
|
||||
* This reduces memory usage by 90% for typical queries
|
||||
*/
|
||||
import { Vector } from '../coreTypes.js';
|
||||
export declare class LightweightEmbedder {
|
||||
private stats;
|
||||
embed(text: string | string[]): Promise<Vector | Vector[]>;
|
||||
private embedSingle;
|
||||
getStats(): {
|
||||
totalEmbeddings: number;
|
||||
cacheHitRate: number;
|
||||
precomputedHits: number;
|
||||
simpleComputes: number;
|
||||
onnxComputes: number;
|
||||
};
|
||||
loadPrecomputed(filePath?: string): Promise<void>;
|
||||
}
|
||||
|
|
@ -0,0 +1,128 @@
|
|||
/**
|
||||
* Lightweight Embedding Alternative
|
||||
*
|
||||
* Uses pre-computed embeddings for common terms
|
||||
* Falls back to ONNX for unknown terms
|
||||
*
|
||||
* This reduces memory usage by 90% for typical queries
|
||||
*/
|
||||
import { singletonModelManager } from './SingletonModelManager.js';
|
||||
// Pre-computed embeddings for top 10,000 common terms
|
||||
// In production, this would be loaded from a file
|
||||
const PRECOMPUTED_EMBEDDINGS = {
|
||||
// Programming languages
|
||||
'javascript': new Array(384).fill(0).map((_, i) => Math.sin(i * 0.1)),
|
||||
'python': new Array(384).fill(0).map((_, i) => Math.cos(i * 0.1)),
|
||||
'typescript': new Array(384).fill(0).map((_, i) => Math.sin(i * 0.15)),
|
||||
'java': new Array(384).fill(0).map((_, i) => Math.cos(i * 0.15)),
|
||||
'rust': new Array(384).fill(0).map((_, i) => Math.sin(i * 0.2)),
|
||||
'go': new Array(384).fill(0).map((_, i) => Math.cos(i * 0.2)),
|
||||
// Frameworks
|
||||
'react': new Array(384).fill(0).map((_, i) => Math.sin(i * 0.25)),
|
||||
'vue': new Array(384).fill(0).map((_, i) => Math.cos(i * 0.25)),
|
||||
'angular': new Array(384).fill(0).map((_, i) => Math.sin(i * 0.3)),
|
||||
'svelte': new Array(384).fill(0).map((_, i) => Math.cos(i * 0.3)),
|
||||
// Databases
|
||||
'postgresql': new Array(384).fill(0).map((_, i) => Math.sin(i * 0.35)),
|
||||
'mysql': new Array(384).fill(0).map((_, i) => Math.cos(i * 0.35)),
|
||||
'mongodb': new Array(384).fill(0).map((_, i) => Math.sin(i * 0.4)),
|
||||
'redis': new Array(384).fill(0).map((_, i) => Math.cos(i * 0.4)),
|
||||
// Common terms
|
||||
'database': new Array(384).fill(0).map((_, i) => Math.sin(i * 0.45)),
|
||||
'api': new Array(384).fill(0).map((_, i) => Math.cos(i * 0.45)),
|
||||
'server': new Array(384).fill(0).map((_, i) => Math.sin(i * 0.5)),
|
||||
'client': new Array(384).fill(0).map((_, i) => Math.cos(i * 0.5)),
|
||||
'frontend': new Array(384).fill(0).map((_, i) => Math.sin(i * 0.55)),
|
||||
'backend': new Array(384).fill(0).map((_, i) => Math.cos(i * 0.55)),
|
||||
// Add more pre-computed embeddings here...
|
||||
};
|
||||
// Simple word similarity using character n-grams
|
||||
function computeSimpleEmbedding(text) {
|
||||
const normalized = text.toLowerCase().trim();
|
||||
const vector = new Array(384).fill(0);
|
||||
// Character trigrams for simple semantic similarity
|
||||
for (let i = 0; i < normalized.length - 2; i++) {
|
||||
const trigram = normalized.slice(i, i + 3);
|
||||
const hash = trigram.charCodeAt(0) * 31 +
|
||||
trigram.charCodeAt(1) * 7 +
|
||||
trigram.charCodeAt(2);
|
||||
const index = Math.abs(hash) % 384;
|
||||
vector[index] += 1 / (normalized.length - 2);
|
||||
}
|
||||
// Normalize vector
|
||||
const magnitude = Math.sqrt(vector.reduce((sum, val) => sum + val * val, 0));
|
||||
if (magnitude > 0) {
|
||||
for (let i = 0; i < vector.length; i++) {
|
||||
vector[i] /= magnitude;
|
||||
}
|
||||
}
|
||||
return vector;
|
||||
}
|
||||
export class LightweightEmbedder {
|
||||
constructor() {
|
||||
this.stats = {
|
||||
precomputedHits: 0,
|
||||
simpleComputes: 0,
|
||||
onnxComputes: 0
|
||||
};
|
||||
}
|
||||
async embed(text) {
|
||||
if (Array.isArray(text)) {
|
||||
return Promise.all(text.map(t => this.embedSingle(t)));
|
||||
}
|
||||
return this.embedSingle(text);
|
||||
}
|
||||
async embedSingle(text) {
|
||||
const normalized = text.toLowerCase().trim();
|
||||
// 1. Check pre-computed embeddings (instant, zero memory)
|
||||
if (PRECOMPUTED_EMBEDDINGS[normalized]) {
|
||||
this.stats.precomputedHits++;
|
||||
return PRECOMPUTED_EMBEDDINGS[normalized];
|
||||
}
|
||||
// 2. Check for close matches in pre-computed
|
||||
for (const [term, embedding] of Object.entries(PRECOMPUTED_EMBEDDINGS)) {
|
||||
if (normalized.includes(term) || term.includes(normalized)) {
|
||||
this.stats.precomputedHits++;
|
||||
// Return slightly modified version to maintain uniqueness
|
||||
return embedding.map(v => v * 0.95);
|
||||
}
|
||||
}
|
||||
// 3. For short text, use simple embedding (fast, low memory)
|
||||
if (normalized.length < 50) {
|
||||
this.stats.simpleComputes++;
|
||||
return computeSimpleEmbedding(normalized);
|
||||
}
|
||||
// 4. Last resort: Use SingletonModelManager for complex text
|
||||
console.log('⚠️ Using singleton model for complex text...');
|
||||
this.stats.onnxComputes++;
|
||||
return await singletonModelManager.embed(text);
|
||||
}
|
||||
getStats() {
|
||||
return {
|
||||
...this.stats,
|
||||
totalEmbeddings: this.stats.precomputedHits +
|
||||
this.stats.simpleComputes +
|
||||
this.stats.onnxComputes,
|
||||
cacheHitRate: this.stats.precomputedHits /
|
||||
(this.stats.precomputedHits +
|
||||
this.stats.simpleComputes +
|
||||
this.stats.onnxComputes)
|
||||
};
|
||||
}
|
||||
// Pre-load common embeddings from file
|
||||
async loadPrecomputed(filePath) {
|
||||
if (!filePath)
|
||||
return;
|
||||
try {
|
||||
const fs = await import('fs/promises');
|
||||
const data = await fs.readFile(filePath, 'utf-8');
|
||||
const embeddings = JSON.parse(data);
|
||||
Object.assign(PRECOMPUTED_EMBEDDINGS, embeddings);
|
||||
console.log(`✅ Loaded ${Object.keys(embeddings).length} pre-computed embeddings`);
|
||||
}
|
||||
catch (error) {
|
||||
console.warn('Could not load pre-computed embeddings:', error);
|
||||
}
|
||||
}
|
||||
}
|
||||
//# sourceMappingURL=lightweight-embedder.js.map
|
||||
File diff suppressed because one or more lines are too long
39
.recovery-workspace/dist-backup-20250910-141917/embeddings/model-manager.d.ts
vendored
Normal file
39
.recovery-workspace/dist-backup-20250910-141917/embeddings/model-manager.d.ts
vendored
Normal file
|
|
@ -0,0 +1,39 @@
|
|||
/**
|
||||
* Model Manager - Ensures transformer models are available at runtime
|
||||
*
|
||||
* Strategy (in order):
|
||||
* 1. Check local cache first (instant)
|
||||
* 2. Try Soulcraft CDN (fastest when available)
|
||||
* 3. Try GitHub release tar.gz with extraction (reliable backup)
|
||||
* 4. Fall back to Hugging Face (always works)
|
||||
*
|
||||
* NO USER CONFIGURATION REQUIRED - Everything is automatic!
|
||||
*/
|
||||
export declare class ModelManager {
|
||||
private static instance;
|
||||
private modelsPath;
|
||||
private isInitialized;
|
||||
private constructor();
|
||||
static getInstance(): ModelManager;
|
||||
private getModelsPath;
|
||||
ensureModels(modelName?: string): Promise<boolean>;
|
||||
private verifyModelFiles;
|
||||
/**
|
||||
* Check which model variants are available locally
|
||||
*/
|
||||
getAvailableModels(modelName?: string): {
|
||||
fp32: boolean;
|
||||
q8: boolean;
|
||||
};
|
||||
/**
|
||||
* Get the best available model variant based on preference and availability
|
||||
*/
|
||||
getBestAvailableModel(preferredType?: 'fp32' | 'q8', modelName?: string): 'fp32' | 'q8' | null;
|
||||
private tryModelSource;
|
||||
private downloadAndExtractFromGitHub;
|
||||
/**
|
||||
* Pre-download models for deployment
|
||||
* This is what npm run download-models calls
|
||||
*/
|
||||
static predownload(): Promise<void>;
|
||||
}
|
||||
|
|
@ -0,0 +1,245 @@
|
|||
/**
|
||||
* Model Manager - Ensures transformer models are available at runtime
|
||||
*
|
||||
* Strategy (in order):
|
||||
* 1. Check local cache first (instant)
|
||||
* 2. Try Soulcraft CDN (fastest when available)
|
||||
* 3. Try GitHub release tar.gz with extraction (reliable backup)
|
||||
* 4. Fall back to Hugging Face (always works)
|
||||
*
|
||||
* NO USER CONFIGURATION REQUIRED - Everything is automatic!
|
||||
*/
|
||||
import { existsSync } from 'fs';
|
||||
import { mkdir, writeFile } from 'fs/promises';
|
||||
import { join } from 'path';
|
||||
import { env } from '@huggingface/transformers';
|
||||
// Model sources in order of preference
|
||||
const MODEL_SOURCES = {
|
||||
// CDN - Fastest when available (currently active)
|
||||
cdn: {
|
||||
host: 'https://models.soulcraft.com/models',
|
||||
pathTemplate: '{model}/', // e.g., Xenova/all-MiniLM-L6-v2/
|
||||
testFile: 'config.json' // File to test availability
|
||||
},
|
||||
// GitHub Release - tar.gz fallback (already exists and works)
|
||||
githubRelease: {
|
||||
tarUrl: 'https://github.com/soulcraftlabs/brainy/releases/download/models-v1/all-MiniLM-L6-v2.tar.gz'
|
||||
},
|
||||
// Original Hugging Face - final fallback (always works)
|
||||
huggingface: {
|
||||
host: 'https://huggingface.co',
|
||||
pathTemplate: '{model}/resolve/{revision}/' // Default transformers.js pattern
|
||||
}
|
||||
};
|
||||
// Model verification files - BOTH fp32 and q8 variants
|
||||
const REQUIRED_FILES = [
|
||||
'config.json',
|
||||
'tokenizer.json',
|
||||
'tokenizer_config.json'
|
||||
];
|
||||
const MODEL_VARIANTS = {
|
||||
fp32: 'onnx/model.onnx',
|
||||
q8: 'onnx/model_quantized.onnx'
|
||||
};
|
||||
export class ModelManager {
|
||||
constructor() {
|
||||
this.isInitialized = false;
|
||||
// Determine models path
|
||||
this.modelsPath = this.getModelsPath();
|
||||
}
|
||||
static getInstance() {
|
||||
if (!ModelManager.instance) {
|
||||
ModelManager.instance = new ModelManager();
|
||||
}
|
||||
return ModelManager.instance;
|
||||
}
|
||||
getModelsPath() {
|
||||
// Check various possible locations
|
||||
const paths = [
|
||||
process.env.BRAINY_MODELS_PATH,
|
||||
'./models',
|
||||
join(process.cwd(), 'models'),
|
||||
join(process.env.HOME || '', '.brainy', 'models'),
|
||||
env.cacheDir
|
||||
];
|
||||
// Find first existing path or use default
|
||||
for (const path of paths) {
|
||||
if (path && existsSync(path)) {
|
||||
return path;
|
||||
}
|
||||
}
|
||||
// Default to local models directory
|
||||
return join(process.cwd(), 'models');
|
||||
}
|
||||
async ensureModels(modelName = 'Xenova/all-MiniLM-L6-v2') {
|
||||
if (this.isInitialized) {
|
||||
return true;
|
||||
}
|
||||
// Configure transformers.js environment
|
||||
env.cacheDir = this.modelsPath;
|
||||
env.allowLocalModels = true;
|
||||
env.useFSCache = true;
|
||||
// Check if model already exists locally
|
||||
const modelPath = join(this.modelsPath, ...modelName.split('/'));
|
||||
if (await this.verifyModelFiles(modelPath)) {
|
||||
console.log('✅ Models found in cache:', modelPath);
|
||||
env.allowRemoteModels = false; // Use local only
|
||||
this.isInitialized = true;
|
||||
return true;
|
||||
}
|
||||
// Try to download from our sources
|
||||
console.log('📥 Downloading transformer models...');
|
||||
// Try CDN first (fastest when available)
|
||||
if (await this.tryModelSource('Soulcraft CDN', MODEL_SOURCES.cdn, modelName)) {
|
||||
this.isInitialized = true;
|
||||
return true;
|
||||
}
|
||||
// Try GitHub release with tar.gz extraction (reliable backup)
|
||||
if (await this.downloadAndExtractFromGitHub(modelName)) {
|
||||
this.isInitialized = true;
|
||||
return true;
|
||||
}
|
||||
// Fall back to Hugging Face (always works)
|
||||
console.log('⚠️ Using Hugging Face fallback for models');
|
||||
env.remoteHost = MODEL_SOURCES.huggingface.host;
|
||||
env.remotePathTemplate = MODEL_SOURCES.huggingface.pathTemplate;
|
||||
env.allowRemoteModels = true;
|
||||
this.isInitialized = true;
|
||||
return true;
|
||||
}
|
||||
async verifyModelFiles(modelPath) {
|
||||
// Check if essential files exist
|
||||
for (const file of REQUIRED_FILES) {
|
||||
const fullPath = join(modelPath, file);
|
||||
if (!existsSync(fullPath)) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
// At least one model variant must exist (fp32 or q8)
|
||||
const fp32Exists = existsSync(join(modelPath, MODEL_VARIANTS.fp32));
|
||||
const q8Exists = existsSync(join(modelPath, MODEL_VARIANTS.q8));
|
||||
return fp32Exists || q8Exists;
|
||||
}
|
||||
/**
|
||||
* Check which model variants are available locally
|
||||
*/
|
||||
getAvailableModels(modelName = 'Xenova/all-MiniLM-L6-v2') {
|
||||
const modelPath = join(this.modelsPath, modelName);
|
||||
return {
|
||||
fp32: existsSync(join(modelPath, MODEL_VARIANTS.fp32)),
|
||||
q8: existsSync(join(modelPath, MODEL_VARIANTS.q8))
|
||||
};
|
||||
}
|
||||
/**
|
||||
* Get the best available model variant based on preference and availability
|
||||
*/
|
||||
getBestAvailableModel(preferredType = 'fp32', modelName = 'Xenova/all-MiniLM-L6-v2') {
|
||||
const available = this.getAvailableModels(modelName);
|
||||
// If preferred type is available, use it
|
||||
if (available[preferredType]) {
|
||||
return preferredType;
|
||||
}
|
||||
// Otherwise fall back to what's available
|
||||
if (preferredType === 'q8' && available.fp32) {
|
||||
console.warn('⚠️ Q8 model requested but not available, falling back to FP32');
|
||||
return 'fp32';
|
||||
}
|
||||
if (preferredType === 'fp32' && available.q8) {
|
||||
console.warn('⚠️ FP32 model requested but not available, falling back to Q8');
|
||||
return 'q8';
|
||||
}
|
||||
return null;
|
||||
}
|
||||
async tryModelSource(name, source, modelName) {
|
||||
try {
|
||||
console.log(`📥 Trying ${name}...`);
|
||||
// Test if the source is accessible by trying to fetch a test file
|
||||
const testFile = source.testFile || 'config.json';
|
||||
const modelPath = source.pathTemplate.replace('{model}', modelName).replace('{revision}', 'main');
|
||||
const testUrl = `${source.host}/${modelPath}${testFile}`;
|
||||
const response = await fetch(testUrl).catch(() => null);
|
||||
if (response && response.ok) {
|
||||
console.log(`✅ ${name} is available`);
|
||||
// Configure transformers.js to use this source
|
||||
env.remoteHost = source.host;
|
||||
env.remotePathTemplate = source.pathTemplate;
|
||||
env.allowRemoteModels = true;
|
||||
// The model will be downloaded automatically by transformers.js when needed
|
||||
return true;
|
||||
}
|
||||
else {
|
||||
console.log(`⚠️ ${name} not available (${response?.status || 'unreachable'})`);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
catch (error) {
|
||||
console.log(`⚠️ ${name} check failed:`, error.message);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
async downloadAndExtractFromGitHub(modelName) {
|
||||
try {
|
||||
console.log('📥 Trying GitHub Release (tar.gz)...');
|
||||
// Download tar.gz file
|
||||
const response = await fetch(MODEL_SOURCES.githubRelease.tarUrl);
|
||||
if (!response.ok) {
|
||||
console.log(`⚠️ GitHub Release not available (${response.status})`);
|
||||
return false;
|
||||
}
|
||||
// Since we can't use tar-stream, we'll use Node's built-in child_process
|
||||
// to extract using system tar command (available on all Unix systems)
|
||||
const buffer = await response.arrayBuffer();
|
||||
const modelPath = join(this.modelsPath, ...modelName.split('/'));
|
||||
// Create model directory
|
||||
await mkdir(modelPath, { recursive: true });
|
||||
// Write tar.gz to temp file and extract
|
||||
const tempFile = join(this.modelsPath, 'temp-model.tar.gz');
|
||||
await writeFile(tempFile, Buffer.from(buffer));
|
||||
// Extract using system tar command
|
||||
const { exec } = await import('child_process');
|
||||
const { promisify } = await import('util');
|
||||
const execAsync = promisify(exec);
|
||||
try {
|
||||
// Extract and strip the first directory component
|
||||
await execAsync(`tar -xzf ${tempFile} -C ${modelPath} --strip-components=1`, {
|
||||
cwd: this.modelsPath
|
||||
});
|
||||
// Clean up temp file
|
||||
const { unlink } = await import('fs/promises');
|
||||
await unlink(tempFile);
|
||||
console.log('✅ GitHub Release models extracted and cached locally');
|
||||
// Configure to use local models now
|
||||
env.allowRemoteModels = false;
|
||||
return true;
|
||||
}
|
||||
catch (extractError) {
|
||||
console.log('⚠️ Tar extraction failed, trying alternative method');
|
||||
return false;
|
||||
}
|
||||
}
|
||||
catch (error) {
|
||||
console.log('⚠️ GitHub Release download failed:', error.message);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
/**
|
||||
* Pre-download models for deployment
|
||||
* This is what npm run download-models calls
|
||||
*/
|
||||
static async predownload() {
|
||||
const manager = ModelManager.getInstance();
|
||||
const success = await manager.ensureModels();
|
||||
if (!success) {
|
||||
throw new Error('Failed to download models');
|
||||
}
|
||||
console.log('✅ Models downloaded successfully');
|
||||
}
|
||||
}
|
||||
// Auto-initialize on import in production
|
||||
if (process.env.NODE_ENV === 'production' && process.env.SKIP_MODEL_CHECK !== 'true') {
|
||||
ModelManager.getInstance().ensureModels().catch(error => {
|
||||
console.error('⚠️ Model initialization failed:', error);
|
||||
// Don't throw - allow app to start and try downloading on first use
|
||||
});
|
||||
}
|
||||
//# sourceMappingURL=model-manager.js.map
|
||||
File diff suppressed because one or more lines are too long
38
.recovery-workspace/dist-backup-20250910-141917/embeddings/universal-memory-manager.d.ts
vendored
Normal file
38
.recovery-workspace/dist-backup-20250910-141917/embeddings/universal-memory-manager.d.ts
vendored
Normal file
|
|
@ -0,0 +1,38 @@
|
|||
/**
|
||||
* Universal Memory Manager for Embeddings
|
||||
*
|
||||
* Works in ALL environments: Node.js, browsers, serverless, workers
|
||||
* Solves transformers.js memory leak with environment-specific strategies
|
||||
*/
|
||||
import { Vector, EmbeddingFunction } from '../coreTypes.js';
|
||||
interface MemoryStats {
|
||||
embeddings: number;
|
||||
memoryUsage: string;
|
||||
restarts: number;
|
||||
strategy: string;
|
||||
}
|
||||
export declare class UniversalMemoryManager {
|
||||
private embeddingFunction;
|
||||
private embedCount;
|
||||
private restartCount;
|
||||
private lastRestart;
|
||||
private strategy;
|
||||
private maxEmbeddings;
|
||||
constructor();
|
||||
getEmbeddingFunction(): Promise<EmbeddingFunction>;
|
||||
embed(data: string | string[]): Promise<Vector>;
|
||||
private checkMemoryLimits;
|
||||
private ensureEmbeddingFunction;
|
||||
private initNodeDirect;
|
||||
private initServerless;
|
||||
private initBrowser;
|
||||
private initFallback;
|
||||
private initDirect;
|
||||
private cleanup;
|
||||
getMemoryStats(): MemoryStats;
|
||||
dispose(): Promise<void>;
|
||||
}
|
||||
export declare const universalMemoryManager: UniversalMemoryManager;
|
||||
export declare function getUniversalEmbeddingFunction(): Promise<EmbeddingFunction>;
|
||||
export declare function getEmbeddingMemoryStats(): MemoryStats;
|
||||
export {};
|
||||
|
|
@ -0,0 +1,166 @@
|
|||
/**
|
||||
* Universal Memory Manager for Embeddings
|
||||
*
|
||||
* Works in ALL environments: Node.js, browsers, serverless, workers
|
||||
* Solves transformers.js memory leak with environment-specific strategies
|
||||
*/
|
||||
import { getModelPrecision } from '../config/modelPrecisionManager.js';
|
||||
// Environment detection
|
||||
const isNode = typeof process !== 'undefined' && process.versions?.node;
|
||||
const isBrowser = typeof window !== 'undefined' && typeof document !== 'undefined';
|
||||
const isServerless = typeof process !== 'undefined' && (process.env.VERCEL ||
|
||||
process.env.NETLIFY ||
|
||||
process.env.AWS_LAMBDA_FUNCTION_NAME ||
|
||||
process.env.FUNCTIONS_WORKER_RUNTIME);
|
||||
export class UniversalMemoryManager {
|
||||
constructor() {
|
||||
// CRITICAL FIX: Never use worker threads with ONNX Runtime
|
||||
// Worker threads cause HandleScope V8 API errors due to isolate issues
|
||||
// Always use direct embedding on main thread for ONNX compatibility
|
||||
this.embeddingFunction = null;
|
||||
this.embedCount = 0;
|
||||
this.restartCount = 0;
|
||||
this.lastRestart = 0;
|
||||
if (isServerless) {
|
||||
this.strategy = 'serverless-restart';
|
||||
this.maxEmbeddings = 50; // Restart frequently in serverless
|
||||
}
|
||||
else if (isNode && !isBrowser) {
|
||||
// CHANGED: Use direct strategy instead of node-worker to avoid V8 isolate issues
|
||||
this.strategy = 'node-direct';
|
||||
this.maxEmbeddings = 200; // Main thread can handle more with single model instance
|
||||
}
|
||||
else if (isBrowser) {
|
||||
this.strategy = 'browser-dispose';
|
||||
this.maxEmbeddings = 25; // Browser memory is limited
|
||||
}
|
||||
else {
|
||||
this.strategy = 'fallback-dispose';
|
||||
this.maxEmbeddings = 75;
|
||||
}
|
||||
console.log(`🧠 Universal Memory Manager: Using ${this.strategy} strategy`);
|
||||
console.log('✅ UNIVERSAL: Memory-safe embedding system initialized');
|
||||
}
|
||||
async getEmbeddingFunction() {
|
||||
return async (data) => {
|
||||
return this.embed(data);
|
||||
};
|
||||
}
|
||||
async embed(data) {
|
||||
// Check if we need to restart/cleanup
|
||||
await this.checkMemoryLimits();
|
||||
// Ensure embedding function is available
|
||||
await this.ensureEmbeddingFunction();
|
||||
// Perform embedding
|
||||
const result = await this.embeddingFunction.embed(data);
|
||||
this.embedCount++;
|
||||
return result;
|
||||
}
|
||||
async checkMemoryLimits() {
|
||||
if (this.embedCount >= this.maxEmbeddings) {
|
||||
console.log(`🔄 Memory cleanup: ${this.embedCount} embeddings processed`);
|
||||
await this.cleanup();
|
||||
}
|
||||
}
|
||||
async ensureEmbeddingFunction() {
|
||||
if (this.embeddingFunction) {
|
||||
return;
|
||||
}
|
||||
switch (this.strategy) {
|
||||
case 'node-direct':
|
||||
await this.initNodeDirect();
|
||||
break;
|
||||
case 'serverless-restart':
|
||||
await this.initServerless();
|
||||
break;
|
||||
case 'browser-dispose':
|
||||
await this.initBrowser();
|
||||
break;
|
||||
default:
|
||||
await this.initFallback();
|
||||
}
|
||||
}
|
||||
async initNodeDirect() {
|
||||
if (isNode) {
|
||||
// CRITICAL: Use direct embedding to avoid worker thread V8 isolate issues
|
||||
// This prevents HandleScope errors and ensures single model instance
|
||||
console.log('✅ Using Node.js direct embedding (main thread - ONNX compatible)');
|
||||
await this.initDirect();
|
||||
}
|
||||
}
|
||||
async initServerless() {
|
||||
// In serverless, use direct embedding but restart more aggressively
|
||||
await this.initDirect();
|
||||
console.log('✅ Using serverless strategy with aggressive cleanup');
|
||||
}
|
||||
async initBrowser() {
|
||||
// In browser, use direct embedding with disposal
|
||||
await this.initDirect();
|
||||
console.log('✅ Using browser strategy with disposal');
|
||||
}
|
||||
async initFallback() {
|
||||
await this.initDirect();
|
||||
console.log('✅ Using fallback direct embedding strategy');
|
||||
}
|
||||
async initDirect() {
|
||||
try {
|
||||
// Dynamic import to handle different environments
|
||||
const { TransformerEmbedding } = await import('../utils/embedding.js');
|
||||
this.embeddingFunction = new TransformerEmbedding({
|
||||
verbose: false,
|
||||
precision: getModelPrecision(), // Use centrally managed precision
|
||||
localFilesOnly: process.env.BRAINY_ALLOW_REMOTE_MODELS !== 'true'
|
||||
});
|
||||
await this.embeddingFunction.init();
|
||||
console.log('✅ Direct embedding function initialized');
|
||||
}
|
||||
catch (error) {
|
||||
throw new Error(`Failed to initialize embedding function: ${error instanceof Error ? error.message : String(error)}`);
|
||||
}
|
||||
}
|
||||
async cleanup() {
|
||||
const startTime = Date.now();
|
||||
// SingletonModelManager persists - we just reset our counters
|
||||
// The singleton model stays alive for consistency across all operations
|
||||
// Reset counters
|
||||
this.embedCount = 0;
|
||||
this.restartCount++;
|
||||
this.lastRestart = Date.now();
|
||||
const cleanupTime = Date.now() - startTime;
|
||||
console.log(`🧹 Memory counters reset in ${cleanupTime}ms (strategy: ${this.strategy})`);
|
||||
console.log('ℹ️ Singleton model persists for consistency across all operations');
|
||||
}
|
||||
getMemoryStats() {
|
||||
let memoryUsage = 'unknown';
|
||||
// Get memory stats based on environment
|
||||
if (isNode && typeof process !== 'undefined') {
|
||||
const mem = process.memoryUsage();
|
||||
memoryUsage = `${(mem.heapUsed / 1024 / 1024).toFixed(2)} MB`;
|
||||
}
|
||||
else if (isBrowser && performance.memory) {
|
||||
const mem = performance.memory;
|
||||
memoryUsage = `${(mem.usedJSHeapSize / 1024 / 1024).toFixed(2)} MB`;
|
||||
}
|
||||
return {
|
||||
embeddings: this.embedCount,
|
||||
memoryUsage,
|
||||
restarts: this.restartCount,
|
||||
strategy: this.strategy
|
||||
};
|
||||
}
|
||||
async dispose() {
|
||||
// SingletonModelManager persists - nothing to dispose
|
||||
console.log('ℹ️ Universal Memory Manager: Singleton model persists');
|
||||
}
|
||||
}
|
||||
// Export singleton instance
|
||||
export const universalMemoryManager = new UniversalMemoryManager();
|
||||
// Export convenience function
|
||||
export async function getUniversalEmbeddingFunction() {
|
||||
return universalMemoryManager.getEmbeddingFunction();
|
||||
}
|
||||
// Export memory stats function
|
||||
export function getEmbeddingMemoryStats() {
|
||||
return universalMemoryManager.getMemoryStats();
|
||||
}
|
||||
//# sourceMappingURL=universal-memory-manager.js.map
|
||||
|
|
@ -0,0 +1 @@
|
|||
{"version":3,"file":"universal-memory-manager.js","sourceRoot":"","sources":["../../src/embeddings/universal-memory-manager.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAGH,OAAO,EAAE,iBAAiB,EAAE,MAAM,oCAAoC,CAAA;AAEtE,wBAAwB;AACxB,MAAM,MAAM,GAAG,OAAO,OAAO,KAAK,WAAW,IAAI,OAAO,CAAC,QAAQ,EAAE,IAAI,CAAA;AACvE,MAAM,SAAS,GAAG,OAAO,MAAM,KAAK,WAAW,IAAI,OAAO,QAAQ,KAAK,WAAW,CAAA;AAClF,MAAM,YAAY,GAAG,OAAO,OAAO,KAAK,WAAW,IAAI,CACrD,OAAO,CAAC,GAAG,CAAC,MAAM;IAClB,OAAO,CAAC,GAAG,CAAC,OAAO;IACnB,OAAO,CAAC,GAAG,CAAC,wBAAwB;IACpC,OAAO,CAAC,GAAG,CAAC,wBAAwB,CACrC,CAAA;AASD,MAAM,OAAO,sBAAsB;IAQjC;QACE,2DAA2D;QAC3D,uEAAuE;QACvE,oEAAoE;QAV9D,sBAAiB,GAAQ,IAAI,CAAA;QAC7B,eAAU,GAAG,CAAC,CAAA;QACd,iBAAY,GAAG,CAAC,CAAA;QAChB,gBAAW,GAAG,CAAC,CAAA;QASrB,IAAI,YAAY,EAAE,CAAC;YACjB,IAAI,CAAC,QAAQ,GAAG,oBAAoB,CAAA;YACpC,IAAI,CAAC,aAAa,GAAG,EAAE,CAAA,CAAE,mCAAmC;QAC9D,CAAC;aAAM,IAAI,MAAM,IAAI,CAAC,SAAS,EAAE,CAAC;YAChC,iFAAiF;YACjF,IAAI,CAAC,QAAQ,GAAG,aAAa,CAAA;YAC7B,IAAI,CAAC,aAAa,GAAG,GAAG,CAAA,CAAC,yDAAyD;QACpF,CAAC;aAAM,IAAI,SAAS,EAAE,CAAC;YACrB,IAAI,CAAC,QAAQ,GAAG,iBAAiB,CAAA;YACjC,IAAI,CAAC,aAAa,GAAG,EAAE,CAAA,CAAE,4BAA4B;QACvD,CAAC;aAAM,CAAC;YACN,IAAI,CAAC,QAAQ,GAAG,kBAAkB,CAAA;YAClC,IAAI,CAAC,aAAa,GAAG,EAAE,CAAA;QACzB,CAAC;QAED,OAAO,CAAC,GAAG,CAAC,sCAAsC,IAAI,CAAC,QAAQ,WAAW,CAAC,CAAA;QAC3E,OAAO,CAAC,GAAG,CAAC,uDAAuD,CAAC,CAAA;IACtE,CAAC;IAED,KAAK,CAAC,oBAAoB;QACxB,OAAO,KAAK,EAAE,IAAuB,EAAmB,EAAE;YACxD,OAAO,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAA;QACzB,CAAC,CAAA;IACH,CAAC;IAED,KAAK,CAAC,KAAK,CAAC,IAAuB;QACjC,sCAAsC;QACtC,MAAM,IAAI,CAAC,iBAAiB,EAAE,CAAA;QAE9B,yCAAyC;QACzC,MAAM,IAAI,CAAC,uBAAuB,EAAE,CAAA;QAEpC,oBAAoB;QACpB,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,iBAAiB,CAAC,KAAK,CAAC,IAAI,CAAC,CAAA;QACvD,IAAI,CAAC,UAAU,EAAE,CAAA;QAEjB,OAAO,MAAM,CAAA;IACf,CAAC;IAEO,KAAK,CAAC,iBAAiB;QAC7B,IAAI,IAAI,CAAC,UAAU,IAAI,IAAI,CAAC,aAAa,EAAE,CAAC;YAC1C,OAAO,CAAC,GAAG,CAAC,sBAAsB,IAAI,CAAC,UAAU,uBAAuB,CAAC,CAAA;YACzE,MAAM,IAAI,CAAC,OAAO,EAAE,CAAA;QACtB,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,uBAAuB;QACnC,IAAI,IAAI,CAAC,iBAAiB,EAAE,CAAC;YAC3B,OAAM;QACR,CAAC;QAED,QAAQ,IAAI,CAAC,QAAQ,EAAE,CAAC;YACtB,KAAK,aAAa;gBAChB,MAAM,IAAI,CAAC,cAAc,EAAE,CAAA;gBAC3B,MAAK;YAEP,KAAK,oBAAoB;gBACvB,MAAM,IAAI,CAAC,cAAc,EAAE,CAAA;gBAC3B,MAAK;YAEP,KAAK,iBAAiB;gBACpB,MAAM,IAAI,CAAC,WAAW,EAAE,CAAA;gBACxB,MAAK;YAEP;gBACE,MAAM,IAAI,CAAC,YAAY,EAAE,CAAA;QAC7B,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,cAAc;QAC1B,IAAI,MAAM,EAAE,CAAC;YACX,0EAA0E;YAC1E,qEAAqE;YACrE,OAAO,CAAC,GAAG,CAAC,kEAAkE,CAAC,CAAA;YAC/E,MAAM,IAAI,CAAC,UAAU,EAAE,CAAA;QACzB,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,cAAc;QAC1B,oEAAoE;QACpE,MAAM,IAAI,CAAC,UAAU,EAAE,CAAA;QACvB,OAAO,CAAC,GAAG,CAAC,qDAAqD,CAAC,CAAA;IACpE,CAAC;IAEO,KAAK,CAAC,WAAW;QACvB,iDAAiD;QACjD,MAAM,IAAI,CAAC,UAAU,EAAE,CAAA;QACvB,OAAO,CAAC,GAAG,CAAC,wCAAwC,CAAC,CAAA;IACvD,CAAC;IAEO,KAAK,CAAC,YAAY;QACxB,MAAM,IAAI,CAAC,UAAU,EAAE,CAAA;QACvB,OAAO,CAAC,GAAG,CAAC,4CAA4C,CAAC,CAAA;IAC3D,CAAC;IAEO,KAAK,CAAC,UAAU;QACtB,IAAI,CAAC;YACH,kDAAkD;YAClD,MAAM,EAAE,oBAAoB,EAAE,GAAG,MAAM,MAAM,CAAC,uBAAuB,CAAC,CAAA;YAEtE,IAAI,CAAC,iBAAiB,GAAG,IAAI,oBAAoB,CAAC;gBAChD,OAAO,EAAE,KAAK;gBACd,SAAS,EAAE,iBAAiB,EAAE,EAAE,kCAAkC;gBAClE,cAAc,EAAE,OAAO,CAAC,GAAG,CAAC,0BAA0B,KAAK,MAAM;aAClE,CAAC,CAAA;YAEF,MAAM,IAAI,CAAC,iBAAiB,CAAC,IAAI,EAAE,CAAA;YACnC,OAAO,CAAC,GAAG,CAAC,yCAAyC,CAAC,CAAA;QACxD,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,IAAI,KAAK,CAAC,4CAA4C,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAA;QACvH,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,OAAO;QACnB,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAA;QAE5B,8DAA8D;QAC9D,wEAAwE;QAExE,iBAAiB;QACjB,IAAI,CAAC,UAAU,GAAG,CAAC,CAAA;QACnB,IAAI,CAAC,YAAY,EAAE,CAAA;QACnB,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC,GAAG,EAAE,CAAA;QAE7B,MAAM,WAAW,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,CAAA;QAC1C,OAAO,CAAC,GAAG,CAAC,+BAA+B,WAAW,iBAAiB,IAAI,CAAC,QAAQ,GAAG,CAAC,CAAA;QACxF,OAAO,CAAC,GAAG,CAAC,mEAAmE,CAAC,CAAA;IAClF,CAAC;IAED,cAAc;QACZ,IAAI,WAAW,GAAG,SAAS,CAAA;QAE3B,wCAAwC;QACxC,IAAI,MAAM,IAAI,OAAO,OAAO,KAAK,WAAW,EAAE,CAAC;YAC7C,MAAM,GAAG,GAAG,OAAO,CAAC,WAAW,EAAE,CAAA;YACjC,WAAW,GAAG,GAAG,CAAC,GAAG,CAAC,QAAQ,GAAG,IAAI,GAAG,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK,CAAA;QAC/D,CAAC;aAAM,IAAI,SAAS,IAAK,WAAmB,CAAC,MAAM,EAAE,CAAC;YACpD,MAAM,GAAG,GAAI,WAAmB,CAAC,MAAM,CAAA;YACvC,WAAW,GAAG,GAAG,CAAC,GAAG,CAAC,cAAc,GAAG,IAAI,GAAG,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK,CAAA;QACrE,CAAC;QAED,OAAO;YACL,UAAU,EAAE,IAAI,CAAC,UAAU;YAC3B,WAAW;YACX,QAAQ,EAAE,IAAI,CAAC,YAAY;YAC3B,QAAQ,EAAE,IAAI,CAAC,QAAQ;SACxB,CAAA;IACH,CAAC;IAED,KAAK,CAAC,OAAO;QACX,sDAAsD;QACtD,OAAO,CAAC,GAAG,CAAC,uDAAuD,CAAC,CAAA;IACtE,CAAC;CACF;AAED,4BAA4B;AAC5B,MAAM,CAAC,MAAM,sBAAsB,GAAG,IAAI,sBAAsB,EAAE,CAAA;AAElE,8BAA8B;AAC9B,MAAM,CAAC,KAAK,UAAU,6BAA6B;IACjD,OAAO,sBAAsB,CAAC,oBAAoB,EAAE,CAAA;AACtD,CAAC;AAED,+BAA+B;AAC/B,MAAM,UAAU,uBAAuB;IACrC,OAAO,sBAAsB,CAAC,cAAc,EAAE,CAAA;AAChD,CAAC"}
|
||||
7
.recovery-workspace/dist-backup-20250910-141917/embeddings/worker-embedding.d.ts
vendored
Normal file
7
.recovery-workspace/dist-backup-20250910-141917/embeddings/worker-embedding.d.ts
vendored
Normal file
|
|
@ -0,0 +1,7 @@
|
|||
/**
|
||||
* Worker process for embeddings - Workaround for transformers.js memory leak
|
||||
*
|
||||
* This worker can be killed and restarted to release memory completely.
|
||||
* Based on 2024 research: dispose() doesn't fully free memory in transformers.js
|
||||
*/
|
||||
export {};
|
||||
|
|
@ -0,0 +1,73 @@
|
|||
/**
|
||||
* Worker process for embeddings - Workaround for transformers.js memory leak
|
||||
*
|
||||
* This worker can be killed and restarted to release memory completely.
|
||||
* Based on 2024 research: dispose() doesn't fully free memory in transformers.js
|
||||
*/
|
||||
import { TransformerEmbedding } from '../utils/embedding.js';
|
||||
import { parentPort } from 'worker_threads';
|
||||
import { getModelPrecision } from '../config/modelPrecisionManager.js';
|
||||
let model = null;
|
||||
let requestCount = 0;
|
||||
const MAX_REQUESTS = 100; // Restart worker after 100 requests to prevent memory leak
|
||||
async function initModel() {
|
||||
if (!model) {
|
||||
model = new TransformerEmbedding({
|
||||
verbose: false,
|
||||
precision: getModelPrecision(), // Use centrally managed precision
|
||||
localFilesOnly: process.env.BRAINY_ALLOW_REMOTE_MODELS !== 'true'
|
||||
});
|
||||
await model.init();
|
||||
console.log('🔧 Worker: Model initialized');
|
||||
}
|
||||
}
|
||||
if (parentPort) {
|
||||
parentPort.on('message', async (message) => {
|
||||
try {
|
||||
const { id, type, data } = message;
|
||||
switch (type) {
|
||||
case 'embed':
|
||||
await initModel();
|
||||
const embeddings = await model.embed(data);
|
||||
parentPort.postMessage({ id, success: true, result: embeddings });
|
||||
requestCount++;
|
||||
// Proactively restart worker to prevent memory leak
|
||||
if (requestCount >= MAX_REQUESTS) {
|
||||
console.log(`🔄 Worker: Restarting after ${requestCount} requests (memory leak prevention)`);
|
||||
process.exit(0); // Parent will restart us
|
||||
}
|
||||
break;
|
||||
case 'dispose':
|
||||
// SingletonModelManager persists - just acknowledge
|
||||
console.log('ℹ️ Worker: Singleton model persists');
|
||||
parentPort.postMessage({ id, success: true });
|
||||
break;
|
||||
case 'restart':
|
||||
// Force restart to clear memory
|
||||
console.log('🔄 Worker: Force restart requested');
|
||||
process.exit(0);
|
||||
break;
|
||||
default:
|
||||
parentPort.postMessage({
|
||||
id,
|
||||
success: false,
|
||||
error: `Unknown message type: ${type}`
|
||||
});
|
||||
}
|
||||
}
|
||||
catch (error) {
|
||||
parentPort.postMessage({
|
||||
id: message.id,
|
||||
success: false,
|
||||
error: error instanceof Error ? error.message : String(error)
|
||||
});
|
||||
}
|
||||
});
|
||||
console.log('🚀 Embedding worker started');
|
||||
parentPort.postMessage({ type: 'ready' });
|
||||
}
|
||||
else {
|
||||
console.error('❌ Worker: parentPort is null, cannot communicate with main thread');
|
||||
process.exit(1);
|
||||
}
|
||||
//# sourceMappingURL=worker-embedding.js.map
|
||||
|
|
@ -0,0 +1 @@
|
|||
{"version":3,"file":"worker-embedding.js","sourceRoot":"","sources":["../../src/embeddings/worker-embedding.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,EAAE,oBAAoB,EAAE,MAAM,uBAAuB,CAAA;AAC5D,OAAO,EAAE,UAAU,EAAE,MAAM,gBAAgB,CAAA;AAC3C,OAAO,EAAE,iBAAiB,EAAE,MAAM,oCAAoC,CAAA;AAEtE,IAAI,KAAK,GAAgC,IAAI,CAAA;AAC7C,IAAI,YAAY,GAAG,CAAC,CAAA;AACpB,MAAM,YAAY,GAAG,GAAG,CAAA,CAAC,2DAA2D;AAEpF,KAAK,UAAU,SAAS;IACtB,IAAI,CAAC,KAAK,EAAE,CAAC;QACX,KAAK,GAAG,IAAI,oBAAoB,CAAC;YAC/B,OAAO,EAAE,KAAK;YACd,SAAS,EAAE,iBAAiB,EAAE,EAAE,kCAAkC;YAClE,cAAc,EAAE,OAAO,CAAC,GAAG,CAAC,0BAA0B,KAAK,MAAM;SAClE,CAAC,CAAA;QACF,MAAM,KAAK,CAAC,IAAI,EAAE,CAAA;QAClB,OAAO,CAAC,GAAG,CAAC,8BAA8B,CAAC,CAAA;IAC7C,CAAC;AACH,CAAC;AAED,IAAI,UAAU,EAAE,CAAC;IACf,UAAU,CAAC,EAAE,CAAC,SAAS,EAAE,KAAK,EAAE,OAAO,EAAE,EAAE;QACzC,IAAI,CAAC;YACH,MAAM,EAAE,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,GAAG,OAAO,CAAA;YAElC,QAAQ,IAAI,EAAE,CAAC;gBACb,KAAK,OAAO;oBACV,MAAM,SAAS,EAAE,CAAA;oBACjB,MAAM,UAAU,GAAG,MAAM,KAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAA;oBAC3C,UAAW,CAAC,WAAW,CAAC,EAAE,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,UAAU,EAAE,CAAC,CAAA;oBAElE,YAAY,EAAE,CAAA;oBAEd,oDAAoD;oBACpD,IAAI,YAAY,IAAI,YAAY,EAAE,CAAC;wBACjC,OAAO,CAAC,GAAG,CAAC,+BAA+B,YAAY,oCAAoC,CAAC,CAAA;wBAC5F,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA,CAAC,yBAAyB;oBAC3C,CAAC;oBACD,MAAK;gBAEP,KAAK,SAAS;oBACZ,oDAAoD;oBACpD,OAAO,CAAC,GAAG,CAAC,qCAAqC,CAAC,CAAA;oBAClD,UAAW,CAAC,WAAW,CAAC,EAAE,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC,CAAA;oBAC9C,MAAK;gBAEP,KAAK,SAAS;oBACZ,gCAAgC;oBAChC,OAAO,CAAC,GAAG,CAAC,oCAAoC,CAAC,CAAA;oBACjD,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;oBACf,MAAK;gBAEP;oBACE,UAAW,CAAC,WAAW,CAAC;wBACtB,EAAE;wBACF,OAAO,EAAE,KAAK;wBACd,KAAK,EAAE,yBAAyB,IAAI,EAAE;qBACvC,CAAC,CAAA;YACN,CAAC;QACH,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,UAAW,CAAC,WAAW,CAAC;gBACtB,EAAE,EAAE,OAAO,CAAC,EAAE;gBACd,OAAO,EAAE,KAAK;gBACd,KAAK,EAAE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC;aAC9D,CAAC,CAAA;QACJ,CAAC;IACH,CAAC,CAAC,CAAA;IAEF,OAAO,CAAC,GAAG,CAAC,6BAA6B,CAAC,CAAA;IAC1C,UAAU,CAAC,WAAW,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,CAAC,CAAA;AAC3C,CAAC;KAAM,CAAC;IACN,OAAO,CAAC,KAAK,CAAC,mEAAmE,CAAC,CAAA;IAClF,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;AACjB,CAAC"}
|
||||
28
.recovery-workspace/dist-backup-20250910-141917/embeddings/worker-manager.d.ts
vendored
Normal file
28
.recovery-workspace/dist-backup-20250910-141917/embeddings/worker-manager.d.ts
vendored
Normal file
|
|
@ -0,0 +1,28 @@
|
|||
/**
|
||||
* Worker Manager for Memory-Safe Embeddings
|
||||
*
|
||||
* Manages worker lifecycle to prevent transformers.js memory leaks
|
||||
* Workers are automatically restarted when memory usage grows too high
|
||||
*/
|
||||
import { Vector, EmbeddingFunction } from '../coreTypes.js';
|
||||
export declare class WorkerEmbeddingManager {
|
||||
private worker;
|
||||
private requestId;
|
||||
private pendingRequests;
|
||||
private isRestarting;
|
||||
private totalRequests;
|
||||
getEmbeddingFunction(): Promise<EmbeddingFunction>;
|
||||
embed(data: string | string[]): Promise<Vector>;
|
||||
private ensureWorker;
|
||||
private createWorker;
|
||||
dispose(): Promise<void>;
|
||||
forceRestart(): Promise<void>;
|
||||
getStats(): {
|
||||
totalRequests: number;
|
||||
pendingRequests: number;
|
||||
workerActive: boolean;
|
||||
isRestarting: boolean;
|
||||
};
|
||||
}
|
||||
export declare const workerEmbeddingManager: WorkerEmbeddingManager;
|
||||
export declare function getWorkerEmbeddingFunction(): Promise<EmbeddingFunction>;
|
||||
|
|
@ -0,0 +1,162 @@
|
|||
/**
|
||||
* Worker Manager for Memory-Safe Embeddings
|
||||
*
|
||||
* Manages worker lifecycle to prevent transformers.js memory leaks
|
||||
* Workers are automatically restarted when memory usage grows too high
|
||||
*/
|
||||
import { Worker } from 'worker_threads';
|
||||
import { join, dirname } from 'path';
|
||||
import { fileURLToPath } from 'url';
|
||||
// Get current directory for worker path
|
||||
const __filename = fileURLToPath(import.meta.url);
|
||||
const __dirname = dirname(__filename);
|
||||
export class WorkerEmbeddingManager {
|
||||
constructor() {
|
||||
this.worker = null;
|
||||
this.requestId = 0;
|
||||
this.pendingRequests = new Map();
|
||||
this.isRestarting = false;
|
||||
this.totalRequests = 0;
|
||||
}
|
||||
async getEmbeddingFunction() {
|
||||
return async (data) => {
|
||||
return this.embed(data);
|
||||
};
|
||||
}
|
||||
async embed(data) {
|
||||
await this.ensureWorker();
|
||||
const id = ++this.requestId;
|
||||
this.totalRequests++;
|
||||
return new Promise((resolve, reject) => {
|
||||
const timeout = setTimeout(() => {
|
||||
this.pendingRequests.delete(id);
|
||||
reject(new Error('Embedding request timed out (120s)'));
|
||||
}, 120000);
|
||||
this.pendingRequests.set(id, { resolve, reject, timeout });
|
||||
this.worker.postMessage({
|
||||
id,
|
||||
type: 'embed',
|
||||
data
|
||||
});
|
||||
});
|
||||
}
|
||||
async ensureWorker() {
|
||||
if (this.worker && !this.isRestarting) {
|
||||
return;
|
||||
}
|
||||
if (this.isRestarting) {
|
||||
// Wait for restart to complete
|
||||
return new Promise((resolve) => {
|
||||
const checkRestart = () => {
|
||||
if (!this.isRestarting) {
|
||||
resolve();
|
||||
}
|
||||
else {
|
||||
setTimeout(checkRestart, 100);
|
||||
}
|
||||
};
|
||||
checkRestart();
|
||||
});
|
||||
}
|
||||
await this.createWorker();
|
||||
}
|
||||
async createWorker() {
|
||||
this.isRestarting = true;
|
||||
// Kill existing worker if any
|
||||
if (this.worker) {
|
||||
this.worker.terminate();
|
||||
this.worker = null;
|
||||
}
|
||||
// Clear pending requests
|
||||
for (const [id, request] of this.pendingRequests) {
|
||||
if (request.timeout) {
|
||||
clearTimeout(request.timeout);
|
||||
}
|
||||
request.reject(new Error('Worker restarted'));
|
||||
}
|
||||
this.pendingRequests.clear();
|
||||
console.log('🔄 Starting embedding worker...');
|
||||
// Create new worker
|
||||
const workerPath = join(__dirname, 'worker-embedding.js');
|
||||
this.worker = new Worker(workerPath);
|
||||
// Handle worker messages
|
||||
this.worker.on('message', (message) => {
|
||||
if (message.type === 'ready') {
|
||||
console.log('✅ Embedding worker ready');
|
||||
this.isRestarting = false;
|
||||
return;
|
||||
}
|
||||
const { id, success, result, error } = message;
|
||||
const request = this.pendingRequests.get(id);
|
||||
if (request) {
|
||||
if (request.timeout) {
|
||||
clearTimeout(request.timeout);
|
||||
}
|
||||
this.pendingRequests.delete(id);
|
||||
if (success) {
|
||||
request.resolve(result);
|
||||
}
|
||||
else {
|
||||
request.reject(new Error(error));
|
||||
}
|
||||
}
|
||||
});
|
||||
// Handle worker exit
|
||||
this.worker.on('exit', (code) => {
|
||||
console.log(`🔄 Embedding worker exited with code ${code}`);
|
||||
if (code !== 0 && !this.isRestarting) {
|
||||
console.log('🔄 Worker crashed, will restart on next request');
|
||||
}
|
||||
this.worker = null;
|
||||
});
|
||||
// Wait for worker to be ready
|
||||
return new Promise((resolve, reject) => {
|
||||
const timeout = setTimeout(() => {
|
||||
reject(new Error('Worker startup timeout'));
|
||||
}, 30000);
|
||||
const checkReady = () => {
|
||||
if (!this.isRestarting) {
|
||||
clearTimeout(timeout);
|
||||
resolve();
|
||||
}
|
||||
else {
|
||||
setTimeout(checkReady, 100);
|
||||
}
|
||||
};
|
||||
checkReady();
|
||||
});
|
||||
}
|
||||
async dispose() {
|
||||
if (this.worker) {
|
||||
this.worker.terminate();
|
||||
this.worker = null;
|
||||
}
|
||||
// Clear pending requests
|
||||
for (const [id, request] of this.pendingRequests) {
|
||||
if (request.timeout) {
|
||||
clearTimeout(request.timeout);
|
||||
}
|
||||
request.reject(new Error('Manager disposed'));
|
||||
}
|
||||
this.pendingRequests.clear();
|
||||
}
|
||||
async forceRestart() {
|
||||
console.log('🔄 Force restarting embedding worker (memory cleanup)');
|
||||
await this.createWorker();
|
||||
}
|
||||
getStats() {
|
||||
return {
|
||||
totalRequests: this.totalRequests,
|
||||
pendingRequests: this.pendingRequests.size,
|
||||
workerActive: this.worker !== null,
|
||||
isRestarting: this.isRestarting
|
||||
};
|
||||
}
|
||||
}
|
||||
// Export singleton instance
|
||||
export const workerEmbeddingManager = new WorkerEmbeddingManager();
|
||||
// Export convenience function
|
||||
export async function getWorkerEmbeddingFunction() {
|
||||
return workerEmbeddingManager.getEmbeddingFunction();
|
||||
}
|
||||
//# sourceMappingURL=worker-manager.js.map
|
||||
|
|
@ -0,0 +1 @@
|
|||
{"version":3,"file":"worker-manager.js","sourceRoot":"","sources":["../../src/embeddings/worker-manager.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,EAAE,MAAM,EAAE,MAAM,gBAAgB,CAAA;AACvC,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,MAAM,CAAA;AACpC,OAAO,EAAE,aAAa,EAAE,MAAM,KAAK,CAAA;AAGnC,wCAAwC;AACxC,MAAM,UAAU,GAAG,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;AACjD,MAAM,SAAS,GAAG,OAAO,CAAC,UAAU,CAAC,CAAA;AAQrC,MAAM,OAAO,sBAAsB;IAAnC;QACU,WAAM,GAAkB,IAAI,CAAA;QAC5B,cAAS,GAAG,CAAC,CAAA;QACb,oBAAe,GAAG,IAAI,GAAG,EAA0B,CAAA;QACnD,iBAAY,GAAG,KAAK,CAAA;QACpB,kBAAa,GAAG,CAAC,CAAA;IA6J3B,CAAC;IA3JC,KAAK,CAAC,oBAAoB;QACxB,OAAO,KAAK,EAAE,IAAuB,EAAmB,EAAE;YACxD,OAAO,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAA;QACzB,CAAC,CAAA;IACH,CAAC;IAED,KAAK,CAAC,KAAK,CAAC,IAAuB;QACjC,MAAM,IAAI,CAAC,YAAY,EAAE,CAAA;QAEzB,MAAM,EAAE,GAAG,EAAE,IAAI,CAAC,SAAS,CAAA;QAC3B,IAAI,CAAC,aAAa,EAAE,CAAA;QAEpB,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YACrC,MAAM,OAAO,GAAG,UAAU,CAAC,GAAG,EAAE;gBAC9B,IAAI,CAAC,eAAe,CAAC,MAAM,CAAC,EAAE,CAAC,CAAA;gBAC/B,MAAM,CAAC,IAAI,KAAK,CAAC,oCAAoC,CAAC,CAAC,CAAA;YACzD,CAAC,EAAE,MAAM,CAAC,CAAA;YAEV,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,EAAE,EAAE,EAAE,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,CAAC,CAAA;YAE1D,IAAI,CAAC,MAAO,CAAC,WAAW,CAAC;gBACvB,EAAE;gBACF,IAAI,EAAE,OAAO;gBACb,IAAI;aACL,CAAC,CAAA;QACJ,CAAC,CAAC,CAAA;IACJ,CAAC;IAEO,KAAK,CAAC,YAAY;QACxB,IAAI,IAAI,CAAC,MAAM,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE,CAAC;YACtC,OAAM;QACR,CAAC;QAED,IAAI,IAAI,CAAC,YAAY,EAAE,CAAC;YACtB,+BAA+B;YAC/B,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE;gBAC7B,MAAM,YAAY,GAAG,GAAG,EAAE;oBACxB,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE,CAAC;wBACvB,OAAO,EAAE,CAAA;oBACX,CAAC;yBAAM,CAAC;wBACN,UAAU,CAAC,YAAY,EAAE,GAAG,CAAC,CAAA;oBAC/B,CAAC;gBACH,CAAC,CAAA;gBACD,YAAY,EAAE,CAAA;YAChB,CAAC,CAAC,CAAA;QACJ,CAAC;QAED,MAAM,IAAI,CAAC,YAAY,EAAE,CAAA;IAC3B,CAAC;IAEO,KAAK,CAAC,YAAY;QACxB,IAAI,CAAC,YAAY,GAAG,IAAI,CAAA;QAExB,8BAA8B;QAC9B,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;YAChB,IAAI,CAAC,MAAM,CAAC,SAAS,EAAE,CAAA;YACvB,IAAI,CAAC,MAAM,GAAG,IAAI,CAAA;QACpB,CAAC;QAED,yBAAyB;QACzB,KAAK,MAAM,CAAC,EAAE,EAAE,OAAO,CAAC,IAAI,IAAI,CAAC,eAAe,EAAE,CAAC;YACjD,IAAI,OAAO,CAAC,OAAO,EAAE,CAAC;gBACpB,YAAY,CAAC,OAAO,CAAC,OAAO,CAAC,CAAA;YAC/B,CAAC;YACD,OAAO,CAAC,MAAM,CAAC,IAAI,KAAK,CAAC,kBAAkB,CAAC,CAAC,CAAA;QAC/C,CAAC;QACD,IAAI,CAAC,eAAe,CAAC,KAAK,EAAE,CAAA;QAE5B,OAAO,CAAC,GAAG,CAAC,iCAAiC,CAAC,CAAA;QAE9C,oBAAoB;QACpB,MAAM,UAAU,GAAG,IAAI,CAAC,SAAS,EAAE,qBAAqB,CAAC,CAAA;QACzD,IAAI,CAAC,MAAM,GAAG,IAAI,MAAM,CAAC,UAAU,CAAC,CAAA;QAEpC,yBAAyB;QACzB,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC,SAAS,EAAE,CAAC,OAAO,EAAE,EAAE;YACpC,IAAI,OAAO,CAAC,IAAI,KAAK,OAAO,EAAE,CAAC;gBAC7B,OAAO,CAAC,GAAG,CAAC,0BAA0B,CAAC,CAAA;gBACvC,IAAI,CAAC,YAAY,GAAG,KAAK,CAAA;gBACzB,OAAM;YACR,CAAC;YAED,MAAM,EAAE,EAAE,EAAE,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,GAAG,OAAO,CAAA;YAC9C,MAAM,OAAO,GAAG,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,EAAE,CAAC,CAAA;YAE5C,IAAI,OAAO,EAAE,CAAC;gBACZ,IAAI,OAAO,CAAC,OAAO,EAAE,CAAC;oBACpB,YAAY,CAAC,OAAO,CAAC,OAAO,CAAC,CAAA;gBAC/B,CAAC;gBACD,IAAI,CAAC,eAAe,CAAC,MAAM,CAAC,EAAE,CAAC,CAAA;gBAE/B,IAAI,OAAO,EAAE,CAAC;oBACZ,OAAO,CAAC,OAAO,CAAC,MAAM,CAAC,CAAA;gBACzB,CAAC;qBAAM,CAAC;oBACN,OAAO,CAAC,MAAM,CAAC,IAAI,KAAK,CAAC,KAAK,CAAC,CAAC,CAAA;gBAClC,CAAC;YACH,CAAC;QACH,CAAC,CAAC,CAAA;QAEF,qBAAqB;QACrB,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,EAAE;YAC9B,OAAO,CAAC,GAAG,CAAC,wCAAwC,IAAI,EAAE,CAAC,CAAA;YAC3D,IAAI,IAAI,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE,CAAC;gBACrC,OAAO,CAAC,GAAG,CAAC,iDAAiD,CAAC,CAAA;YAChE,CAAC;YACD,IAAI,CAAC,MAAM,GAAG,IAAI,CAAA;QACpB,CAAC,CAAC,CAAA;QAEF,8BAA8B;QAC9B,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YACrC,MAAM,OAAO,GAAG,UAAU,CAAC,GAAG,EAAE;gBAC9B,MAAM,CAAC,IAAI,KAAK,CAAC,wBAAwB,CAAC,CAAC,CAAA;YAC7C,CAAC,EAAE,KAAK,CAAC,CAAA;YAET,MAAM,UAAU,GAAG,GAAG,EAAE;gBACtB,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE,CAAC;oBACvB,YAAY,CAAC,OAAO,CAAC,CAAA;oBACrB,OAAO,EAAE,CAAA;gBACX,CAAC;qBAAM,CAAC;oBACN,UAAU,CAAC,UAAU,EAAE,GAAG,CAAC,CAAA;gBAC7B,CAAC;YACH,CAAC,CAAA;YACD,UAAU,EAAE,CAAA;QACd,CAAC,CAAC,CAAA;IACJ,CAAC;IAED,KAAK,CAAC,OAAO;QACX,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;YAChB,IAAI,CAAC,MAAM,CAAC,SAAS,EAAE,CAAA;YACvB,IAAI,CAAC,MAAM,GAAG,IAAI,CAAA;QACpB,CAAC;QAED,yBAAyB;QACzB,KAAK,MAAM,CAAC,EAAE,EAAE,OAAO,CAAC,IAAI,IAAI,CAAC,eAAe,EAAE,CAAC;YACjD,IAAI,OAAO,CAAC,OAAO,EAAE,CAAC;gBACpB,YAAY,CAAC,OAAO,CAAC,OAAO,CAAC,CAAA;YAC/B,CAAC;YACD,OAAO,CAAC,MAAM,CAAC,IAAI,KAAK,CAAC,kBAAkB,CAAC,CAAC,CAAA;QAC/C,CAAC;QACD,IAAI,CAAC,eAAe,CAAC,KAAK,EAAE,CAAA;IAC9B,CAAC;IAED,KAAK,CAAC,YAAY;QAChB,OAAO,CAAC,GAAG,CAAC,uDAAuD,CAAC,CAAA;QACpE,MAAM,IAAI,CAAC,YAAY,EAAE,CAAA;IAC3B,CAAC;IAED,QAAQ;QACN,OAAO;YACL,aAAa,EAAE,IAAI,CAAC,aAAa;YACjC,eAAe,EAAE,IAAI,CAAC,eAAe,CAAC,IAAI;YAC1C,YAAY,EAAE,IAAI,CAAC,MAAM,KAAK,IAAI;YAClC,YAAY,EAAE,IAAI,CAAC,YAAY;SAChC,CAAA;IACH,CAAC;CACF;AAED,4BAA4B;AAC5B,MAAM,CAAC,MAAM,sBAAsB,GAAG,IAAI,sBAAsB,EAAE,CAAA;AAElE,8BAA8B;AAC9B,MAAM,CAAC,KAAK,UAAU,0BAA0B;IAC9C,OAAO,sBAAsB,CAAC,oBAAoB,EAAE,CAAA;AACtD,CAAC"}
|
||||
Loading…
Add table
Add a link
Reference in a new issue