David Snelling
c303ead318
- Remove top-level Node.js imports that break bundlers - Use universal adapters for crypto operations - Add dynamic imports for Node.js-specific modules - Add browser field to package.json for bundler hints - Maintain full Node.js functionality while enabling browser usage This allows Brainy to be used with modern bundlers (Vite, Webpack, etc.) without requiring Node.js polyfills. Browser environments get core features while Node.js retains all capabilities including filesystem and networking.
339 lines
No EOL
11 KiB
TypeScript
339 lines
No EOL
11 KiB
TypeScript
/**
|
|
* MODEL GUARDIAN - CRITICAL PATH
|
|
*
|
|
* THIS IS THE MOST CRITICAL COMPONENT OF BRAINY
|
|
* Without the exact model, users CANNOT access their data
|
|
*
|
|
* Requirements:
|
|
* 1. Model MUST be Xenova/all-MiniLM-L6-v2 (never changes)
|
|
* 2. Model MUST be available at runtime
|
|
* 3. Model MUST produce consistent 384-dim embeddings
|
|
* 4. System MUST fail fast if model unavailable in production
|
|
*/
|
|
|
|
import { env } from '@huggingface/transformers'
|
|
import { createHash } from '../universal/crypto.js'
|
|
|
|
// CRITICAL: These values MUST NEVER CHANGE
|
|
const CRITICAL_MODEL_CONFIG = {
|
|
modelName: 'Xenova/all-MiniLM-L6-v2',
|
|
modelHash: {
|
|
// SHA256 of model.onnx - computed from actual model
|
|
'onnx/model.onnx': 'add_actual_hash_here',
|
|
'tokenizer.json': 'add_actual_hash_here'
|
|
} as Record<string, string>,
|
|
modelSize: {
|
|
'onnx/model.onnx': 90387606, // Exact size in bytes (updated to match actual file)
|
|
'tokenizer.json': 711661
|
|
} as Record<string, number>,
|
|
embeddingDimensions: 384,
|
|
fallbackSources: [
|
|
// Primary: Our Google Cloud Storage CDN (we control this, fastest)
|
|
{
|
|
name: 'Soulcraft CDN (Primary)',
|
|
url: 'https://models.soulcraft.com/models/all-MiniLM-L6-v2.tar.gz',
|
|
type: 'tarball'
|
|
},
|
|
// Secondary: GitHub releases backup
|
|
{
|
|
name: 'GitHub Backup',
|
|
url: 'https://github.com/soulcraftlabs/brainy-models/releases/download/v1.0.0/all-MiniLM-L6-v2.tar.gz',
|
|
type: 'tarball'
|
|
},
|
|
// Tertiary: Hugging Face (original source)
|
|
{
|
|
name: 'Hugging Face',
|
|
url: 'huggingface',
|
|
type: 'transformers'
|
|
}
|
|
]
|
|
}
|
|
|
|
export class ModelGuardian {
|
|
private static instance: ModelGuardian
|
|
private isVerified = false
|
|
private modelPath: string
|
|
private lastVerification: Date | null = null
|
|
|
|
private constructor() {
|
|
this.modelPath = this.detectModelPath()
|
|
}
|
|
|
|
static getInstance(): ModelGuardian {
|
|
if (!ModelGuardian.instance) {
|
|
ModelGuardian.instance = new ModelGuardian()
|
|
}
|
|
return ModelGuardian.instance
|
|
}
|
|
|
|
/**
|
|
* CRITICAL: Verify model availability and integrity
|
|
* This MUST be called before any embedding operations
|
|
*/
|
|
async ensureCriticalModel(): Promise<void> {
|
|
console.log('DEBUG: ensureCriticalModel called')
|
|
console.log('🛡️ MODEL GUARDIAN: Verifying critical model availability...')
|
|
console.log(`🚀 Debug: Model path: ${this.modelPath}`)
|
|
console.log(`🚀 Debug: Already verified: ${this.isVerified}`)
|
|
|
|
// Check if already verified in this session
|
|
if (this.isVerified && this.lastVerification) {
|
|
const hoursSinceVerification =
|
|
(Date.now() - this.lastVerification.getTime()) / (1000 * 60 * 60)
|
|
|
|
if (hoursSinceVerification < 24) {
|
|
console.log('✅ Model previously verified in this session')
|
|
return
|
|
}
|
|
}
|
|
|
|
// Step 1: Check if model exists locally
|
|
console.log('🔍 Debug: Calling verifyLocalModel()')
|
|
const modelExists = await this.verifyLocalModel()
|
|
|
|
if (modelExists) {
|
|
console.log('✅ Critical model verified locally')
|
|
this.isVerified = true
|
|
this.lastVerification = new Date()
|
|
this.configureTransformers()
|
|
return
|
|
}
|
|
|
|
// Step 2: In production, FAIL FAST (Node.js only)
|
|
if (typeof window === 'undefined' && process.env.NODE_ENV === 'production' && !process.env.BRAINY_ALLOW_RUNTIME_DOWNLOAD) {
|
|
throw new Error(
|
|
'🚨 CRITICAL FAILURE: Transformer model not found in production!\n' +
|
|
'The model is REQUIRED for Brainy to function.\n' +
|
|
'Users CANNOT access their data without it.\n' +
|
|
'Solution: Run "npm run download-models" during build stage.'
|
|
)
|
|
}
|
|
|
|
// Step 3: Attempt to download from fallback sources
|
|
console.warn('⚠️ Model not found locally, attempting download...')
|
|
|
|
for (const source of CRITICAL_MODEL_CONFIG.fallbackSources) {
|
|
try {
|
|
console.log(`📥 Trying ${source.name}...`)
|
|
await this.downloadFromSource(source)
|
|
|
|
// Verify the download
|
|
if (await this.verifyLocalModel()) {
|
|
console.log(`✅ Successfully downloaded from ${source.name}`)
|
|
this.isVerified = true
|
|
this.lastVerification = new Date()
|
|
this.configureTransformers()
|
|
return
|
|
}
|
|
} catch (error) {
|
|
console.warn(`❌ ${source.name} failed:`, (error as Error).message)
|
|
}
|
|
}
|
|
|
|
// Step 4: CRITICAL FAILURE
|
|
throw new Error(
|
|
'🚨 CRITICAL FAILURE: Cannot obtain transformer model!\n' +
|
|
'Tried all fallback sources.\n' +
|
|
'Brainy CANNOT function without the model.\n' +
|
|
'Users CANNOT access their data.\n' +
|
|
'Please check network connectivity or pre-download models.'
|
|
)
|
|
}
|
|
|
|
/**
|
|
* Verify the local model files exist and are correct
|
|
*/
|
|
private async verifyLocalModel(): Promise<boolean> {
|
|
// Browser doesn't have local file access
|
|
if (typeof window !== 'undefined') {
|
|
console.log('⚠️ Model verification skipped in browser environment')
|
|
return false
|
|
}
|
|
|
|
// Dynamically import Node.js modules
|
|
const fs = await import('node:fs')
|
|
const fsPromises = await import('node:fs/promises')
|
|
const path = await import('node:path')
|
|
|
|
const modelBasePath = path.join(this.modelPath, ...CRITICAL_MODEL_CONFIG.modelName.split('/'))
|
|
console.log(`🔍 Debug: Checking model at path: ${modelBasePath}`)
|
|
console.log(`🔍 Debug: Model path components: ${this.modelPath} + ${CRITICAL_MODEL_CONFIG.modelName.split('/')}`)
|
|
|
|
// Check critical files
|
|
const criticalFiles = [
|
|
'onnx/model.onnx',
|
|
'tokenizer.json',
|
|
'config.json'
|
|
]
|
|
|
|
for (const file of criticalFiles) {
|
|
const filePath = path.join(modelBasePath, file)
|
|
console.log(`🔍 Debug: Checking file: ${filePath}`)
|
|
|
|
if (!fs.existsSync(filePath)) {
|
|
console.log(`❌ Missing critical file: ${file} at ${filePath}`)
|
|
return false
|
|
}
|
|
|
|
// Verify size for critical files
|
|
if (CRITICAL_MODEL_CONFIG.modelSize[file]) {
|
|
const stats = await fsPromises.stat(filePath)
|
|
const expectedSize = CRITICAL_MODEL_CONFIG.modelSize[file]
|
|
|
|
if (Math.abs(stats.size - expectedSize) > 1000) { // Allow 1KB variance
|
|
console.error(
|
|
`❌ CRITICAL: Model file size mismatch!\n` +
|
|
`File: ${file}\n` +
|
|
`Expected: ${expectedSize} bytes\n` +
|
|
`Actual: ${stats.size} bytes\n` +
|
|
`This indicates model corruption or version mismatch!`
|
|
)
|
|
return false
|
|
}
|
|
}
|
|
|
|
// SHA256 verification for ultimate security
|
|
if (CRITICAL_MODEL_CONFIG.modelHash && CRITICAL_MODEL_CONFIG.modelHash[file]) {
|
|
const hash = await this.computeFileHash(filePath)
|
|
if (hash !== CRITICAL_MODEL_CONFIG.modelHash[file]) {
|
|
console.error(
|
|
`❌ CRITICAL: Model hash mismatch for ${file}!\n` +
|
|
`Expected: ${CRITICAL_MODEL_CONFIG.modelHash[file]}\n` +
|
|
`Got: ${hash}\n` +
|
|
`This indicates model tampering or corruption!`
|
|
)
|
|
return false
|
|
}
|
|
}
|
|
}
|
|
|
|
return true
|
|
}
|
|
|
|
/**
|
|
* Compute SHA256 hash of a file
|
|
*/
|
|
private async computeFileHash(filePath: string): Promise<string> {
|
|
try {
|
|
const { readFile } = await import('node:fs/promises')
|
|
const { createHash } = await import('node:crypto')
|
|
const fileBuffer = await readFile(filePath)
|
|
const hash = createHash('sha256').update(fileBuffer).digest('hex')
|
|
return hash
|
|
} catch (error) {
|
|
console.error(`Failed to compute hash for ${filePath}:`, error)
|
|
return ''
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Download model from a fallback source
|
|
*/
|
|
private async downloadFromSource(source: any): Promise<void> {
|
|
if (source.type === 'transformers') {
|
|
// Use transformers.js native download
|
|
const { pipeline } = await import('@huggingface/transformers')
|
|
env.cacheDir = this.modelPath
|
|
env.allowRemoteModels = true
|
|
|
|
const extractor = await pipeline(
|
|
'feature-extraction',
|
|
CRITICAL_MODEL_CONFIG.modelName
|
|
)
|
|
|
|
// Test the model
|
|
const test = await extractor('test', { pooling: 'mean', normalize: true })
|
|
if (test.data.length !== CRITICAL_MODEL_CONFIG.embeddingDimensions) {
|
|
throw new Error(
|
|
`CRITICAL: Model dimension mismatch! ` +
|
|
`Expected ${CRITICAL_MODEL_CONFIG.embeddingDimensions}, ` +
|
|
`got ${test.data.length}`
|
|
)
|
|
}
|
|
} else if (source.type === 'tarball') {
|
|
// Tarball extraction would require additional dependencies
|
|
// Skip this source and try next fallback
|
|
console.warn(`⚠️ Tarball extraction not available for ${source.name}. Trying next source...`)
|
|
return // Will continue to next source in the loop
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Configure transformers.js to use verified local model
|
|
*/
|
|
private configureTransformers(): void {
|
|
env.localModelPath = this.modelPath
|
|
env.allowRemoteModels = false // Force local only after verification
|
|
console.log('🔒 Transformers configured to use verified local model')
|
|
}
|
|
|
|
/**
|
|
* Detect where models should be stored
|
|
*/
|
|
private detectModelPath(): string {
|
|
// Browser always uses default path
|
|
if (typeof window !== 'undefined') {
|
|
return './models'
|
|
}
|
|
|
|
// Use require for synchronous access in Node.js
|
|
try {
|
|
const fs = require('node:fs')
|
|
const path = require('node:path')
|
|
|
|
const candidates = [
|
|
process.env.BRAINY_MODELS_PATH,
|
|
'./models',
|
|
path.join(process.cwd(), 'models'),
|
|
path.join(process.env.HOME || '', '.brainy', 'models'),
|
|
'/opt/models', // Lambda/container path
|
|
env.cacheDir
|
|
]
|
|
|
|
for (const candidatePath of candidates) {
|
|
if (candidatePath && fs.existsSync(candidatePath)) {
|
|
const modelPath = path.join(candidatePath, ...CRITICAL_MODEL_CONFIG.modelName.split('/'))
|
|
if (fs.existsSync(path.join(modelPath, 'onnx', 'model.onnx'))) {
|
|
return candidatePath // Return the models directory, not its parent
|
|
}
|
|
}
|
|
}
|
|
} catch (e) {
|
|
// If Node.js modules not available, return default
|
|
}
|
|
|
|
// Default
|
|
return './models'
|
|
}
|
|
|
|
/**
|
|
* Get model status for diagnostics
|
|
*/
|
|
async getStatus(): Promise<{
|
|
verified: boolean
|
|
path: string
|
|
lastVerification: Date | null
|
|
modelName: string
|
|
dimensions: number
|
|
}> {
|
|
return {
|
|
verified: this.isVerified,
|
|
path: this.modelPath,
|
|
lastVerification: this.lastVerification,
|
|
modelName: CRITICAL_MODEL_CONFIG.modelName,
|
|
dimensions: CRITICAL_MODEL_CONFIG.embeddingDimensions
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Force re-verification (for testing)
|
|
*/
|
|
async forceReverify(): Promise<void> {
|
|
this.isVerified = false
|
|
this.lastVerification = null
|
|
await this.ensureCriticalModel()
|
|
}
|
|
}
|
|
|
|
// Export singleton instance
|
|
export const modelGuardian = ModelGuardian.getInstance() |