David Snelling
26c7d61185
Current state: - Unified augmentation system to BrainyAugmentation interface - Changed methods to specific noun/verb naming (addNoun, getNoun, etc) - Made old methods private - Combined getNouns into single unified method - Neural API exists and is complete - Triple Intelligence uses correct Brainy operators (not MongoDB) Issues identified: - Documentation incorrectly shows MongoDB operators (code is correct) - Need to ensure all features are properly exposed - Need to verify nothing was lost in simplification This commit serves as a rollback point before applying fixes.
108 lines
No EOL
2.9 KiB
JavaScript
108 lines
No EOL
2.9 KiB
JavaScript
#!/usr/bin/env node
|
|
/**
|
|
* Ensures transformer models are available for production
|
|
* This script handles model availability in multiple ways:
|
|
* 1. Check if models exist locally
|
|
* 2. Download from CDN if needed
|
|
* 3. Verify model integrity
|
|
*/
|
|
|
|
import { existsSync } from 'fs'
|
|
import { readFile, mkdir, writeFile } from 'fs/promises'
|
|
import { join, dirname } from 'path'
|
|
import { createHash } from 'crypto'
|
|
import { fileURLToPath } from 'url'
|
|
|
|
const __dirname = dirname(fileURLToPath(import.meta.url))
|
|
const PROJECT_ROOT = join(__dirname, '..')
|
|
|
|
// Model configuration
|
|
const MODEL_CONFIG = {
|
|
name: 'Xenova/all-MiniLM-L6-v2',
|
|
files: {
|
|
'onnx/model.onnx': {
|
|
size: 90555481, // 86.3 MB
|
|
sha256: 'expected_hash_here' // We'd compute this from actual model
|
|
},
|
|
'tokenizer.json': {
|
|
size: 711661,
|
|
sha256: 'expected_hash_here'
|
|
},
|
|
'tokenizer_config.json': {
|
|
size: 366,
|
|
sha256: 'expected_hash_here'
|
|
},
|
|
'config.json': {
|
|
size: 650,
|
|
sha256: 'expected_hash_here'
|
|
}
|
|
}
|
|
}
|
|
|
|
// CDN URLs for model files (would be your own CDN in production)
|
|
const CDN_BASE = 'https://cdn.soulcraft.com/models'
|
|
|
|
async function ensureModels() {
|
|
const modelsDir = join(PROJECT_ROOT, 'models', 'Xenova', 'all-MiniLM-L6-v2')
|
|
|
|
console.log('🔍 Checking for transformer models...')
|
|
|
|
// Check if all model files exist
|
|
let missingFiles = []
|
|
for (const [filePath, info] of Object.entries(MODEL_CONFIG.files)) {
|
|
const fullPath = join(modelsDir, filePath)
|
|
if (!existsSync(fullPath)) {
|
|
missingFiles.push(filePath)
|
|
}
|
|
}
|
|
|
|
if (missingFiles.length === 0) {
|
|
console.log('✅ All model files present')
|
|
|
|
// Optionally verify integrity
|
|
if (process.env.VERIFY_MODELS === 'true') {
|
|
console.log('🔐 Verifying model integrity...')
|
|
// Add hash verification here
|
|
}
|
|
|
|
return true
|
|
}
|
|
|
|
console.log(`⚠️ Missing ${missingFiles.length} model files`)
|
|
|
|
// In production, models should be pre-bundled
|
|
if (process.env.NODE_ENV === 'production' && !process.env.ALLOW_MODEL_DOWNLOAD) {
|
|
throw new Error(
|
|
'Critical: Transformer models not found in production. ' +
|
|
'Run "npm run download-models" during build stage.'
|
|
)
|
|
}
|
|
|
|
// Development: offer to download
|
|
if (process.env.CI !== 'true') {
|
|
console.log('📥 Would download models from CDN in development')
|
|
console.log(' Run: npm run download-models')
|
|
}
|
|
|
|
return false
|
|
}
|
|
|
|
// Export for use in main code
|
|
export async function verifyModelsAvailable() {
|
|
try {
|
|
return await ensureModels()
|
|
} catch (error) {
|
|
console.error('❌ Model verification failed:', error.message)
|
|
return false
|
|
}
|
|
}
|
|
|
|
// Run if called directly
|
|
if (import.meta.url === `file://${process.argv[1]}`) {
|
|
ensureModels()
|
|
.then(success => process.exit(success ? 0 : 1))
|
|
.catch(error => {
|
|
console.error(error)
|
|
process.exit(1)
|
|
})
|
|
} |