refactor: simplify build system and improve model loading flexibility
- Remove Rollup bundling in favor of direct TypeScript compilation - Move from bundled models to dynamic model loading with configurable paths - Add Docker deployment examples and documentation - Implement robust model loader with fallback mechanisms - Update storage adapters for better cross-environment compatibility - Add comprehensive tests for model loading and package installation - Simplify package.json scripts and remove complex build configurations - Clean up deprecated demo files and old bundling scripts BREAKING CHANGE: Models are no longer bundled with the package. They are now loaded dynamically from CDN or custom paths.
This commit is contained in:
parent
89413ebec2
commit
52a43d51d4
51 changed files with 4835 additions and 8007 deletions
|
|
@ -1,308 +0,0 @@
|
|||
#!/usr/bin/env node
|
||||
|
||||
/* eslint-env node */
|
||||
/* eslint-disable no-console */
|
||||
|
||||
/**
|
||||
* Demonstration: Optional Model Bundling Package
|
||||
*
|
||||
* This script demonstrates how the @soulcraft/brainy-models package
|
||||
* provides maximum reliability by eliminating network dependencies
|
||||
* for model loading.
|
||||
*
|
||||
* Original Issue: "When the Brainy library is used by other libraries,
|
||||
* there are always problems loading the model - it takes a long time to load,
|
||||
* times out, or fails completely."
|
||||
*
|
||||
* Solution: Optional separate package @soulcraft/brainy-models for maximum reliability
|
||||
*/
|
||||
|
||||
import fs from 'fs'
|
||||
import path from 'path'
|
||||
import { fileURLToPath } from 'url'
|
||||
|
||||
const __filename = fileURLToPath(import.meta.url)
|
||||
const __dirname = path.dirname(__filename)
|
||||
|
||||
console.log('🚀 Demonstration: Optional Model Bundling Package')
|
||||
console.log('='.repeat(60))
|
||||
console.log()
|
||||
|
||||
/**
|
||||
* Simulate the original problem with online model loading
|
||||
*/
|
||||
async function simulateOnlineModelLoadingProblems() {
|
||||
console.log('❌ PROBLEM: Online Model Loading Issues')
|
||||
console.log('─'.repeat(40))
|
||||
|
||||
const problems = [
|
||||
'🐌 Slow loading: 30-60 seconds on first use',
|
||||
'⏰ Timeouts: Network requests fail after timeout',
|
||||
'🌐 Network dependency: Requires internet connection',
|
||||
'💥 Complete failures: TensorFlow Hub unavailable',
|
||||
'🔄 Inconsistent performance: Variable load times',
|
||||
'📡 Offline issues: Cannot work without internet'
|
||||
]
|
||||
|
||||
for (const problem of problems) {
|
||||
console.log(` ${problem}`)
|
||||
await new Promise((resolve) => setTimeout(resolve, 500)) // Simulate delay
|
||||
}
|
||||
|
||||
console.log()
|
||||
console.log(
|
||||
'💡 These issues make Brainy unreliable when used by other libraries!'
|
||||
)
|
||||
console.log()
|
||||
}
|
||||
|
||||
/**
|
||||
* Demonstrate the solution with bundled models
|
||||
*/
|
||||
async function demonstrateBundledModelSolution() {
|
||||
console.log('✅ SOLUTION: Optional Model Bundling Package')
|
||||
console.log('─'.repeat(40))
|
||||
|
||||
const solutions = [
|
||||
'📦 Package: @soulcraft/brainy-models',
|
||||
'🔒 Maximum reliability: 100% offline operation',
|
||||
'⚡ Fast loading: < 1 second startup time',
|
||||
'🌐 No network dependency: Works completely offline',
|
||||
'📊 Consistent performance: Predictable load times',
|
||||
'🗜️ Multiple variants: Original, Float16, Int8 compressed',
|
||||
'💾 Local storage: ~25MB for complete model',
|
||||
'🛠️ Easy integration: Drop-in replacement'
|
||||
]
|
||||
|
||||
for (const solution of solutions) {
|
||||
console.log(` ${solution}`)
|
||||
await new Promise((resolve) => setTimeout(resolve, 300))
|
||||
}
|
||||
|
||||
console.log()
|
||||
}
|
||||
|
||||
/**
|
||||
* Show package structure and features
|
||||
*/
|
||||
function showPackageStructure() {
|
||||
console.log('📁 Package Structure')
|
||||
console.log('─'.repeat(20))
|
||||
|
||||
const packagePath = path.join(__dirname, 'brainy-models-package')
|
||||
|
||||
if (fs.existsSync(packagePath)) {
|
||||
console.log(' ✅ @soulcraft/brainy-models/')
|
||||
console.log(' ├── 📄 package.json (Package configuration)')
|
||||
console.log(' ├── 📖 README.md (Comprehensive documentation)')
|
||||
console.log(' ├── 🔧 tsconfig.json (TypeScript configuration)')
|
||||
console.log(' ├── 📂 src/')
|
||||
console.log(' │ └── 📄 index.ts (Main API)')
|
||||
console.log(' ├── 📂 scripts/')
|
||||
console.log(' │ ├── 📄 download-full-models.js (Model downloader)')
|
||||
console.log(' │ └── 📄 compress-models.js (Model compression)')
|
||||
console.log(' ├── 📂 test/')
|
||||
console.log(' │ └── 📄 test-models.js (Comprehensive tests)')
|
||||
console.log(' └── 📂 models/')
|
||||
console.log(' └── 📂 universal-sentence-encoder/')
|
||||
console.log(' ├── 📄 model.json (Model configuration)')
|
||||
console.log(' ├── 📄 metadata.json (Model metadata)')
|
||||
console.log(' ├── 📄 *.bin (Model weights)')
|
||||
console.log(' └── 📂 compressed/ (Optimized variants)')
|
||||
console.log()
|
||||
} else {
|
||||
console.log(' ⚠️ Package directory not found at expected location')
|
||||
console.log()
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Show installation and usage examples
|
||||
*/
|
||||
function showUsageExamples() {
|
||||
console.log('💻 Installation & Usage')
|
||||
console.log('─'.repeat(25))
|
||||
|
||||
console.log('📥 Installation:')
|
||||
console.log(' npm install @soulcraft/brainy-models')
|
||||
console.log()
|
||||
|
||||
console.log('🔧 Basic Usage:')
|
||||
console.log(` import { BundledUniversalSentenceEncoder } from '@soulcraft/brainy-models'
|
||||
|
||||
const encoder = new BundledUniversalSentenceEncoder({
|
||||
verbose: true,
|
||||
preferCompressed: false
|
||||
})
|
||||
|
||||
await encoder.load() // < 1 second, no network required!
|
||||
|
||||
const embeddings = await encoder.embedToArrays([
|
||||
'Hello world',
|
||||
'Machine learning is amazing'
|
||||
])
|
||||
|
||||
console.log('Generated embeddings:', embeddings.length)
|
||||
encoder.dispose()`)
|
||||
console.log()
|
||||
|
||||
console.log('🔗 Integration with Brainy:')
|
||||
console.log(` import Brainy from '@soulcraft/brainy'
|
||||
import { BundledUniversalSentenceEncoder } from '@soulcraft/brainy-models'
|
||||
|
||||
const bundledEncoder = new BundledUniversalSentenceEncoder()
|
||||
await bundledEncoder.load()
|
||||
|
||||
const brainy = new Brainy({
|
||||
customEmbedding: async (texts) => {
|
||||
return await bundledEncoder.embedToArrays(texts)
|
||||
}
|
||||
})
|
||||
|
||||
// Now Brainy uses bundled models - maximum reliability!`)
|
||||
console.log()
|
||||
}
|
||||
|
||||
/**
|
||||
* Show model compression features
|
||||
*/
|
||||
function showCompressionFeatures() {
|
||||
console.log('🗜️ Model Compression & Optimization')
|
||||
console.log('─'.repeat(35))
|
||||
|
||||
const variants = [
|
||||
{
|
||||
name: 'Original (Float32)',
|
||||
size: '~25MB',
|
||||
accuracy: 'Maximum',
|
||||
memory: 'High',
|
||||
useCase: 'Production applications'
|
||||
},
|
||||
{
|
||||
name: 'Float16 Compressed',
|
||||
size: '~12-15MB',
|
||||
accuracy: 'Very High',
|
||||
memory: 'Medium',
|
||||
useCase: 'Balanced performance'
|
||||
},
|
||||
{
|
||||
name: 'Int8 Quantized',
|
||||
size: '~6-8MB',
|
||||
accuracy: 'High',
|
||||
memory: 'Low',
|
||||
useCase: 'Memory-constrained'
|
||||
}
|
||||
]
|
||||
|
||||
for (const variant of variants) {
|
||||
console.log(` 📊 ${variant.name}`)
|
||||
console.log(` Size: ${variant.size}`)
|
||||
console.log(` Accuracy: ${variant.accuracy}`)
|
||||
console.log(` Memory: ${variant.memory}`)
|
||||
console.log(` Use case: ${variant.useCase}`)
|
||||
console.log()
|
||||
}
|
||||
|
||||
console.log('🎯 Optimization Scripts:')
|
||||
console.log(' npm run download-models # Download full models')
|
||||
console.log(' npm run compress-models # Create optimized variants')
|
||||
console.log(' npm test # Verify functionality')
|
||||
console.log()
|
||||
}
|
||||
|
||||
/**
|
||||
* Show reliability comparison
|
||||
*/
|
||||
function showReliabilityComparison() {
|
||||
console.log('📊 Reliability Comparison')
|
||||
console.log('─'.repeat(25))
|
||||
|
||||
const comparison = [
|
||||
['Feature', 'Online Loading', 'Bundled Models'],
|
||||
['─'.repeat(15), '─'.repeat(15), '─'.repeat(15)],
|
||||
['Reliability', 'Network dependent', '100% offline ✅'],
|
||||
['First load time', '30-60 seconds', '< 1 second ✅'],
|
||||
['Subsequent loads', 'Cached (~1s)', '< 1 second ✅'],
|
||||
['Package size', '~3KB ✅', '~25MB'],
|
||||
['Network required', 'Yes (first time)', 'No ✅'],
|
||||
['Offline support', 'Limited', 'Complete ✅'],
|
||||
['Startup time', 'Variable', 'Consistent ✅'],
|
||||
['Memory usage', 'Standard', 'Configurable ✅']
|
||||
]
|
||||
|
||||
for (const row of comparison) {
|
||||
console.log(` ${row[0].padEnd(17)} ${row[1].padEnd(17)} ${row[2]}`)
|
||||
}
|
||||
console.log()
|
||||
}
|
||||
|
||||
/**
|
||||
* Show when to use each approach
|
||||
*/
|
||||
function showWhenToUse() {
|
||||
console.log('🎯 When to Use Each Approach')
|
||||
console.log('─'.repeat(30))
|
||||
|
||||
console.log('✅ Use Bundled Models When:')
|
||||
const bundledUseCases = [
|
||||
'Production applications requiring maximum reliability',
|
||||
'Offline or air-gapped environments',
|
||||
'Applications with strict SLA requirements',
|
||||
'Edge computing and IoT devices',
|
||||
'Development environments with unreliable internet'
|
||||
]
|
||||
|
||||
for (const useCase of bundledUseCases) {
|
||||
console.log(` • ${useCase}`)
|
||||
}
|
||||
console.log()
|
||||
|
||||
console.log('✅ Use Online Loading When:')
|
||||
const onlineUseCases = [
|
||||
'Development and prototyping',
|
||||
'Applications where package size matters',
|
||||
'Environments with reliable internet connectivity',
|
||||
'Applications that rarely use embeddings'
|
||||
]
|
||||
|
||||
for (const useCase of onlineUseCases) {
|
||||
console.log(` • ${useCase}`)
|
||||
}
|
||||
console.log()
|
||||
}
|
||||
|
||||
/**
|
||||
* Main demonstration
|
||||
*/
|
||||
async function runDemo() {
|
||||
try {
|
||||
await simulateOnlineModelLoadingProblems()
|
||||
await demonstrateBundledModelSolution()
|
||||
showPackageStructure()
|
||||
showUsageExamples()
|
||||
showCompressionFeatures()
|
||||
showReliabilityComparison()
|
||||
showWhenToUse()
|
||||
|
||||
console.log('🎉 Summary')
|
||||
console.log('─'.repeat(10))
|
||||
console.log(
|
||||
'The @soulcraft/brainy-models package solves the original reliability'
|
||||
)
|
||||
console.log('issues by providing:')
|
||||
console.log()
|
||||
console.log(' ✅ Complete offline operation (no network dependencies)')
|
||||
console.log(' ✅ Fast, consistent loading times (< 1 second)')
|
||||
console.log(' ✅ Multiple optimized variants for different use cases')
|
||||
console.log(' ✅ Easy integration with existing Brainy applications')
|
||||
console.log(' ✅ Comprehensive documentation and examples')
|
||||
console.log()
|
||||
console.log('🚀 Ready for production use with maximum reliability!')
|
||||
} catch (error) {
|
||||
console.error('❌ Demo failed:', error)
|
||||
process.exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
// Run the demonstration
|
||||
runDemo().catch(console.error)
|
||||
201
scripts/extract-models.js
Normal file
201
scripts/extract-models.js
Normal file
|
|
@ -0,0 +1,201 @@
|
|||
#!/usr/bin/env node
|
||||
|
||||
/**
|
||||
* Extract Brainy Models Script
|
||||
*
|
||||
* Automatically extracts models from @soulcraft/brainy-models during Docker builds
|
||||
* Works across all cloud providers (Google Cloud Run, AWS Lambda/ECS, Azure Container Instances, Cloudflare Workers)
|
||||
*/
|
||||
|
||||
import { existsSync, mkdirSync, cpSync, readFileSync, writeFileSync } from 'fs'
|
||||
import { join, dirname } from 'path'
|
||||
import { fileURLToPath } from 'url'
|
||||
|
||||
const __filename = fileURLToPath(import.meta.url)
|
||||
const __dirname = dirname(__filename)
|
||||
|
||||
function log(message) {
|
||||
console.log(`[Brainy Model Extractor] ${message}`)
|
||||
}
|
||||
|
||||
async function extractModels() {
|
||||
try {
|
||||
log('🔍 Checking for @soulcraft/brainy-models...')
|
||||
|
||||
// Get the project root (one level up from scripts/)
|
||||
const projectRoot = join(__dirname, '..')
|
||||
const modelsPackagePath = join(projectRoot, 'node_modules', '@soulcraft', 'brainy-models')
|
||||
|
||||
if (!existsSync(modelsPackagePath)) {
|
||||
log('⚠️ @soulcraft/brainy-models not found - skipping model extraction')
|
||||
log(' Models will be downloaded at runtime (slower startup)')
|
||||
return false
|
||||
}
|
||||
|
||||
log('✅ Found @soulcraft/brainy-models package')
|
||||
|
||||
// Create the models directory in the project root
|
||||
const targetModelsDir = join(projectRoot, 'models')
|
||||
|
||||
if (existsSync(targetModelsDir)) {
|
||||
log('📁 Models directory already exists - removing old version')
|
||||
// Remove existing models directory to ensure clean extraction
|
||||
try {
|
||||
import('fs').then(fs => {
|
||||
fs.rmSync(targetModelsDir, { recursive: true, force: true })
|
||||
})
|
||||
} catch (error) {
|
||||
log(`⚠️ Could not remove existing models directory: ${error.message}`)
|
||||
}
|
||||
}
|
||||
|
||||
log('📦 Creating models directory...')
|
||||
mkdirSync(targetModelsDir, { recursive: true })
|
||||
|
||||
// Look for models in the package
|
||||
const possibleModelsPaths = [
|
||||
join(modelsPackagePath, 'models'),
|
||||
join(modelsPackagePath, 'dist', 'models'),
|
||||
modelsPackagePath // Root of the package
|
||||
]
|
||||
|
||||
let modelsSourcePath = null
|
||||
for (const path of possibleModelsPaths) {
|
||||
if (existsSync(path)) {
|
||||
// Check if this directory contains model files
|
||||
try {
|
||||
const fs = await import('fs')
|
||||
const files = fs.readdirSync(path)
|
||||
if (files.length > 0) {
|
||||
modelsSourcePath = path
|
||||
break
|
||||
}
|
||||
} catch (error) {
|
||||
continue
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (!modelsSourcePath) {
|
||||
log('❌ Could not find models in @soulcraft/brainy-models package')
|
||||
return false
|
||||
}
|
||||
|
||||
log(`📋 Copying models from: ${modelsSourcePath}`)
|
||||
log(`📋 Copying models to: ${targetModelsDir}`)
|
||||
|
||||
// Copy all models
|
||||
try {
|
||||
cpSync(modelsSourcePath, targetModelsDir, {
|
||||
recursive: true,
|
||||
force: true,
|
||||
filter: (src, dest) => {
|
||||
// Skip node_modules and other unnecessary files
|
||||
const filename = src.split('/').pop() || ''
|
||||
return !filename.startsWith('.') && filename !== 'node_modules'
|
||||
}
|
||||
})
|
||||
|
||||
log('✅ Models extracted successfully!')
|
||||
|
||||
// Create a marker file to indicate successful extraction
|
||||
const markerFile = join(targetModelsDir, '.brainy-models-extracted')
|
||||
writeFileSync(markerFile, JSON.stringify({
|
||||
extractedAt: new Date().toISOString(),
|
||||
sourcePackage: '@soulcraft/brainy-models',
|
||||
extractorVersion: '1.0.0'
|
||||
}, null, 2))
|
||||
|
||||
// List extracted models
|
||||
try {
|
||||
const fs = await import('fs')
|
||||
const extractedItems = fs.readdirSync(targetModelsDir)
|
||||
log(`📊 Extracted items: ${extractedItems.join(', ')}`)
|
||||
} catch (error) {
|
||||
log('📊 Model extraction completed (could not list contents)')
|
||||
}
|
||||
|
||||
return true
|
||||
|
||||
} catch (error) {
|
||||
log(`❌ Failed to copy models: ${error.message}`)
|
||||
return false
|
||||
}
|
||||
|
||||
} catch (error) {
|
||||
log(`❌ Model extraction failed: ${error.message}`)
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
// Auto-detect environment and provide helpful information
|
||||
function detectEnvironment() {
|
||||
const envs = []
|
||||
|
||||
// Docker detection
|
||||
if (existsSync('/.dockerenv') || process.env.DOCKER_CONTAINER) {
|
||||
envs.push('Docker')
|
||||
}
|
||||
|
||||
// Cloud provider detection
|
||||
if (process.env.GOOGLE_CLOUD_PROJECT || process.env.GAE_SERVICE) {
|
||||
envs.push('Google Cloud')
|
||||
}
|
||||
|
||||
if (process.env.AWS_EXECUTION_ENV || process.env.AWS_LAMBDA_FUNCTION_NAME) {
|
||||
envs.push('AWS')
|
||||
}
|
||||
|
||||
if (process.env.AZURE_CLIENT_ID || process.env.WEBSITE_SITE_NAME) {
|
||||
envs.push('Azure')
|
||||
}
|
||||
|
||||
if (process.env.CF_PAGES || process.env.CLOUDFLARE_ACCOUNT_ID) {
|
||||
envs.push('Cloudflare')
|
||||
}
|
||||
|
||||
if (process.env.VERCEL || process.env.VERCEL_ENV) {
|
||||
envs.push('Vercel')
|
||||
}
|
||||
|
||||
if (process.env.NETLIFY || process.env.NETLIFY_BUILD_BASE) {
|
||||
envs.push('Netlify')
|
||||
}
|
||||
|
||||
return envs
|
||||
}
|
||||
|
||||
// Main execution
|
||||
async function main() {
|
||||
log('🚀 Starting Brainy model extraction...')
|
||||
|
||||
const detectedEnvs = detectEnvironment()
|
||||
if (detectedEnvs.length > 0) {
|
||||
log(`🌐 Detected environment(s): ${detectedEnvs.join(', ')}`)
|
||||
}
|
||||
|
||||
const success = await extractModels()
|
||||
|
||||
if (success) {
|
||||
log('🎉 Model extraction completed successfully!')
|
||||
log('💡 Models are now embedded in your container/deployment')
|
||||
log('💡 No runtime model downloads required!')
|
||||
|
||||
// Set environment variable hint for runtime
|
||||
log('💡 Runtime will automatically detect extracted models')
|
||||
} else {
|
||||
log('⚠️ Model extraction failed or skipped')
|
||||
log('💡 Application will fall back to runtime model downloads')
|
||||
log('💡 Consider installing @soulcraft/brainy-models for better performance')
|
||||
}
|
||||
}
|
||||
|
||||
// Run if called directly
|
||||
if (import.meta.url === `file://${process.argv[1]}`) {
|
||||
main().catch(error => {
|
||||
console.error('Fatal error:', error)
|
||||
process.exit(1)
|
||||
})
|
||||
}
|
||||
|
||||
export { extractModels, detectEnvironment }
|
||||
Loading…
Add table
Add a link
Reference in a new issue