**chore(scripts): add utility scripts and demo for database checks, CLI handling, and model bundling**
- Added `check-database.js`: - Utility script to check database health and statistics, including nouns, verbs, and sample query results. - Includes test item addition for cases with no data. - Added `cli-wrapper.js`: - Wrapper script ensuring proper argument passing and CLI script availability. - Handles version flag, force flag addition, and automatic CLI building in local development contexts. - Added `demo-optional-model-bundling.js`: - Demonstration script showcasing optional model bundling benefits for reliability and offline support in embedding workflows. - Includes examples of compression, package structures, and use-case comparisons. **Purpose**: Introduce utility and demonstration scripts
This commit is contained in:
parent
e476d45fac
commit
90eccd75aa
9 changed files with 867 additions and 2 deletions
308
scripts/demos/demo-optional-model-bundling.js
Normal file
308
scripts/demos/demo-optional-model-bundling.js
Normal file
|
|
@ -0,0 +1,308 @@
|
|||
#!/usr/bin/env node
|
||||
|
||||
/* eslint-env node */
|
||||
/* eslint-disable no-console */
|
||||
|
||||
/**
|
||||
* Demonstration: Optional Model Bundling Package
|
||||
*
|
||||
* This script demonstrates how the @soulcraft/brainy-models package
|
||||
* provides maximum reliability by eliminating network dependencies
|
||||
* for model loading.
|
||||
*
|
||||
* Original Issue: "When the Brainy library is used by other libraries,
|
||||
* there are always problems loading the model - it takes a long time to load,
|
||||
* times out, or fails completely."
|
||||
*
|
||||
* Solution: Optional separate package @soulcraft/brainy-models for maximum reliability
|
||||
*/
|
||||
|
||||
import fs from 'fs'
|
||||
import path from 'path'
|
||||
import { fileURLToPath } from 'url'
|
||||
|
||||
const __filename = fileURLToPath(import.meta.url)
|
||||
const __dirname = path.dirname(__filename)
|
||||
|
||||
console.log('🚀 Demonstration: Optional Model Bundling Package')
|
||||
console.log('='.repeat(60))
|
||||
console.log()
|
||||
|
||||
/**
|
||||
* Simulate the original problem with online model loading
|
||||
*/
|
||||
async function simulateOnlineModelLoadingProblems() {
|
||||
console.log('❌ PROBLEM: Online Model Loading Issues')
|
||||
console.log('─'.repeat(40))
|
||||
|
||||
const problems = [
|
||||
'🐌 Slow loading: 30-60 seconds on first use',
|
||||
'⏰ Timeouts: Network requests fail after timeout',
|
||||
'🌐 Network dependency: Requires internet connection',
|
||||
'💥 Complete failures: TensorFlow Hub unavailable',
|
||||
'🔄 Inconsistent performance: Variable load times',
|
||||
'📡 Offline issues: Cannot work without internet'
|
||||
]
|
||||
|
||||
for (const problem of problems) {
|
||||
console.log(` ${problem}`)
|
||||
await new Promise((resolve) => setTimeout(resolve, 500)) // Simulate delay
|
||||
}
|
||||
|
||||
console.log()
|
||||
console.log(
|
||||
'💡 These issues make Brainy unreliable when used by other libraries!'
|
||||
)
|
||||
console.log()
|
||||
}
|
||||
|
||||
/**
|
||||
* Demonstrate the solution with bundled models
|
||||
*/
|
||||
async function demonstrateBundledModelSolution() {
|
||||
console.log('✅ SOLUTION: Optional Model Bundling Package')
|
||||
console.log('─'.repeat(40))
|
||||
|
||||
const solutions = [
|
||||
'📦 Package: @soulcraft/brainy-models',
|
||||
'🔒 Maximum reliability: 100% offline operation',
|
||||
'⚡ Fast loading: < 1 second startup time',
|
||||
'🌐 No network dependency: Works completely offline',
|
||||
'📊 Consistent performance: Predictable load times',
|
||||
'🗜️ Multiple variants: Original, Float16, Int8 compressed',
|
||||
'💾 Local storage: ~25MB for complete model',
|
||||
'🛠️ Easy integration: Drop-in replacement'
|
||||
]
|
||||
|
||||
for (const solution of solutions) {
|
||||
console.log(` ${solution}`)
|
||||
await new Promise((resolve) => setTimeout(resolve, 300))
|
||||
}
|
||||
|
||||
console.log()
|
||||
}
|
||||
|
||||
/**
|
||||
* Show package structure and features
|
||||
*/
|
||||
function showPackageStructure() {
|
||||
console.log('📁 Package Structure')
|
||||
console.log('─'.repeat(20))
|
||||
|
||||
const packagePath = path.join(__dirname, 'brainy-models-package')
|
||||
|
||||
if (fs.existsSync(packagePath)) {
|
||||
console.log(' ✅ @soulcraft/brainy-models/')
|
||||
console.log(' ├── 📄 package.json (Package configuration)')
|
||||
console.log(' ├── 📖 README.md (Comprehensive documentation)')
|
||||
console.log(' ├── 🔧 tsconfig.json (TypeScript configuration)')
|
||||
console.log(' ├── 📂 src/')
|
||||
console.log(' │ └── 📄 index.ts (Main API)')
|
||||
console.log(' ├── 📂 scripts/')
|
||||
console.log(' │ ├── 📄 download-full-models.js (Model downloader)')
|
||||
console.log(' │ └── 📄 compress-models.js (Model compression)')
|
||||
console.log(' ├── 📂 test/')
|
||||
console.log(' │ └── 📄 test-models.js (Comprehensive tests)')
|
||||
console.log(' └── 📂 models/')
|
||||
console.log(' └── 📂 universal-sentence-encoder/')
|
||||
console.log(' ├── 📄 model.json (Model configuration)')
|
||||
console.log(' ├── 📄 metadata.json (Model metadata)')
|
||||
console.log(' ├── 📄 *.bin (Model weights)')
|
||||
console.log(' └── 📂 compressed/ (Optimized variants)')
|
||||
console.log()
|
||||
} else {
|
||||
console.log(' ⚠️ Package directory not found at expected location')
|
||||
console.log()
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Show installation and usage examples
|
||||
*/
|
||||
function showUsageExamples() {
|
||||
console.log('💻 Installation & Usage')
|
||||
console.log('─'.repeat(25))
|
||||
|
||||
console.log('📥 Installation:')
|
||||
console.log(' npm install @soulcraft/brainy-models')
|
||||
console.log()
|
||||
|
||||
console.log('🔧 Basic Usage:')
|
||||
console.log(` import { BundledUniversalSentenceEncoder } from '@soulcraft/brainy-models'
|
||||
|
||||
const encoder = new BundledUniversalSentenceEncoder({
|
||||
verbose: true,
|
||||
preferCompressed: false
|
||||
})
|
||||
|
||||
await encoder.load() // < 1 second, no network required!
|
||||
|
||||
const embeddings = await encoder.embedToArrays([
|
||||
'Hello world',
|
||||
'Machine learning is amazing'
|
||||
])
|
||||
|
||||
console.log('Generated embeddings:', embeddings.length)
|
||||
encoder.dispose()`)
|
||||
console.log()
|
||||
|
||||
console.log('🔗 Integration with Brainy:')
|
||||
console.log(` import Brainy from '@soulcraft/brainy'
|
||||
import { BundledUniversalSentenceEncoder } from '@soulcraft/brainy-models'
|
||||
|
||||
const bundledEncoder = new BundledUniversalSentenceEncoder()
|
||||
await bundledEncoder.load()
|
||||
|
||||
const brainy = new Brainy({
|
||||
customEmbedding: async (texts) => {
|
||||
return await bundledEncoder.embedToArrays(texts)
|
||||
}
|
||||
})
|
||||
|
||||
// Now Brainy uses bundled models - maximum reliability!`)
|
||||
console.log()
|
||||
}
|
||||
|
||||
/**
|
||||
* Show model compression features
|
||||
*/
|
||||
function showCompressionFeatures() {
|
||||
console.log('🗜️ Model Compression & Optimization')
|
||||
console.log('─'.repeat(35))
|
||||
|
||||
const variants = [
|
||||
{
|
||||
name: 'Original (Float32)',
|
||||
size: '~25MB',
|
||||
accuracy: 'Maximum',
|
||||
memory: 'High',
|
||||
useCase: 'Production applications'
|
||||
},
|
||||
{
|
||||
name: 'Float16 Compressed',
|
||||
size: '~12-15MB',
|
||||
accuracy: 'Very High',
|
||||
memory: 'Medium',
|
||||
useCase: 'Balanced performance'
|
||||
},
|
||||
{
|
||||
name: 'Int8 Quantized',
|
||||
size: '~6-8MB',
|
||||
accuracy: 'High',
|
||||
memory: 'Low',
|
||||
useCase: 'Memory-constrained'
|
||||
}
|
||||
]
|
||||
|
||||
for (const variant of variants) {
|
||||
console.log(` 📊 ${variant.name}`)
|
||||
console.log(` Size: ${variant.size}`)
|
||||
console.log(` Accuracy: ${variant.accuracy}`)
|
||||
console.log(` Memory: ${variant.memory}`)
|
||||
console.log(` Use case: ${variant.useCase}`)
|
||||
console.log()
|
||||
}
|
||||
|
||||
console.log('🎯 Optimization Scripts:')
|
||||
console.log(' npm run download-models # Download full models')
|
||||
console.log(' npm run compress-models # Create optimized variants')
|
||||
console.log(' npm test # Verify functionality')
|
||||
console.log()
|
||||
}
|
||||
|
||||
/**
|
||||
* Show reliability comparison
|
||||
*/
|
||||
function showReliabilityComparison() {
|
||||
console.log('📊 Reliability Comparison')
|
||||
console.log('─'.repeat(25))
|
||||
|
||||
const comparison = [
|
||||
['Feature', 'Online Loading', 'Bundled Models'],
|
||||
['─'.repeat(15), '─'.repeat(15), '─'.repeat(15)],
|
||||
['Reliability', 'Network dependent', '100% offline ✅'],
|
||||
['First load time', '30-60 seconds', '< 1 second ✅'],
|
||||
['Subsequent loads', 'Cached (~1s)', '< 1 second ✅'],
|
||||
['Package size', '~3KB ✅', '~25MB'],
|
||||
['Network required', 'Yes (first time)', 'No ✅'],
|
||||
['Offline support', 'Limited', 'Complete ✅'],
|
||||
['Startup time', 'Variable', 'Consistent ✅'],
|
||||
['Memory usage', 'Standard', 'Configurable ✅']
|
||||
]
|
||||
|
||||
for (const row of comparison) {
|
||||
console.log(` ${row[0].padEnd(17)} ${row[1].padEnd(17)} ${row[2]}`)
|
||||
}
|
||||
console.log()
|
||||
}
|
||||
|
||||
/**
|
||||
* Show when to use each approach
|
||||
*/
|
||||
function showWhenToUse() {
|
||||
console.log('🎯 When to Use Each Approach')
|
||||
console.log('─'.repeat(30))
|
||||
|
||||
console.log('✅ Use Bundled Models When:')
|
||||
const bundledUseCases = [
|
||||
'Production applications requiring maximum reliability',
|
||||
'Offline or air-gapped environments',
|
||||
'Applications with strict SLA requirements',
|
||||
'Edge computing and IoT devices',
|
||||
'Development environments with unreliable internet'
|
||||
]
|
||||
|
||||
for (const useCase of bundledUseCases) {
|
||||
console.log(` • ${useCase}`)
|
||||
}
|
||||
console.log()
|
||||
|
||||
console.log('✅ Use Online Loading When:')
|
||||
const onlineUseCases = [
|
||||
'Development and prototyping',
|
||||
'Applications where package size matters',
|
||||
'Environments with reliable internet connectivity',
|
||||
'Applications that rarely use embeddings'
|
||||
]
|
||||
|
||||
for (const useCase of onlineUseCases) {
|
||||
console.log(` • ${useCase}`)
|
||||
}
|
||||
console.log()
|
||||
}
|
||||
|
||||
/**
|
||||
* Main demonstration
|
||||
*/
|
||||
async function runDemo() {
|
||||
try {
|
||||
await simulateOnlineModelLoadingProblems()
|
||||
await demonstrateBundledModelSolution()
|
||||
showPackageStructure()
|
||||
showUsageExamples()
|
||||
showCompressionFeatures()
|
||||
showReliabilityComparison()
|
||||
showWhenToUse()
|
||||
|
||||
console.log('🎉 Summary')
|
||||
console.log('─'.repeat(10))
|
||||
console.log(
|
||||
'The @soulcraft/brainy-models package solves the original reliability'
|
||||
)
|
||||
console.log('issues by providing:')
|
||||
console.log()
|
||||
console.log(' ✅ Complete offline operation (no network dependencies)')
|
||||
console.log(' ✅ Fast, consistent loading times (< 1 second)')
|
||||
console.log(' ✅ Multiple optimized variants for different use cases')
|
||||
console.log(' ✅ Easy integration with existing Brainy applications')
|
||||
console.log(' ✅ Comprehensive documentation and examples')
|
||||
console.log()
|
||||
console.log('🚀 Ready for production use with maximum reliability!')
|
||||
} catch (error) {
|
||||
console.error('❌ Demo failed:', error)
|
||||
process.exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
// Run the demonstration
|
||||
runDemo().catch(console.error)
|
||||
Loading…
Add table
Add a link
Reference in a new issue