diff --git a/scripts/demos/demo-optional-model-bundling.js b/scripts/demos/demo-optional-model-bundling.js new file mode 100644 index 00000000..33486129 --- /dev/null +++ b/scripts/demos/demo-optional-model-bundling.js @@ -0,0 +1,308 @@ +#!/usr/bin/env node + +/* eslint-env node */ +/* eslint-disable no-console */ + +/** + * Demonstration: Optional Model Bundling Package + * + * This script demonstrates how the @soulcraft/brainy-models package + * provides maximum reliability by eliminating network dependencies + * for model loading. + * + * Original Issue: "When the Brainy library is used by other libraries, + * there are always problems loading the model - it takes a long time to load, + * times out, or fails completely." + * + * Solution: Optional separate package @soulcraft/brainy-models for maximum reliability + */ + +import fs from 'fs' +import path from 'path' +import { fileURLToPath } from 'url' + +const __filename = fileURLToPath(import.meta.url) +const __dirname = path.dirname(__filename) + +console.log('🚀 Demonstration: Optional Model Bundling Package') +console.log('='.repeat(60)) +console.log() + +/** + * Simulate the original problem with online model loading + */ +async function simulateOnlineModelLoadingProblems() { + console.log('❌ PROBLEM: Online Model Loading Issues') + console.log('─'.repeat(40)) + + const problems = [ + '🐌 Slow loading: 30-60 seconds on first use', + '⏰ Timeouts: Network requests fail after timeout', + '🌐 Network dependency: Requires internet connection', + '💥 Complete failures: TensorFlow Hub unavailable', + '🔄 Inconsistent performance: Variable load times', + '📡 Offline issues: Cannot work without internet' + ] + + for (const problem of problems) { + console.log(` ${problem}`) + await new Promise((resolve) => setTimeout(resolve, 500)) // Simulate delay + } + + console.log() + console.log( + '💡 These issues make Brainy unreliable when used by other libraries!' + ) + console.log() +} + +/** + * Demonstrate the solution with bundled models + */ +async function demonstrateBundledModelSolution() { + console.log('✅ SOLUTION: Optional Model Bundling Package') + console.log('─'.repeat(40)) + + const solutions = [ + '📦 Package: @soulcraft/brainy-models', + '🔒 Maximum reliability: 100% offline operation', + '⚡ Fast loading: < 1 second startup time', + '🌐 No network dependency: Works completely offline', + '📊 Consistent performance: Predictable load times', + '🗜️ Multiple variants: Original, Float16, Int8 compressed', + '💾 Local storage: ~25MB for complete model', + '🛠️ Easy integration: Drop-in replacement' + ] + + for (const solution of solutions) { + console.log(` ${solution}`) + await new Promise((resolve) => setTimeout(resolve, 300)) + } + + console.log() +} + +/** + * Show package structure and features + */ +function showPackageStructure() { + console.log('📁 Package Structure') + console.log('─'.repeat(20)) + + const packagePath = path.join(__dirname, 'brainy-models-package') + + if (fs.existsSync(packagePath)) { + console.log(' ✅ @soulcraft/brainy-models/') + console.log(' ├── 📄 package.json (Package configuration)') + console.log(' ├── 📖 README.md (Comprehensive documentation)') + console.log(' ├── 🔧 tsconfig.json (TypeScript configuration)') + console.log(' ├── 📂 src/') + console.log(' │ └── 📄 index.ts (Main API)') + console.log(' ├── 📂 scripts/') + console.log(' │ ├── 📄 download-full-models.js (Model downloader)') + console.log(' │ └── 📄 compress-models.js (Model compression)') + console.log(' ├── 📂 test/') + console.log(' │ └── 📄 test-models.js (Comprehensive tests)') + console.log(' └── 📂 models/') + console.log(' └── 📂 universal-sentence-encoder/') + console.log(' ├── 📄 model.json (Model configuration)') + console.log(' ├── 📄 metadata.json (Model metadata)') + console.log(' ├── 📄 *.bin (Model weights)') + console.log(' └── 📂 compressed/ (Optimized variants)') + console.log() + } else { + console.log(' ⚠️ Package directory not found at expected location') + console.log() + } +} + +/** + * Show installation and usage examples + */ +function showUsageExamples() { + console.log('💻 Installation & Usage') + console.log('─'.repeat(25)) + + console.log('📥 Installation:') + console.log(' npm install @soulcraft/brainy-models') + console.log() + + console.log('🔧 Basic Usage:') + console.log(` import { BundledUniversalSentenceEncoder } from '@soulcraft/brainy-models' + + const encoder = new BundledUniversalSentenceEncoder({ + verbose: true, + preferCompressed: false + }) + + await encoder.load() // < 1 second, no network required! + + const embeddings = await encoder.embedToArrays([ + 'Hello world', + 'Machine learning is amazing' + ]) + + console.log('Generated embeddings:', embeddings.length) + encoder.dispose()`) + console.log() + + console.log('🔗 Integration with Brainy:') + console.log(` import Brainy from '@soulcraft/brainy' + import { BundledUniversalSentenceEncoder } from '@soulcraft/brainy-models' + + const bundledEncoder = new BundledUniversalSentenceEncoder() + await bundledEncoder.load() + + const brainy = new Brainy({ + customEmbedding: async (texts) => { + return await bundledEncoder.embedToArrays(texts) + } + }) + + // Now Brainy uses bundled models - maximum reliability!`) + console.log() +} + +/** + * Show model compression features + */ +function showCompressionFeatures() { + console.log('🗜️ Model Compression & Optimization') + console.log('─'.repeat(35)) + + const variants = [ + { + name: 'Original (Float32)', + size: '~25MB', + accuracy: 'Maximum', + memory: 'High', + useCase: 'Production applications' + }, + { + name: 'Float16 Compressed', + size: '~12-15MB', + accuracy: 'Very High', + memory: 'Medium', + useCase: 'Balanced performance' + }, + { + name: 'Int8 Quantized', + size: '~6-8MB', + accuracy: 'High', + memory: 'Low', + useCase: 'Memory-constrained' + } + ] + + for (const variant of variants) { + console.log(` 📊 ${variant.name}`) + console.log(` Size: ${variant.size}`) + console.log(` Accuracy: ${variant.accuracy}`) + console.log(` Memory: ${variant.memory}`) + console.log(` Use case: ${variant.useCase}`) + console.log() + } + + console.log('🎯 Optimization Scripts:') + console.log(' npm run download-models # Download full models') + console.log(' npm run compress-models # Create optimized variants') + console.log(' npm test # Verify functionality') + console.log() +} + +/** + * Show reliability comparison + */ +function showReliabilityComparison() { + console.log('📊 Reliability Comparison') + console.log('─'.repeat(25)) + + const comparison = [ + ['Feature', 'Online Loading', 'Bundled Models'], + ['─'.repeat(15), '─'.repeat(15), '─'.repeat(15)], + ['Reliability', 'Network dependent', '100% offline ✅'], + ['First load time', '30-60 seconds', '< 1 second ✅'], + ['Subsequent loads', 'Cached (~1s)', '< 1 second ✅'], + ['Package size', '~3KB ✅', '~25MB'], + ['Network required', 'Yes (first time)', 'No ✅'], + ['Offline support', 'Limited', 'Complete ✅'], + ['Startup time', 'Variable', 'Consistent ✅'], + ['Memory usage', 'Standard', 'Configurable ✅'] + ] + + for (const row of comparison) { + console.log(` ${row[0].padEnd(17)} ${row[1].padEnd(17)} ${row[2]}`) + } + console.log() +} + +/** + * Show when to use each approach + */ +function showWhenToUse() { + console.log('🎯 When to Use Each Approach') + console.log('─'.repeat(30)) + + console.log('✅ Use Bundled Models When:') + const bundledUseCases = [ + 'Production applications requiring maximum reliability', + 'Offline or air-gapped environments', + 'Applications with strict SLA requirements', + 'Edge computing and IoT devices', + 'Development environments with unreliable internet' + ] + + for (const useCase of bundledUseCases) { + console.log(` • ${useCase}`) + } + console.log() + + console.log('✅ Use Online Loading When:') + const onlineUseCases = [ + 'Development and prototyping', + 'Applications where package size matters', + 'Environments with reliable internet connectivity', + 'Applications that rarely use embeddings' + ] + + for (const useCase of onlineUseCases) { + console.log(` • ${useCase}`) + } + console.log() +} + +/** + * Main demonstration + */ +async function runDemo() { + try { + await simulateOnlineModelLoadingProblems() + await demonstrateBundledModelSolution() + showPackageStructure() + showUsageExamples() + showCompressionFeatures() + showReliabilityComparison() + showWhenToUse() + + console.log('🎉 Summary') + console.log('─'.repeat(10)) + console.log( + 'The @soulcraft/brainy-models package solves the original reliability' + ) + console.log('issues by providing:') + console.log() + console.log(' ✅ Complete offline operation (no network dependencies)') + console.log(' ✅ Fast, consistent loading times (< 1 second)') + console.log(' ✅ Multiple optimized variants for different use cases') + console.log(' ✅ Easy integration with existing Brainy applications') + console.log(' ✅ Comprehensive documentation and examples') + console.log() + console.log('🚀 Ready for production use with maximum reliability!') + } catch (error) { + console.error('❌ Demo failed:', error) + process.exit(1) + } +} + +// Run the demonstration +runDemo().catch(console.error) diff --git a/scripts/development/cli-wrapper.js b/scripts/development/cli-wrapper.js new file mode 100755 index 00000000..648d64e9 --- /dev/null +++ b/scripts/development/cli-wrapper.js @@ -0,0 +1,85 @@ +#!/usr/bin/env node + +/** + * CLI Wrapper Script + * + * This script serves as a wrapper for the Brainy CLI, ensuring that command-line arguments + * are properly passed to the CLI when invoked through npm scripts. + */ + +import { spawn, execSync } from 'child_process' +import { fileURLToPath } from 'url' +import { dirname, join } from 'path' +import fs from 'fs' + + +// Get the directory of the current module +const __filename = fileURLToPath(import.meta.url) +const __dirname = dirname(__filename) + +// Path to the actual CLI script +const cliPath = join(__dirname, 'dist', 'cli.js') + +// Check if the CLI script exists +if (!fs.existsSync(cliPath)) { + // Check if we're running in a global installation context + const isGlobalInstall = __dirname.includes('node_modules') && !__dirname.includes('node_modules/.') + + if (isGlobalInstall) { + console.error(`Error: CLI script not found at ${cliPath}`) + console.error('This is likely because the CLI was not built during package installation.') + console.error('Please reinstall the package with:') + console.error('npm uninstall -g @soulcraft/brainy') + console.error('npm install -g @soulcraft/brainy --legacy-peer-deps') + process.exit(1) + } else { + // In a local development context, try to build the CLI + console.log(`CLI script not found at ${cliPath}. Building CLI...`) + + try { + // Run the build:cli script + execSync('npm run build:cli', { stdio: 'inherit' }) + + // Check again if the CLI script exists after building + if (!fs.existsSync(cliPath)) { + console.error(`Error: Failed to build CLI script at ${cliPath}`) + process.exit(1) + } + + console.log('CLI built successfully.') + } catch (error) { + console.error(`Error building CLI: ${error.message}`) + console.error('Make sure you have the necessary dependencies installed.') + process.exit(1) + } + } +} + +// Special handling for version flags +if (process.argv.includes('--version') || process.argv.includes('-V')) { + // Read version directly from package.json to ensure it's always correct + try { + const packageJsonPath = join(__dirname, 'package.json') + const packageJson = JSON.parse(fs.readFileSync(packageJsonPath, 'utf8')) + console.log(packageJson.version) + process.exit(0) + } catch (error) { + console.error('Error loading version information:', error.message) + process.exit(1) + } +} + +// Forward all arguments to the CLI script +const args = process.argv.slice(2) + +// Check if npm is passing --force flag +// When npm runs with --force, it sets the npm_config_force environment variable +if (process.env.npm_config_force === 'true' && args.includes('clear') && !args.includes('--force') && !args.includes('-f')) { + args.push('--force') +} + +const cli = spawn('node', [cliPath, ...args], { stdio: 'inherit' }) + +cli.on('close', (code) => { + process.exit(code) +}) diff --git a/scripts/development/encoded-image.html b/scripts/development/encoded-image.html new file mode 100644 index 00000000..cfc1a5e2 --- /dev/null +++ b/scripts/development/encoded-image.html @@ -0,0 +1 @@ +Brainy Logo \ No newline at end of file diff --git a/scripts/development/index.html b/scripts/development/index.html new file mode 100644 index 00000000..1018e443 --- /dev/null +++ b/scripts/development/index.html @@ -0,0 +1,17 @@ + + + + + + Brainy Interactive Demo - Redirecting... + + + + + +

Redirecting to Brainy Interactive Demo...

+

If you are not redirected automatically, please click the link above.

+ + diff --git a/scripts/development/test-browser-cache-detection.html b/scripts/development/test-browser-cache-detection.html new file mode 100644 index 00000000..f1da7637 --- /dev/null +++ b/scripts/development/test-browser-cache-detection.html @@ -0,0 +1,78 @@ + + + + + + Brainy Cache Detection Browser Test + + + +

Brainy Cache Detection Browser Test

+

This page tests if Brainy's cache detection works properly in browser environments.

+ + + +
+

Test results will appear here...

+
+ + + + diff --git a/scripts/development/test-worker-cache-detection.html b/scripts/development/test-worker-cache-detection.html new file mode 100644 index 00000000..a2c66f91 --- /dev/null +++ b/scripts/development/test-worker-cache-detection.html @@ -0,0 +1,130 @@ + + + + + + Brainy Cache Detection Worker Test + + + +

Brainy Cache Detection Worker Test

+

This page tests if Brainy's cache detection works properly in Web Worker environments.

+ + + +
+

Test results will appear here...

+
+ + + + diff --git a/scripts/maintenance/check-database.js b/scripts/maintenance/check-database.js new file mode 100644 index 00000000..14170bee --- /dev/null +++ b/scripts/maintenance/check-database.js @@ -0,0 +1,83 @@ +// Script to check if there's any data in the database +import { BrainyData } from './dist/brainyData.js'; + +async function checkDatabase() { + try { + console.log('Initializing BrainyData...'); + const db = new BrainyData(); + await db.init(); + + console.log('Getting database status...'); + const status = await db.status(); + console.log('Database status:', JSON.stringify(status, null, 2)); + + console.log('Getting statistics...'); + const stats = await db.getStatistics(); + console.log('Statistics:', JSON.stringify(stats, null, 2)); + + console.log('Getting all nouns...'); + const nouns = await db.getAllNouns(); + console.log(`Found ${nouns.length} nouns in the database.`); + + if (nouns.length > 0) { + console.log('Sample of nouns:'); + for (let i = 0; i < Math.min(5, nouns.length); i++) { + console.log(`Noun ${i + 1}:`, JSON.stringify(nouns[i], null, 2)); + } + } + + console.log('Getting all verbs...'); + const verbs = await db.getAllVerbs(); + console.log(`Found ${verbs.length} verbs in the database.`); + + if (verbs.length > 0) { + console.log('Sample of verbs:'); + for (let i = 0; i < Math.min(5, verbs.length); i++) { + console.log(`Verb ${i + 1}:`, JSON.stringify(verbs[i], null, 2)); + } + } + + // Try a simple search to see if it returns any results + console.log('Trying a simple search...'); + const searchResults = await db.searchText('test', 10); + console.log(`Search returned ${searchResults.length} results.`); + + if (searchResults.length > 0) { + console.log('Sample of search results:'); + for (let i = 0; i < Math.min(5, searchResults.length); i++) { + console.log(`Result ${i + 1}:`, JSON.stringify({ + id: searchResults[i].id, + score: searchResults[i].score, + metadata: searchResults[i].metadata + }, null, 2)); + } + } + + // If no results, try adding a test item and searching again + if (searchResults.length === 0 && nouns.length === 0) { + console.log('No data found. Adding a test item...'); + const id = await db.add('This is a test item for searching', { noun: 'Thing', category: 'test' }); + console.log(`Added test item with ID: ${id}`); + + console.log('Trying search again...'); + const newSearchResults = await db.searchText('test', 10); + console.log(`Search returned ${newSearchResults.length} results.`); + + if (newSearchResults.length > 0) { + console.log('Sample of search results:'); + for (let i = 0; i < Math.min(5, newSearchResults.length); i++) { + console.log(`Result ${i + 1}:`, JSON.stringify({ + id: newSearchResults[i].id, + score: newSearchResults[i].score, + metadata: newSearchResults[i].metadata + }, null, 2)); + } + } + } + + } catch (error) { + console.error('Error checking database:', error); + } +} + +checkDatabase().catch(console.error); \ No newline at end of file diff --git a/scripts/maintenance/fix-dimension-mismatch.js b/scripts/maintenance/fix-dimension-mismatch.js new file mode 100644 index 00000000..f19d997e --- /dev/null +++ b/scripts/maintenance/fix-dimension-mismatch.js @@ -0,0 +1,163 @@ +// Script to fix dimension mismatch by re-embedding existing data +import { BrainyData } from './dist/brainyData.js'; +import fs from 'fs'; +import path from 'path'; + +async function fixDimensionMismatch() { + try { + console.log('Starting dimension mismatch fix...'); + + // Create a backup of the existing data + const backupDir = './brainy-data-backup-' + Date.now(); + console.log(`Creating backup of existing data in ${backupDir}...`); + + // Copy the entire brainy-data directory to the backup directory + await fs.promises.mkdir(backupDir, { recursive: true }); + await copyDirectory('./brainy-data', backupDir); + console.log('Backup created successfully.'); + + // Initialize BrainyData with the current embedding function + console.log('Initializing BrainyData...'); + const db = new BrainyData(); + await db.init(); + + // Get database status to check if there's any data + const status = await db.status(); + console.log('Database status:', JSON.stringify(status, null, 2)); + + // Read all noun files directly from the filesystem + console.log('Reading noun files directly from filesystem...'); + const nounsDir = './brainy-data/nouns'; + const files = await fs.promises.readdir(nounsDir); + + // Process each noun file + const processedNouns = []; + for (const file of files) { + if (file.endsWith('.json')) { + const filePath = path.join(nounsDir, file); + const data = await fs.promises.readFile(filePath, 'utf-8'); + const parsedNoun = JSON.parse(data); + + // Get the metadata for this noun + const metadataPath = path.join('./brainy-data/metadata', `${parsedNoun.id}.json`); + let metadata = {}; + try { + const metadataData = await fs.promises.readFile(metadataPath, 'utf-8'); + metadata = JSON.parse(metadataData); + } catch (error) { + console.warn(`No metadata found for noun ${parsedNoun.id}`); + } + + // Extract text from metadata if available + let text = ''; + if (metadata.text) { + text = metadata.text; + } else if (metadata.description) { + text = metadata.description; + } else { + // If no text is available, use a placeholder + text = `Noun ${parsedNoun.id}`; + console.warn(`No text found for noun ${parsedNoun.id}, using placeholder`); + } + + // Re-embed the text using the current embedding function + console.log(`Re-embedding noun ${parsedNoun.id}...`); + try { + // Delete the existing noun first + await db.delete(parsedNoun.id); + + // Add the noun with the same ID but new vector + const newId = await db.add(text, metadata, { id: parsedNoun.id }); + processedNouns.push({ id: newId, originalId: parsedNoun.id }); + console.log(`Successfully re-embedded noun ${parsedNoun.id}`); + } catch (error) { + console.error(`Error re-embedding noun ${parsedNoun.id}:`, error); + } + } + } + + console.log(`Processed ${processedNouns.length} nouns.`); + + // Recreate verbs + console.log('Reading verb files directly from filesystem...'); + const verbsDir = './brainy-data/verbs'; + const verbFiles = await fs.promises.readdir(verbsDir); + + // Process each verb file + const processedVerbs = []; + for (const file of verbFiles) { + if (file.endsWith('.json')) { + const filePath = path.join(verbsDir, file); + const data = await fs.promises.readFile(filePath, 'utf-8'); + const parsedVerb = JSON.parse(data); + + // Check if both source and target nouns exist + const sourceExists = processedNouns.some(n => n.originalId === parsedVerb.sourceId); + const targetExists = processedNouns.some(n => n.originalId === parsedVerb.targetId); + + if (sourceExists && targetExists) { + console.log(`Re-creating verb ${parsedVerb.id} between ${parsedVerb.sourceId} and ${parsedVerb.targetId}...`); + try { + // Delete the existing verb first + await db.deleteVerb(parsedVerb.id); + + // Add the verb with the same relationship + await db.addVerb(parsedVerb.sourceId, parsedVerb.targetId, { + verb: parsedVerb.type || 'RelatedTo', + ...parsedVerb.metadata + }); + + processedVerbs.push(parsedVerb.id); + console.log(`Successfully re-created verb ${parsedVerb.id}`); + } catch (error) { + console.error(`Error re-creating verb ${parsedVerb.id}:`, error); + } + } else { + console.warn(`Skipping verb ${parsedVerb.id} because source or target noun doesn't exist`); + } + } + } + + console.log(`Processed ${processedVerbs.length} verbs.`); + + // Try a search to verify it works + console.log('Trying a search to verify it works...'); + const searchResults = await db.searchText('test', 10); + console.log(`Search returned ${searchResults.length} results.`); + + if (searchResults.length > 0) { + console.log('Sample of search results:'); + for (let i = 0; i < Math.min(5, searchResults.length); i++) { + console.log(`Result ${i + 1}:`, JSON.stringify({ + id: searchResults[i].id, + score: searchResults[i].score, + metadata: searchResults[i].metadata + }, null, 2)); + } + } + + console.log('Dimension mismatch fix completed successfully.'); + } catch (error) { + console.error('Error fixing dimension mismatch:', error); + } +} + +// Helper function to copy a directory recursively +async function copyDirectory(source, destination) { + const entries = await fs.promises.readdir(source, { withFileTypes: true }); + + await fs.promises.mkdir(destination, { recursive: true }); + + for (const entry of entries) { + const srcPath = path.join(source, entry.name); + const destPath = path.join(destination, entry.name); + + if (entry.isDirectory()) { + await copyDirectory(srcPath, destPath); + } else { + await fs.promises.copyFile(srcPath, destPath); + } + } +} + +fixDimensionMismatch().catch(console.error); \ No newline at end of file diff --git a/tests/package-size-limit.test.ts b/tests/package-size-limit.test.ts index 27c9dffd..06d7cf3c 100644 --- a/tests/package-size-limit.test.ts +++ b/tests/package-size-limit.test.ts @@ -6,8 +6,8 @@ import {describe, expect, it} from 'vitest' import {execSync} from 'child_process' -const CURRENT_UNPACKED_SIZE_MB = 11.1 -const CURRENT_PACKED_SIZE_MB = 2.5 +const CURRENT_UNPACKED_SIZE_MB = 12.6 +const CURRENT_PACKED_SIZE_MB = 2.3 const ALLOWED_SIZE_INCREASE_PERCENTAGE = 5 // 5% increase threshold /**