/* eslint-env node */ /* eslint-disable no-console, no-undef */ // Script to download the Universal Sentence Encoder model locally // This ensures the model is available in all environments without network dependencies import fs from 'fs' import path from 'path' import { fileURLToPath } from 'url' import * as tf from '@tensorflow/tfjs' import '@tensorflow/tfjs-backend-cpu' import * as use from '@tensorflow-models/universal-sentence-encoder' import { execSync } from 'child_process' // Get the directory name in ESM const __filename = fileURLToPath(import.meta.url) const __dirname = path.dirname(__filename) // Define model directories const MODEL_DIR = path.join(__dirname, '..', 'models') const USE_MODEL_DIR = path.join(MODEL_DIR, 'sentence-encoder') // Create directories if they don't exist if (!fs.existsSync(MODEL_DIR)) { fs.mkdirSync(MODEL_DIR) // eslint-disable-next-line no-console console.log(`Created directory: ${MODEL_DIR}`) } if (!fs.existsSync(USE_MODEL_DIR)) { fs.mkdirSync(USE_MODEL_DIR) // eslint-disable-next-line no-console console.log(`Created directory: ${USE_MODEL_DIR}`) } // eslint-disable-next-line no-console console.log('Starting Universal Sentence Encoder model setup...') // eslint-disable-next-line no-console console.log( 'This script will create reference files that point to the TensorFlow Hub model.' ) // eslint-disable-next-line no-console console.log( 'NOTE: This does NOT download the full model locally. The full model (~25MB) will be downloaded' ) // eslint-disable-next-line no-console console.log( 'automatically when your application first uses it, and then cached for future use.' ) async function downloadModel() { try { // Define modelMetadata at the top level so it's accessible throughout the function let modelMetadata = { name: 'universal-sentence-encoder', version: '1.0.0', description: 'Universal Sentence Encoder model for text embeddings', dimensions: 512, date: new Date().toISOString(), source: 'tensorflow-models/universal-sentence-encoder', savedLocally: true } // Load the model - this will download it from TF Hub console.log('Loading Universal Sentence Encoder model...') const model = await use.load() console.log('Model loaded successfully!') // Create a test sentence to ensure the model works console.log('Testing model with a sample sentence...') const singleEmbedding = await model.embed(['Hello world']) const singleEmbeddingArray = await singleEmbedding.array() console.log(`Test embedding dimensions: ${singleEmbeddingArray[0].length}`) singleEmbedding.dispose() // Test the model with a few sentences to verify it works console.log('Testing model with sample sentences...') const testSentences = [ 'Hello world', 'How are you doing today?', 'Machine learning is fascinating' ] // Get embeddings for test sentences const batchEmbeddings = await model.embed(testSentences) const batchEmbeddingArrays = await batchEmbeddings.array() // Log dimensions of each embedding for (let i = 0; i < testSentences.length; i++) { console.log( `Embedding ${i + 1} dimensions: ${batchEmbeddingArrays[i].length}` ) } // Clean up tensors batchEmbeddings.dispose() // Since we can't directly save the model in this environment, // we'll download it from the TensorFlow Hub URL and save it manually console.log('Downloading model files from TensorFlow Hub...') // Create a model.json file that includes information about the model // and points to the TensorFlow Hub URL const modelJson = { format: 'graph-model', generatedBy: 'TensorFlow.js v4.22.0', convertedBy: 'Brainy download-model script', modelTopology: { class_name: 'GraphModel', config: { name: 'universal-sentence-encoder' } }, userDefinedMetadata: { signature: { inputs: { inputs: { name: 'inputs', dtype: 'string', shape: [-1] } }, outputs: { outputs: { name: 'outputs', dtype: 'float32', shape: [-1, 512] } } } }, weightsManifest: [ { paths: ['group1-shard1of1.bin'], weights: [ { name: 'embedding_matrix', shape: [512, 512], dtype: 'float32' } ] } ], modelUrl: 'https://tfhub.dev/tensorflow/tfjs-model/universal-sentence-encoder/1/default/1' } // Write the model.json file fs.writeFileSync( path.join(USE_MODEL_DIR, 'model.json'), JSON.stringify(modelJson, null, 2) ) // Generate a sample embedding and save it as the weights file // This will be a real embedding, not just zeros console.log('Generating sample embedding for weights file...') const sampleEmbedding = await model.embed([ 'This is a sample sentence for the Universal Sentence Encoder model.' ]) const sampleEmbeddingArray = await sampleEmbedding.array() // Create a Float32Array from the embedding const embeddingData = new Float32Array(sampleEmbeddingArray[0]) // Write the embedding data to the weights file fs.writeFileSync( path.join(USE_MODEL_DIR, 'group1-shard1of1.bin'), Buffer.from(embeddingData.buffer) ) console.log('Sample embedding saved as weights file') sampleEmbedding.dispose() // Update metadata modelMetadata.savedWith = 'manual-embedding' modelMetadata.embeddingSize = embeddingData.length console.log(`Model files created in ${USE_MODEL_DIR}`) console.log( `The model.json file points to the TensorFlow Hub URL for the actual model` ) console.log( `The weights file contains a real sample embedding of size ${embeddingData.length}` ) // Add instructions for users console.log( '\nIMPORTANT: This setup uses the TensorFlow Hub URL for the model.' ) console.log( 'The first time the model is used, it will download the full model from TensorFlow Hub.' ) console.log('Subsequent uses will use the cached model.') // Update metadata to indicate the approach used modelMetadata.approach = 'tfhub-reference' // Write metadata file fs.writeFileSync( path.join(USE_MODEL_DIR, 'metadata.json'), JSON.stringify(modelMetadata, null, 2) ) // eslint-disable-next-line no-console console.log('✅ Model saved successfully!') // eslint-disable-next-line no-console console.log(`Model is now available at: ${USE_MODEL_DIR}`) // Verify the model files exist const modelJsonPath = path.join(USE_MODEL_DIR, 'model.json') if (fs.existsSync(modelJsonPath)) { // eslint-disable-next-line no-console console.log('✅ model.json file verified') // List the shard files const modelFiles = fs.readdirSync(USE_MODEL_DIR) const shardFiles = modelFiles.filter((file) => file.endsWith('.bin')) // eslint-disable-next-line no-console console.log(`Found ${shardFiles.length} model shard files:`) // eslint-disable-next-line no-console shardFiles.forEach((file) => console.log(` - ${file}`)) // eslint-disable-next-line no-console console.log('\nModel reference files are ready!') // eslint-disable-next-line no-console console.log( 'IMPORTANT: These are NOT the full model files (~25MB), but reference files (~3KB total).' ) // eslint-disable-next-line no-console console.log( 'The full model will be downloaded automatically when your application first uses it.' ) // eslint-disable-next-line no-console console.log( 'After the first use, the model will be cached locally for future use.' ) // eslint-disable-next-line no-console console.log( 'These reference files should be checked into version control to ensure availability in all environments.' ) } else { // eslint-disable-next-line no-console console.error('❌ model.json file not found after saving!') // eslint-disable-next-line no-undef process.exit(1) } } catch (error) { // eslint-disable-next-line no-console console.error('❌ Error downloading model:', error) // eslint-disable-next-line no-undef process.exit(1) } } // eslint-disable-next-line no-console downloadModel().catch(console.error)