**feat(models): add pre-bundled Universal Sentence Encoder for offline use**
- Introduced `@soulcraft/brainy-models` package with pre-bundled TensorFlow models for enhanced offline reliability. - Added `index.d.ts` and `index.js` allowing offline embedding workflows with the Universal Sentence Encoder model. - Included utility scripts for model compression, size retrieval, and availability checks. - Added `metadata.json` and `model.json` defining the Universal Sentence Encoder configuration with offline bundling. - Ensured comprehensive model documentation, error handling, and robust logging for seamless integration. - Supported optional model quantization placeholders for future TensorFlow.js enhancements. **Purpose**: Enable fully offline-ready embedding workflows via pre-bundled Universal Sentence Encoder models, ensuring maximum reliability and air-gapped environment compatibility.
This commit is contained in:
parent
93483572d8
commit
e476d45fac
16 changed files with 7678 additions and 52 deletions
103
brainy-models-package/dist/index.d.ts
vendored
Normal file
103
brainy-models-package/dist/index.d.ts
vendored
Normal file
|
|
@ -0,0 +1,103 @@
|
|||
/**
|
||||
* @soulcraft/brainy-models
|
||||
*
|
||||
* Pre-bundled TensorFlow models for maximum reliability with Brainy vector database.
|
||||
* This package provides offline access to the Universal Sentence Encoder model,
|
||||
* eliminating network dependencies and ensuring consistent performance.
|
||||
*/
|
||||
import * as tf from '@tensorflow/tfjs';
|
||||
export interface ModelMetadata {
|
||||
name: string;
|
||||
version: string;
|
||||
description: string;
|
||||
dimensions: number;
|
||||
downloadDate: string;
|
||||
source: string;
|
||||
approach: string;
|
||||
modelUrl: string;
|
||||
bundledLocally: boolean;
|
||||
reliability: string;
|
||||
}
|
||||
export interface BundledModelOptions {
|
||||
verbose?: boolean;
|
||||
preferCompressed?: boolean;
|
||||
}
|
||||
/**
|
||||
* Bundled Universal Sentence Encoder for offline use
|
||||
*/
|
||||
export declare class BundledUniversalSentenceEncoder {
|
||||
private model;
|
||||
private metadata;
|
||||
private options;
|
||||
constructor(options?: BundledModelOptions);
|
||||
/**
|
||||
* Load the bundled model from local files
|
||||
*/
|
||||
load(): Promise<void>;
|
||||
/**
|
||||
* Generate embeddings for the given texts
|
||||
*/
|
||||
embed(texts: string[]): Promise<tf.Tensor2D>;
|
||||
/**
|
||||
* Generate embeddings and return as JavaScript arrays
|
||||
*/
|
||||
embedToArrays(texts: string[]): Promise<number[][]>;
|
||||
/**
|
||||
* Get model metadata
|
||||
*/
|
||||
getMetadata(): ModelMetadata | null;
|
||||
/**
|
||||
* Check if the model is loaded
|
||||
*/
|
||||
isLoaded(): boolean;
|
||||
/**
|
||||
* Get model information
|
||||
*/
|
||||
getModelInfo(): {
|
||||
inputShape: number[];
|
||||
outputShape: number[];
|
||||
} | null;
|
||||
/**
|
||||
* Dispose of the model and free memory
|
||||
*/
|
||||
dispose(): void;
|
||||
}
|
||||
/**
|
||||
* Model compression utilities
|
||||
*/
|
||||
export declare class ModelCompressor {
|
||||
/**
|
||||
* Compress model weights using quantization
|
||||
* Note: TensorFlow.js doesn't currently support model quantization
|
||||
*/
|
||||
static quantizeModel(modelPath: string, outputPath: string, options?: {
|
||||
dtype?: 'int8' | 'int16';
|
||||
}): Promise<void>;
|
||||
/**
|
||||
* Get model size information by reading files from disk
|
||||
*/
|
||||
static getModelSize(modelPath: string): Promise<{
|
||||
totalSize: number;
|
||||
weightsSize: number;
|
||||
modelJsonSize: number;
|
||||
}>;
|
||||
}
|
||||
/**
|
||||
* Utility functions
|
||||
*/
|
||||
export declare const utils: {
|
||||
/**
|
||||
* Check if bundled models are available
|
||||
*/
|
||||
checkModelsAvailable(): boolean;
|
||||
/**
|
||||
* Get bundled models directory
|
||||
*/
|
||||
getModelsDirectory(): string;
|
||||
/**
|
||||
* List available bundled models
|
||||
*/
|
||||
listAvailableModels(): string[];
|
||||
};
|
||||
export default BundledUniversalSentenceEncoder;
|
||||
//# sourceMappingURL=index.d.ts.map
|
||||
1
brainy-models-package/dist/index.d.ts.map
vendored
Normal file
1
brainy-models-package/dist/index.d.ts.map
vendored
Normal file
|
|
@ -0,0 +1 @@
|
|||
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH,OAAO,KAAK,EAAE,MAAM,kBAAkB,CAAA;AAwBtC,MAAM,WAAW,aAAa;IAC5B,IAAI,EAAE,MAAM,CAAA;IACZ,OAAO,EAAE,MAAM,CAAA;IACf,WAAW,EAAE,MAAM,CAAA;IACnB,UAAU,EAAE,MAAM,CAAA;IAClB,YAAY,EAAE,MAAM,CAAA;IACpB,MAAM,EAAE,MAAM,CAAA;IACd,QAAQ,EAAE,MAAM,CAAA;IAChB,QAAQ,EAAE,MAAM,CAAA;IAChB,cAAc,EAAE,OAAO,CAAA;IACvB,WAAW,EAAE,MAAM,CAAA;CACpB;AAED,MAAM,WAAW,mBAAmB;IAClC,OAAO,CAAC,EAAE,OAAO,CAAA;IACjB,gBAAgB,CAAC,EAAE,OAAO,CAAA;CAC3B;AAED;;GAEG;AACH,qBAAa,+BAA+B;IAC1C,OAAO,CAAC,KAAK,CAA6B;IAC1C,OAAO,CAAC,QAAQ,CAA6B;IAC7C,OAAO,CAAC,OAAO,CAAqB;gBAExB,OAAO,GAAE,mBAAwB;IAQ7C;;OAEG;IACG,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;IAwC3B;;OAEG;IACG,KAAK,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,EAAE,CAAC,QAAQ,CAAC;IAqBlD;;OAEG;IACG,aAAa,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC;IAOzD;;OAEG;IACH,WAAW,IAAI,aAAa,GAAG,IAAI;IAInC;;OAEG;IACH,QAAQ,IAAI,OAAO;IAInB;;OAEG;IACH,YAAY,IAAI;QAAE,UAAU,EAAE,MAAM,EAAE,CAAC;QAAC,WAAW,EAAE,MAAM,EAAE,CAAA;KAAE,GAAG,IAAI;IAWtE;;OAEG;IACH,OAAO,IAAI,IAAI;CAMhB;AAED;;GAEG;AACH,qBAAa,eAAe;IAC1B;;;OAGG;WACU,aAAa,CACxB,SAAS,EAAE,MAAM,EACjB,UAAU,EAAE,MAAM,EAClB,OAAO,GAAE;QAAE,KAAK,CAAC,EAAE,MAAM,GAAG,OAAO,CAAA;KAAO,GACzC,OAAO,CAAC,IAAI,CAAC;IAwBhB;;OAEG;WACU,YAAY,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC;QACpD,SAAS,EAAE,MAAM,CAAA;QACjB,WAAW,EAAE,MAAM,CAAA;QACnB,aAAa,EAAE,MAAM,CAAA;KACtB,CAAC;CAuCH;AAED;;GAEG;AACH,eAAO,MAAM,KAAK;IAChB;;OAEG;4BACqB,OAAO;IAK/B;;OAEG;0BACmB,MAAM;IAI5B;;OAEG;2BACoB,MAAM,EAAE;CAUhC,CAAA;AAGD,eAAe,+BAA+B,CAAA"}
|
||||
237
brainy-models-package/dist/index.js
vendored
Normal file
237
brainy-models-package/dist/index.js
vendored
Normal file
|
|
@ -0,0 +1,237 @@
|
|||
/**
|
||||
* @soulcraft/brainy-models
|
||||
*
|
||||
* Pre-bundled TensorFlow models for maximum reliability with Brainy vector database.
|
||||
* This package provides offline access to the Universal Sentence Encoder model,
|
||||
* eliminating network dependencies and ensuring consistent performance.
|
||||
*/
|
||||
import * as tf from '@tensorflow/tfjs';
|
||||
import { readFileSync, existsSync } from 'fs';
|
||||
import { join, dirname } from 'path';
|
||||
import { fileURLToPath } from 'url';
|
||||
/**
|
||||
* Helper function to safely extract error message from unknown error type
|
||||
*/
|
||||
function getErrorMessage(error) {
|
||||
if (error instanceof Error) {
|
||||
return error.message;
|
||||
}
|
||||
if (typeof error === 'string') {
|
||||
return error;
|
||||
}
|
||||
return String(error);
|
||||
}
|
||||
// Get the package directory
|
||||
const __filename = fileURLToPath(import.meta.url);
|
||||
const __dirname = dirname(__filename);
|
||||
const PACKAGE_ROOT = join(__dirname, '..');
|
||||
const MODELS_DIR = join(PACKAGE_ROOT, 'models');
|
||||
/**
|
||||
* Bundled Universal Sentence Encoder for offline use
|
||||
*/
|
||||
export class BundledUniversalSentenceEncoder {
|
||||
model = null;
|
||||
metadata = null;
|
||||
options;
|
||||
constructor(options = {}) {
|
||||
this.options = {
|
||||
verbose: false,
|
||||
preferCompressed: false,
|
||||
...options
|
||||
};
|
||||
}
|
||||
/**
|
||||
* Load the bundled model from local files
|
||||
*/
|
||||
async load() {
|
||||
try {
|
||||
const modelDir = join(MODELS_DIR, 'universal-sentence-encoder');
|
||||
const modelPath = join(modelDir, 'model.json');
|
||||
const metadataPath = join(modelDir, 'metadata.json');
|
||||
if (!existsSync(modelPath)) {
|
||||
throw new Error(`Bundled model not found at ${modelPath}. ` +
|
||||
'Please run "npm run download-models" to download the model files.');
|
||||
}
|
||||
if (this.options.verbose) {
|
||||
console.log('🔄 Loading bundled Universal Sentence Encoder model...');
|
||||
}
|
||||
// Load metadata
|
||||
if (existsSync(metadataPath)) {
|
||||
const metadataContent = readFileSync(metadataPath, 'utf8');
|
||||
this.metadata = JSON.parse(metadataContent);
|
||||
if (this.options.verbose) {
|
||||
console.log(`📋 Model metadata:`, this.metadata);
|
||||
}
|
||||
}
|
||||
// Load the model
|
||||
this.model = await tf.loadGraphModel(`file://${modelPath}`);
|
||||
if (this.options.verbose) {
|
||||
console.log('✅ Bundled model loaded successfully');
|
||||
console.log(`🔒 Reliability: Maximum (fully offline)`);
|
||||
}
|
||||
}
|
||||
catch (error) {
|
||||
throw new Error(`Failed to load bundled model: ${getErrorMessage(error)}`);
|
||||
}
|
||||
}
|
||||
/**
|
||||
* Generate embeddings for the given texts
|
||||
*/
|
||||
async embed(texts) {
|
||||
if (!this.model) {
|
||||
throw new Error('Model not loaded. Call load() first.');
|
||||
}
|
||||
try {
|
||||
// Convert texts to tensor
|
||||
const inputTensor = tf.tensor1d(texts, 'string');
|
||||
// Run inference
|
||||
const embeddings = this.model.predict(inputTensor);
|
||||
// Clean up input tensor
|
||||
inputTensor.dispose();
|
||||
return embeddings;
|
||||
}
|
||||
catch (error) {
|
||||
throw new Error(`Failed to generate embeddings: ${getErrorMessage(error)}`);
|
||||
}
|
||||
}
|
||||
/**
|
||||
* Generate embeddings and return as JavaScript arrays
|
||||
*/
|
||||
async embedToArrays(texts) {
|
||||
const embeddings = await this.embed(texts);
|
||||
const arrays = await embeddings.array();
|
||||
embeddings.dispose();
|
||||
return arrays;
|
||||
}
|
||||
/**
|
||||
* Get model metadata
|
||||
*/
|
||||
getMetadata() {
|
||||
return this.metadata;
|
||||
}
|
||||
/**
|
||||
* Check if the model is loaded
|
||||
*/
|
||||
isLoaded() {
|
||||
return this.model !== null;
|
||||
}
|
||||
/**
|
||||
* Get model information
|
||||
*/
|
||||
getModelInfo() {
|
||||
if (!this.model) {
|
||||
return null;
|
||||
}
|
||||
return {
|
||||
inputShape: this.model.inputs[0].shape || [],
|
||||
outputShape: this.model.outputs[0].shape || []
|
||||
};
|
||||
}
|
||||
/**
|
||||
* Dispose of the model and free memory
|
||||
*/
|
||||
dispose() {
|
||||
if (this.model) {
|
||||
this.model.dispose();
|
||||
this.model = null;
|
||||
}
|
||||
}
|
||||
}
|
||||
/**
|
||||
* Model compression utilities
|
||||
*/
|
||||
export class ModelCompressor {
|
||||
/**
|
||||
* Compress model weights using quantization
|
||||
* Note: TensorFlow.js doesn't currently support model quantization
|
||||
*/
|
||||
static async quantizeModel(modelPath, outputPath, options = {}) {
|
||||
const { dtype = 'int8' } = options;
|
||||
try {
|
||||
console.log(`🔄 Loading model for quantization: ${modelPath}`);
|
||||
const model = await tf.loadGraphModel(`file://${modelPath}`);
|
||||
console.log(`🗜️ Quantizing model to ${dtype}...`);
|
||||
// TensorFlow.js doesn't have built-in quantization or model serialization APIs yet
|
||||
// This is a placeholder implementation that acknowledges the limitation
|
||||
console.warn('⚠️ Model quantization is not yet supported in TensorFlow.js');
|
||||
console.log(`📋 Model loaded successfully from: ${modelPath}`);
|
||||
console.log(`📋 Target output path: ${outputPath}`);
|
||||
console.log(`📋 Target dtype: ${dtype}`);
|
||||
model.dispose();
|
||||
throw new Error('Model quantization is not yet supported in TensorFlow.js. This feature requires server-side processing with TensorFlow Python.');
|
||||
}
|
||||
catch (error) {
|
||||
throw new Error(`Failed to compress model: ${getErrorMessage(error)}`);
|
||||
}
|
||||
}
|
||||
/**
|
||||
* Get model size information by reading files from disk
|
||||
*/
|
||||
static async getModelSize(modelPath) {
|
||||
try {
|
||||
// Load model to verify it's valid
|
||||
const model = await tf.loadGraphModel(`file://${modelPath}`);
|
||||
model.dispose();
|
||||
// Get model.json size
|
||||
const modelJsonSize = existsSync(modelPath) ? readFileSync(modelPath).length : 0;
|
||||
// Calculate weights size by reading weight files
|
||||
let weightsSize = 0;
|
||||
const modelDir = dirname(modelPath);
|
||||
// Read model.json to get weight file names
|
||||
if (existsSync(modelPath)) {
|
||||
const modelJson = JSON.parse(readFileSync(modelPath, 'utf8'));
|
||||
if (modelJson.weightsManifest) {
|
||||
for (const manifest of modelJson.weightsManifest) {
|
||||
for (const path of manifest.paths) {
|
||||
const weightFilePath = join(modelDir, path);
|
||||
if (existsSync(weightFilePath)) {
|
||||
weightsSize += readFileSync(weightFilePath).length;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
const totalSize = weightsSize + modelJsonSize;
|
||||
return {
|
||||
totalSize,
|
||||
weightsSize,
|
||||
modelJsonSize
|
||||
};
|
||||
}
|
||||
catch (error) {
|
||||
throw new Error(`Failed to get model size: ${getErrorMessage(error)}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
/**
|
||||
* Utility functions
|
||||
*/
|
||||
export const utils = {
|
||||
/**
|
||||
* Check if bundled models are available
|
||||
*/
|
||||
checkModelsAvailable() {
|
||||
const modelPath = join(MODELS_DIR, 'universal-sentence-encoder', 'model.json');
|
||||
return existsSync(modelPath);
|
||||
},
|
||||
/**
|
||||
* Get bundled models directory
|
||||
*/
|
||||
getModelsDirectory() {
|
||||
return MODELS_DIR;
|
||||
},
|
||||
/**
|
||||
* List available bundled models
|
||||
*/
|
||||
listAvailableModels() {
|
||||
const models = [];
|
||||
const useModelPath = join(MODELS_DIR, 'universal-sentence-encoder', 'model.json');
|
||||
if (existsSync(useModelPath)) {
|
||||
models.push('universal-sentence-encoder');
|
||||
}
|
||||
return models;
|
||||
}
|
||||
};
|
||||
// Default export for convenience
|
||||
export default BundledUniversalSentenceEncoder;
|
||||
//# sourceMappingURL=index.js.map
|
||||
1
brainy-models-package/dist/index.js.map
vendored
Normal file
1
brainy-models-package/dist/index.js.map
vendored
Normal file
File diff suppressed because one or more lines are too long
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
|
|
@ -0,0 +1,12 @@
|
|||
{
|
||||
"name": "universal-sentence-encoder",
|
||||
"version": "1.0.0",
|
||||
"description": "Complete Universal Sentence Encoder model bundled for offline use",
|
||||
"dimensions": 512,
|
||||
"downloadDate": "2025-08-01T23:20:17.338Z",
|
||||
"source": "tensorflow-models/universal-sentence-encoder",
|
||||
"approach": "full-bundle",
|
||||
"modelUrl": "https://storage.googleapis.com/tfjs-models/savedmodel/universal_sentence_encoder",
|
||||
"bundledLocally": true,
|
||||
"reliability": "maximum"
|
||||
}
|
||||
7182
brainy-models-package/models/universal-sentence-encoder/model.json
Normal file
7182
brainy-models-package/models/universal-sentence-encoder/model.json
Normal file
File diff suppressed because it is too large
Load diff
28
brainy-models-package/reproduce-error.js
Normal file
28
brainy-models-package/reproduce-error.js
Normal file
|
|
@ -0,0 +1,28 @@
|
|||
#!/usr/bin/env node
|
||||
|
||||
/**
|
||||
* Reproduction script for the TensorFlow.js isNullOrUndefined error
|
||||
*/
|
||||
|
||||
import * as tf from '@tensorflow/tfjs-node'
|
||||
import * as use from '@tensorflow-models/universal-sentence-encoder'
|
||||
|
||||
console.log('🔍 Loading Universal Sentence Encoder model...')
|
||||
|
||||
try {
|
||||
const model = await use.load()
|
||||
console.log('✅ Model loaded successfully')
|
||||
|
||||
console.log('🧪 Testing model functionality...')
|
||||
const testEmbedding = await model.embed(['Hello world'])
|
||||
const testArray = await testEmbedding.array()
|
||||
console.log(
|
||||
`✅ Model test passed - embedding dimensions: ${testArray[0].length}`
|
||||
)
|
||||
testEmbedding.dispose()
|
||||
model.dispose()
|
||||
} catch (error) {
|
||||
console.error('❌ Error:', error)
|
||||
console.error('Stack trace:', error.stack)
|
||||
process.exit(1)
|
||||
}
|
||||
|
|
@ -40,40 +40,69 @@ console.log('for offline use and maximum reliability.\n')
|
|||
/**
|
||||
* Download a file from URL to local path
|
||||
*/
|
||||
async function downloadFile(url, filePath) {
|
||||
async function downloadFile(url, filePath, maxRedirects = 5) {
|
||||
return new Promise((resolve, reject) => {
|
||||
const file = fs.createWriteStream(filePath)
|
||||
|
||||
https.get(url, (response) => {
|
||||
if (response.statusCode !== 200) {
|
||||
reject(new Error(`Failed to download ${url}: ${response.statusCode}`))
|
||||
return
|
||||
}
|
||||
|
||||
const totalSize = parseInt(response.headers['content-length'] || '0')
|
||||
let downloadedSize = 0
|
||||
|
||||
response.on('data', (chunk) => {
|
||||
downloadedSize += chunk.length
|
||||
if (totalSize > 0) {
|
||||
const progress = ((downloadedSize / totalSize) * 100).toFixed(1)
|
||||
process.stdout.write(`\r📥 Downloading: ${progress}% (${downloadedSize}/${totalSize} bytes)`)
|
||||
const handleRequest = (requestUrl, redirectCount = 0) => {
|
||||
https.get(requestUrl, (response) => {
|
||||
// Handle redirects
|
||||
if (response.statusCode >= 300 && response.statusCode < 400) {
|
||||
if (redirectCount >= maxRedirects) {
|
||||
reject(new Error(`Too many redirects (${redirectCount}) for ${url}`))
|
||||
return
|
||||
}
|
||||
|
||||
const location = response.headers.location
|
||||
if (!location) {
|
||||
reject(new Error(`Redirect response without location header for ${url}`))
|
||||
return
|
||||
}
|
||||
|
||||
// Handle relative redirects
|
||||
const redirectUrl = location.startsWith('http') ? location : new URL(location, requestUrl).href
|
||||
console.log(`📍 Following redirect ${redirectCount + 1}: ${redirectUrl}`)
|
||||
|
||||
// Close the current file stream and start over with the redirect URL
|
||||
file.close()
|
||||
fs.unlink(filePath, () => {}) // Delete partial file
|
||||
|
||||
// Recursively handle the redirect
|
||||
return downloadFile(redirectUrl, filePath, maxRedirects).then(resolve).catch(reject)
|
||||
}
|
||||
})
|
||||
|
||||
response.pipe(file)
|
||||
|
||||
file.on('finish', () => {
|
||||
file.close()
|
||||
console.log(`\n✅ Downloaded: ${path.basename(filePath)}`)
|
||||
resolve()
|
||||
})
|
||||
|
||||
file.on('error', (err) => {
|
||||
fs.unlink(filePath, () => {}) // Delete partial file
|
||||
reject(err)
|
||||
})
|
||||
}).on('error', reject)
|
||||
|
||||
if (response.statusCode !== 200) {
|
||||
reject(new Error(`Failed to download ${url}: ${response.statusCode}`))
|
||||
return
|
||||
}
|
||||
|
||||
const totalSize = parseInt(response.headers['content-length'] || '0')
|
||||
let downloadedSize = 0
|
||||
|
||||
response.on('data', (chunk) => {
|
||||
downloadedSize += chunk.length
|
||||
if (totalSize > 0) {
|
||||
const progress = ((downloadedSize / totalSize) * 100).toFixed(1)
|
||||
process.stdout.write(`\r📥 Downloading: ${progress}% (${downloadedSize}/${totalSize} bytes)`)
|
||||
}
|
||||
})
|
||||
|
||||
response.pipe(file)
|
||||
|
||||
file.on('finish', () => {
|
||||
file.close()
|
||||
console.log(`\n✅ Downloaded: ${path.basename(filePath)}`)
|
||||
resolve()
|
||||
})
|
||||
|
||||
file.on('error', (err) => {
|
||||
fs.unlink(filePath, () => {}) // Delete partial file
|
||||
reject(err)
|
||||
})
|
||||
}).on('error', reject)
|
||||
}
|
||||
|
||||
handleRequest(url)
|
||||
})
|
||||
}
|
||||
|
||||
|
|
@ -95,10 +124,11 @@ async function downloadFullModel() {
|
|||
console.log(`✅ Model test passed - embedding dimensions: ${testArray[0].length}`)
|
||||
testEmbedding.dispose()
|
||||
|
||||
// The Universal Sentence Encoder model URL
|
||||
const modelBaseUrl = 'https://tfhub.dev/tensorflow/tfjs-model/universal-sentence-encoder/1/default/1'
|
||||
// The Universal Sentence Encoder model URL (using Google Cloud Storage which still works)
|
||||
const modelBaseUrl = 'https://storage.googleapis.com/tfjs-models/savedmodel/universal_sentence_encoder'
|
||||
|
||||
console.log('📦 Downloading model files...')
|
||||
console.log('Using Google Cloud Storage URLs (TensorFlow Hub URLs are deprecated)...')
|
||||
|
||||
// Download model.json
|
||||
const modelJsonUrl = `${modelBaseUrl}/model.json`
|
||||
|
|
@ -161,7 +191,6 @@ async function downloadFullModel() {
|
|||
console.log('✅ Offline model loads successfully')
|
||||
|
||||
// Clean up
|
||||
model.dispose()
|
||||
offlineModel.dispose()
|
||||
|
||||
console.log('\n✨ Full model bundling completed successfully!')
|
||||
|
|
|
|||
|
|
@ -11,6 +11,19 @@ import { readFileSync, existsSync } from 'fs'
|
|||
import { join, dirname } from 'path'
|
||||
import { fileURLToPath } from 'url'
|
||||
|
||||
/**
|
||||
* Helper function to safely extract error message from unknown error type
|
||||
*/
|
||||
function getErrorMessage(error: unknown): string {
|
||||
if (error instanceof Error) {
|
||||
return error.message
|
||||
}
|
||||
if (typeof error === 'string') {
|
||||
return error
|
||||
}
|
||||
return String(error)
|
||||
}
|
||||
|
||||
// Get the package directory
|
||||
const __filename = fileURLToPath(import.meta.url)
|
||||
const __dirname = dirname(__filename)
|
||||
|
|
@ -90,7 +103,7 @@ export class BundledUniversalSentenceEncoder {
|
|||
}
|
||||
|
||||
} catch (error) {
|
||||
throw new Error(`Failed to load bundled model: ${error.message}`)
|
||||
throw new Error(`Failed to load bundled model: ${getErrorMessage(error)}`)
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -114,7 +127,7 @@ export class BundledUniversalSentenceEncoder {
|
|||
|
||||
return embeddings
|
||||
} catch (error) {
|
||||
throw new Error(`Failed to generate embeddings: ${error.message}`)
|
||||
throw new Error(`Failed to generate embeddings: ${getErrorMessage(error)}`)
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -173,6 +186,7 @@ export class BundledUniversalSentenceEncoder {
|
|||
export class ModelCompressor {
|
||||
/**
|
||||
* Compress model weights using quantization
|
||||
* Note: TensorFlow.js doesn't currently support model quantization
|
||||
*/
|
||||
static async quantizeModel(
|
||||
modelPath: string,
|
||||
|
|
@ -187,23 +201,23 @@ export class ModelCompressor {
|
|||
|
||||
console.log(`🗜️ Quantizing model to ${dtype}...`)
|
||||
|
||||
// Note: TensorFlow.js doesn't have built-in quantization yet,
|
||||
// but we can implement basic weight compression
|
||||
const modelArtifacts = await model.serialize()
|
||||
|
||||
// Save the compressed model
|
||||
await tf.io.fileSystem(outputPath).save(modelArtifacts)
|
||||
|
||||
console.log(`✅ Compressed model saved to: ${outputPath}`)
|
||||
// TensorFlow.js doesn't have built-in quantization or model serialization APIs yet
|
||||
// This is a placeholder implementation that acknowledges the limitation
|
||||
console.warn('⚠️ Model quantization is not yet supported in TensorFlow.js')
|
||||
console.log(`📋 Model loaded successfully from: ${modelPath}`)
|
||||
console.log(`📋 Target output path: ${outputPath}`)
|
||||
console.log(`📋 Target dtype: ${dtype}`)
|
||||
|
||||
model.dispose()
|
||||
|
||||
throw new Error('Model quantization is not yet supported in TensorFlow.js. This feature requires server-side processing with TensorFlow Python.')
|
||||
} catch (error) {
|
||||
throw new Error(`Failed to compress model: ${error.message}`)
|
||||
throw new Error(`Failed to compress model: ${getErrorMessage(error)}`)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get model size information
|
||||
* Get model size information by reading files from disk
|
||||
*/
|
||||
static async getModelSize(modelPath: string): Promise<{
|
||||
totalSize: number
|
||||
|
|
@ -211,22 +225,41 @@ export class ModelCompressor {
|
|||
modelJsonSize: number
|
||||
}> {
|
||||
try {
|
||||
// Load model to verify it's valid
|
||||
const model = await tf.loadGraphModel(`file://${modelPath}`)
|
||||
const artifacts = await model.serialize()
|
||||
|
||||
const weightsSize = artifacts.weightData?.byteLength || 0
|
||||
const modelJsonSize = JSON.stringify(artifacts.modelTopology).length
|
||||
const totalSize = weightsSize + modelJsonSize
|
||||
|
||||
model.dispose()
|
||||
|
||||
// Get model.json size
|
||||
const modelJsonSize = existsSync(modelPath) ? readFileSync(modelPath).length : 0
|
||||
|
||||
// Calculate weights size by reading weight files
|
||||
let weightsSize = 0
|
||||
const modelDir = dirname(modelPath)
|
||||
|
||||
// Read model.json to get weight file names
|
||||
if (existsSync(modelPath)) {
|
||||
const modelJson = JSON.parse(readFileSync(modelPath, 'utf8'))
|
||||
if (modelJson.weightsManifest) {
|
||||
for (const manifest of modelJson.weightsManifest) {
|
||||
for (const path of manifest.paths) {
|
||||
const weightFilePath = join(modelDir, path)
|
||||
if (existsSync(weightFilePath)) {
|
||||
weightsSize += readFileSync(weightFilePath).length
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const totalSize = weightsSize + modelJsonSize
|
||||
|
||||
return {
|
||||
totalSize,
|
||||
weightsSize,
|
||||
modelJsonSize
|
||||
}
|
||||
} catch (error) {
|
||||
throw new Error(`Failed to get model size: ${error.message}`)
|
||||
throw new Error(`Failed to get model size: ${getErrorMessage(error)}`)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue