**feat(models): add pre-bundled Universal Sentence Encoder for offline use**

- Introduced `@soulcraft/brainy-models` package with pre-bundled TensorFlow models for enhanced offline reliability.
- Added `index.d.ts` and `index.js` allowing offline embedding workflows with the Universal Sentence Encoder model.
- Included utility scripts for model compression, size retrieval, and availability checks.
- Added `metadata.json` and `model.json` defining the Universal Sentence Encoder configuration with offline bundling.
- Ensured comprehensive model documentation, error handling, and robust logging for seamless integration.
- Supported optional model quantization placeholders for future TensorFlow.js enhancements.

**Purpose**: Enable fully offline-ready embedding workflows via pre-bundled Universal Sentence Encoder models, ensuring maximum reliability and air-gapped environment compatibility.
This commit is contained in:
David Snelling 2025-08-01 16:22:58 -07:00
parent 93483572d8
commit e476d45fac
16 changed files with 7678 additions and 52 deletions

103
brainy-models-package/dist/index.d.ts vendored Normal file
View file

@ -0,0 +1,103 @@
/**
* @soulcraft/brainy-models
*
* Pre-bundled TensorFlow models for maximum reliability with Brainy vector database.
* This package provides offline access to the Universal Sentence Encoder model,
* eliminating network dependencies and ensuring consistent performance.
*/
import * as tf from '@tensorflow/tfjs';
export interface ModelMetadata {
name: string;
version: string;
description: string;
dimensions: number;
downloadDate: string;
source: string;
approach: string;
modelUrl: string;
bundledLocally: boolean;
reliability: string;
}
export interface BundledModelOptions {
verbose?: boolean;
preferCompressed?: boolean;
}
/**
* Bundled Universal Sentence Encoder for offline use
*/
export declare class BundledUniversalSentenceEncoder {
private model;
private metadata;
private options;
constructor(options?: BundledModelOptions);
/**
* Load the bundled model from local files
*/
load(): Promise<void>;
/**
* Generate embeddings for the given texts
*/
embed(texts: string[]): Promise<tf.Tensor2D>;
/**
* Generate embeddings and return as JavaScript arrays
*/
embedToArrays(texts: string[]): Promise<number[][]>;
/**
* Get model metadata
*/
getMetadata(): ModelMetadata | null;
/**
* Check if the model is loaded
*/
isLoaded(): boolean;
/**
* Get model information
*/
getModelInfo(): {
inputShape: number[];
outputShape: number[];
} | null;
/**
* Dispose of the model and free memory
*/
dispose(): void;
}
/**
* Model compression utilities
*/
export declare class ModelCompressor {
/**
* Compress model weights using quantization
* Note: TensorFlow.js doesn't currently support model quantization
*/
static quantizeModel(modelPath: string, outputPath: string, options?: {
dtype?: 'int8' | 'int16';
}): Promise<void>;
/**
* Get model size information by reading files from disk
*/
static getModelSize(modelPath: string): Promise<{
totalSize: number;
weightsSize: number;
modelJsonSize: number;
}>;
}
/**
* Utility functions
*/
export declare const utils: {
/**
* Check if bundled models are available
*/
checkModelsAvailable(): boolean;
/**
* Get bundled models directory
*/
getModelsDirectory(): string;
/**
* List available bundled models
*/
listAvailableModels(): string[];
};
export default BundledUniversalSentenceEncoder;
//# sourceMappingURL=index.d.ts.map

View file

@ -0,0 +1 @@
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH,OAAO,KAAK,EAAE,MAAM,kBAAkB,CAAA;AAwBtC,MAAM,WAAW,aAAa;IAC5B,IAAI,EAAE,MAAM,CAAA;IACZ,OAAO,EAAE,MAAM,CAAA;IACf,WAAW,EAAE,MAAM,CAAA;IACnB,UAAU,EAAE,MAAM,CAAA;IAClB,YAAY,EAAE,MAAM,CAAA;IACpB,MAAM,EAAE,MAAM,CAAA;IACd,QAAQ,EAAE,MAAM,CAAA;IAChB,QAAQ,EAAE,MAAM,CAAA;IAChB,cAAc,EAAE,OAAO,CAAA;IACvB,WAAW,EAAE,MAAM,CAAA;CACpB;AAED,MAAM,WAAW,mBAAmB;IAClC,OAAO,CAAC,EAAE,OAAO,CAAA;IACjB,gBAAgB,CAAC,EAAE,OAAO,CAAA;CAC3B;AAED;;GAEG;AACH,qBAAa,+BAA+B;IAC1C,OAAO,CAAC,KAAK,CAA6B;IAC1C,OAAO,CAAC,QAAQ,CAA6B;IAC7C,OAAO,CAAC,OAAO,CAAqB;gBAExB,OAAO,GAAE,mBAAwB;IAQ7C;;OAEG;IACG,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;IAwC3B;;OAEG;IACG,KAAK,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,EAAE,CAAC,QAAQ,CAAC;IAqBlD;;OAEG;IACG,aAAa,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC;IAOzD;;OAEG;IACH,WAAW,IAAI,aAAa,GAAG,IAAI;IAInC;;OAEG;IACH,QAAQ,IAAI,OAAO;IAInB;;OAEG;IACH,YAAY,IAAI;QAAE,UAAU,EAAE,MAAM,EAAE,CAAC;QAAC,WAAW,EAAE,MAAM,EAAE,CAAA;KAAE,GAAG,IAAI;IAWtE;;OAEG;IACH,OAAO,IAAI,IAAI;CAMhB;AAED;;GAEG;AACH,qBAAa,eAAe;IAC1B;;;OAGG;WACU,aAAa,CACxB,SAAS,EAAE,MAAM,EACjB,UAAU,EAAE,MAAM,EAClB,OAAO,GAAE;QAAE,KAAK,CAAC,EAAE,MAAM,GAAG,OAAO,CAAA;KAAO,GACzC,OAAO,CAAC,IAAI,CAAC;IAwBhB;;OAEG;WACU,YAAY,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC;QACpD,SAAS,EAAE,MAAM,CAAA;QACjB,WAAW,EAAE,MAAM,CAAA;QACnB,aAAa,EAAE,MAAM,CAAA;KACtB,CAAC;CAuCH;AAED;;GAEG;AACH,eAAO,MAAM,KAAK;IAChB;;OAEG;4BACqB,OAAO;IAK/B;;OAEG;0BACmB,MAAM;IAI5B;;OAEG;2BACoB,MAAM,EAAE;CAUhC,CAAA;AAGD,eAAe,+BAA+B,CAAA"}

237
brainy-models-package/dist/index.js vendored Normal file
View file

@ -0,0 +1,237 @@
/**
* @soulcraft/brainy-models
*
* Pre-bundled TensorFlow models for maximum reliability with Brainy vector database.
* This package provides offline access to the Universal Sentence Encoder model,
* eliminating network dependencies and ensuring consistent performance.
*/
import * as tf from '@tensorflow/tfjs';
import { readFileSync, existsSync } from 'fs';
import { join, dirname } from 'path';
import { fileURLToPath } from 'url';
/**
* Helper function to safely extract error message from unknown error type
*/
function getErrorMessage(error) {
if (error instanceof Error) {
return error.message;
}
if (typeof error === 'string') {
return error;
}
return String(error);
}
// Get the package directory
const __filename = fileURLToPath(import.meta.url);
const __dirname = dirname(__filename);
const PACKAGE_ROOT = join(__dirname, '..');
const MODELS_DIR = join(PACKAGE_ROOT, 'models');
/**
* Bundled Universal Sentence Encoder for offline use
*/
export class BundledUniversalSentenceEncoder {
model = null;
metadata = null;
options;
constructor(options = {}) {
this.options = {
verbose: false,
preferCompressed: false,
...options
};
}
/**
* Load the bundled model from local files
*/
async load() {
try {
const modelDir = join(MODELS_DIR, 'universal-sentence-encoder');
const modelPath = join(modelDir, 'model.json');
const metadataPath = join(modelDir, 'metadata.json');
if (!existsSync(modelPath)) {
throw new Error(`Bundled model not found at ${modelPath}. ` +
'Please run "npm run download-models" to download the model files.');
}
if (this.options.verbose) {
console.log('🔄 Loading bundled Universal Sentence Encoder model...');
}
// Load metadata
if (existsSync(metadataPath)) {
const metadataContent = readFileSync(metadataPath, 'utf8');
this.metadata = JSON.parse(metadataContent);
if (this.options.verbose) {
console.log(`📋 Model metadata:`, this.metadata);
}
}
// Load the model
this.model = await tf.loadGraphModel(`file://${modelPath}`);
if (this.options.verbose) {
console.log('✅ Bundled model loaded successfully');
console.log(`🔒 Reliability: Maximum (fully offline)`);
}
}
catch (error) {
throw new Error(`Failed to load bundled model: ${getErrorMessage(error)}`);
}
}
/**
* Generate embeddings for the given texts
*/
async embed(texts) {
if (!this.model) {
throw new Error('Model not loaded. Call load() first.');
}
try {
// Convert texts to tensor
const inputTensor = tf.tensor1d(texts, 'string');
// Run inference
const embeddings = this.model.predict(inputTensor);
// Clean up input tensor
inputTensor.dispose();
return embeddings;
}
catch (error) {
throw new Error(`Failed to generate embeddings: ${getErrorMessage(error)}`);
}
}
/**
* Generate embeddings and return as JavaScript arrays
*/
async embedToArrays(texts) {
const embeddings = await this.embed(texts);
const arrays = await embeddings.array();
embeddings.dispose();
return arrays;
}
/**
* Get model metadata
*/
getMetadata() {
return this.metadata;
}
/**
* Check if the model is loaded
*/
isLoaded() {
return this.model !== null;
}
/**
* Get model information
*/
getModelInfo() {
if (!this.model) {
return null;
}
return {
inputShape: this.model.inputs[0].shape || [],
outputShape: this.model.outputs[0].shape || []
};
}
/**
* Dispose of the model and free memory
*/
dispose() {
if (this.model) {
this.model.dispose();
this.model = null;
}
}
}
/**
* Model compression utilities
*/
export class ModelCompressor {
/**
* Compress model weights using quantization
* Note: TensorFlow.js doesn't currently support model quantization
*/
static async quantizeModel(modelPath, outputPath, options = {}) {
const { dtype = 'int8' } = options;
try {
console.log(`🔄 Loading model for quantization: ${modelPath}`);
const model = await tf.loadGraphModel(`file://${modelPath}`);
console.log(`🗜️ Quantizing model to ${dtype}...`);
// TensorFlow.js doesn't have built-in quantization or model serialization APIs yet
// This is a placeholder implementation that acknowledges the limitation
console.warn('⚠️ Model quantization is not yet supported in TensorFlow.js');
console.log(`📋 Model loaded successfully from: ${modelPath}`);
console.log(`📋 Target output path: ${outputPath}`);
console.log(`📋 Target dtype: ${dtype}`);
model.dispose();
throw new Error('Model quantization is not yet supported in TensorFlow.js. This feature requires server-side processing with TensorFlow Python.');
}
catch (error) {
throw new Error(`Failed to compress model: ${getErrorMessage(error)}`);
}
}
/**
* Get model size information by reading files from disk
*/
static async getModelSize(modelPath) {
try {
// Load model to verify it's valid
const model = await tf.loadGraphModel(`file://${modelPath}`);
model.dispose();
// Get model.json size
const modelJsonSize = existsSync(modelPath) ? readFileSync(modelPath).length : 0;
// Calculate weights size by reading weight files
let weightsSize = 0;
const modelDir = dirname(modelPath);
// Read model.json to get weight file names
if (existsSync(modelPath)) {
const modelJson = JSON.parse(readFileSync(modelPath, 'utf8'));
if (modelJson.weightsManifest) {
for (const manifest of modelJson.weightsManifest) {
for (const path of manifest.paths) {
const weightFilePath = join(modelDir, path);
if (existsSync(weightFilePath)) {
weightsSize += readFileSync(weightFilePath).length;
}
}
}
}
}
const totalSize = weightsSize + modelJsonSize;
return {
totalSize,
weightsSize,
modelJsonSize
};
}
catch (error) {
throw new Error(`Failed to get model size: ${getErrorMessage(error)}`);
}
}
}
/**
* Utility functions
*/
export const utils = {
/**
* Check if bundled models are available
*/
checkModelsAvailable() {
const modelPath = join(MODELS_DIR, 'universal-sentence-encoder', 'model.json');
return existsSync(modelPath);
},
/**
* Get bundled models directory
*/
getModelsDirectory() {
return MODELS_DIR;
},
/**
* List available bundled models
*/
listAvailableModels() {
const models = [];
const useModelPath = join(MODELS_DIR, 'universal-sentence-encoder', 'model.json');
if (existsSync(useModelPath)) {
models.push('universal-sentence-encoder');
}
return models;
}
};
// Default export for convenience
export default BundledUniversalSentenceEncoder;
//# sourceMappingURL=index.js.map

File diff suppressed because one or more lines are too long

View file

@ -0,0 +1,12 @@
{
"name": "universal-sentence-encoder",
"version": "1.0.0",
"description": "Complete Universal Sentence Encoder model bundled for offline use",
"dimensions": 512,
"downloadDate": "2025-08-01T23:20:17.338Z",
"source": "tensorflow-models/universal-sentence-encoder",
"approach": "full-bundle",
"modelUrl": "https://storage.googleapis.com/tfjs-models/savedmodel/universal_sentence_encoder",
"bundledLocally": true,
"reliability": "maximum"
}

File diff suppressed because it is too large Load diff

View file

@ -0,0 +1,28 @@
#!/usr/bin/env node
/**
* Reproduction script for the TensorFlow.js isNullOrUndefined error
*/
import * as tf from '@tensorflow/tfjs-node'
import * as use from '@tensorflow-models/universal-sentence-encoder'
console.log('🔍 Loading Universal Sentence Encoder model...')
try {
const model = await use.load()
console.log('✅ Model loaded successfully')
console.log('🧪 Testing model functionality...')
const testEmbedding = await model.embed(['Hello world'])
const testArray = await testEmbedding.array()
console.log(
`✅ Model test passed - embedding dimensions: ${testArray[0].length}`
)
testEmbedding.dispose()
model.dispose()
} catch (error) {
console.error('❌ Error:', error)
console.error('Stack trace:', error.stack)
process.exit(1)
}

View file

@ -40,40 +40,69 @@ console.log('for offline use and maximum reliability.\n')
/**
* Download a file from URL to local path
*/
async function downloadFile(url, filePath) {
async function downloadFile(url, filePath, maxRedirects = 5) {
return new Promise((resolve, reject) => {
const file = fs.createWriteStream(filePath)
https.get(url, (response) => {
if (response.statusCode !== 200) {
reject(new Error(`Failed to download ${url}: ${response.statusCode}`))
return
}
const totalSize = parseInt(response.headers['content-length'] || '0')
let downloadedSize = 0
response.on('data', (chunk) => {
downloadedSize += chunk.length
if (totalSize > 0) {
const progress = ((downloadedSize / totalSize) * 100).toFixed(1)
process.stdout.write(`\r📥 Downloading: ${progress}% (${downloadedSize}/${totalSize} bytes)`)
const handleRequest = (requestUrl, redirectCount = 0) => {
https.get(requestUrl, (response) => {
// Handle redirects
if (response.statusCode >= 300 && response.statusCode < 400) {
if (redirectCount >= maxRedirects) {
reject(new Error(`Too many redirects (${redirectCount}) for ${url}`))
return
}
const location = response.headers.location
if (!location) {
reject(new Error(`Redirect response without location header for ${url}`))
return
}
// Handle relative redirects
const redirectUrl = location.startsWith('http') ? location : new URL(location, requestUrl).href
console.log(`📍 Following redirect ${redirectCount + 1}: ${redirectUrl}`)
// Close the current file stream and start over with the redirect URL
file.close()
fs.unlink(filePath, () => {}) // Delete partial file
// Recursively handle the redirect
return downloadFile(redirectUrl, filePath, maxRedirects).then(resolve).catch(reject)
}
})
response.pipe(file)
file.on('finish', () => {
file.close()
console.log(`\n✅ Downloaded: ${path.basename(filePath)}`)
resolve()
})
file.on('error', (err) => {
fs.unlink(filePath, () => {}) // Delete partial file
reject(err)
})
}).on('error', reject)
if (response.statusCode !== 200) {
reject(new Error(`Failed to download ${url}: ${response.statusCode}`))
return
}
const totalSize = parseInt(response.headers['content-length'] || '0')
let downloadedSize = 0
response.on('data', (chunk) => {
downloadedSize += chunk.length
if (totalSize > 0) {
const progress = ((downloadedSize / totalSize) * 100).toFixed(1)
process.stdout.write(`\r📥 Downloading: ${progress}% (${downloadedSize}/${totalSize} bytes)`)
}
})
response.pipe(file)
file.on('finish', () => {
file.close()
console.log(`\n✅ Downloaded: ${path.basename(filePath)}`)
resolve()
})
file.on('error', (err) => {
fs.unlink(filePath, () => {}) // Delete partial file
reject(err)
})
}).on('error', reject)
}
handleRequest(url)
})
}
@ -95,10 +124,11 @@ async function downloadFullModel() {
console.log(`✅ Model test passed - embedding dimensions: ${testArray[0].length}`)
testEmbedding.dispose()
// The Universal Sentence Encoder model URL
const modelBaseUrl = 'https://tfhub.dev/tensorflow/tfjs-model/universal-sentence-encoder/1/default/1'
// The Universal Sentence Encoder model URL (using Google Cloud Storage which still works)
const modelBaseUrl = 'https://storage.googleapis.com/tfjs-models/savedmodel/universal_sentence_encoder'
console.log('📦 Downloading model files...')
console.log('Using Google Cloud Storage URLs (TensorFlow Hub URLs are deprecated)...')
// Download model.json
const modelJsonUrl = `${modelBaseUrl}/model.json`
@ -161,7 +191,6 @@ async function downloadFullModel() {
console.log('✅ Offline model loads successfully')
// Clean up
model.dispose()
offlineModel.dispose()
console.log('\n✨ Full model bundling completed successfully!')

View file

@ -11,6 +11,19 @@ import { readFileSync, existsSync } from 'fs'
import { join, dirname } from 'path'
import { fileURLToPath } from 'url'
/**
* Helper function to safely extract error message from unknown error type
*/
function getErrorMessage(error: unknown): string {
if (error instanceof Error) {
return error.message
}
if (typeof error === 'string') {
return error
}
return String(error)
}
// Get the package directory
const __filename = fileURLToPath(import.meta.url)
const __dirname = dirname(__filename)
@ -90,7 +103,7 @@ export class BundledUniversalSentenceEncoder {
}
} catch (error) {
throw new Error(`Failed to load bundled model: ${error.message}`)
throw new Error(`Failed to load bundled model: ${getErrorMessage(error)}`)
}
}
@ -114,7 +127,7 @@ export class BundledUniversalSentenceEncoder {
return embeddings
} catch (error) {
throw new Error(`Failed to generate embeddings: ${error.message}`)
throw new Error(`Failed to generate embeddings: ${getErrorMessage(error)}`)
}
}
@ -173,6 +186,7 @@ export class BundledUniversalSentenceEncoder {
export class ModelCompressor {
/**
* Compress model weights using quantization
* Note: TensorFlow.js doesn't currently support model quantization
*/
static async quantizeModel(
modelPath: string,
@ -187,23 +201,23 @@ export class ModelCompressor {
console.log(`🗜️ Quantizing model to ${dtype}...`)
// Note: TensorFlow.js doesn't have built-in quantization yet,
// but we can implement basic weight compression
const modelArtifacts = await model.serialize()
// Save the compressed model
await tf.io.fileSystem(outputPath).save(modelArtifacts)
console.log(`✅ Compressed model saved to: ${outputPath}`)
// TensorFlow.js doesn't have built-in quantization or model serialization APIs yet
// This is a placeholder implementation that acknowledges the limitation
console.warn('⚠️ Model quantization is not yet supported in TensorFlow.js')
console.log(`📋 Model loaded successfully from: ${modelPath}`)
console.log(`📋 Target output path: ${outputPath}`)
console.log(`📋 Target dtype: ${dtype}`)
model.dispose()
throw new Error('Model quantization is not yet supported in TensorFlow.js. This feature requires server-side processing with TensorFlow Python.')
} catch (error) {
throw new Error(`Failed to compress model: ${error.message}`)
throw new Error(`Failed to compress model: ${getErrorMessage(error)}`)
}
}
/**
* Get model size information
* Get model size information by reading files from disk
*/
static async getModelSize(modelPath: string): Promise<{
totalSize: number
@ -211,22 +225,41 @@ export class ModelCompressor {
modelJsonSize: number
}> {
try {
// Load model to verify it's valid
const model = await tf.loadGraphModel(`file://${modelPath}`)
const artifacts = await model.serialize()
const weightsSize = artifacts.weightData?.byteLength || 0
const modelJsonSize = JSON.stringify(artifacts.modelTopology).length
const totalSize = weightsSize + modelJsonSize
model.dispose()
// Get model.json size
const modelJsonSize = existsSync(modelPath) ? readFileSync(modelPath).length : 0
// Calculate weights size by reading weight files
let weightsSize = 0
const modelDir = dirname(modelPath)
// Read model.json to get weight file names
if (existsSync(modelPath)) {
const modelJson = JSON.parse(readFileSync(modelPath, 'utf8'))
if (modelJson.weightsManifest) {
for (const manifest of modelJson.weightsManifest) {
for (const path of manifest.paths) {
const weightFilePath = join(modelDir, path)
if (existsSync(weightFilePath)) {
weightsSize += readFileSync(weightFilePath).length
}
}
}
}
}
const totalSize = weightsSize + modelJsonSize
return {
totalSize,
weightsSize,
modelJsonSize
}
} catch (error) {
throw new Error(`Failed to get model size: ${error.message}`)
throw new Error(`Failed to get model size: ${getErrorMessage(error)}`)
}
}
}