feat: add brainy-models-package v0.8.0 with USE-lite model
- Add complete Universal Sentence Encoder Lite model (27MB) - Include vocab.json for tokenization support - Update package to work with @tensorflow-models/universal-sentence-encoder - Ensure offline model loading capability for Docker deployments - Published to npm as @soulcraft/brainy-models@0.8.0
This commit is contained in:
parent
daf3b6243a
commit
e868060057
17 changed files with 8012 additions and 0 deletions
21
brainy-models-package/LICENSE
Normal file
21
brainy-models-package/LICENSE
Normal file
|
|
@ -0,0 +1,21 @@
|
|||
MIT License
|
||||
|
||||
Copyright (c) 2025 Soulcraft Research
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
of this software and associated documentation files (the "Software"), to deal
|
||||
in the Software without restriction, including without limitation the rights
|
||||
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
copies of the Software, and to permit persons to whom the Software is
|
||||
furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in all
|
||||
copies or substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
SOFTWARE.
|
||||
375
brainy-models-package/README.md
Normal file
375
brainy-models-package/README.md
Normal file
|
|
@ -0,0 +1,375 @@
|
|||
<div align="center">
|
||||
<img src="../brainy.png" alt="Brainy Logo" width="200"/>
|
||||
<br/><br/>
|
||||
|
||||
[](../LICENSE)
|
||||
[](https://nodejs.org/)
|
||||
[](https://www.typescriptlang.org/)
|
||||
[](../CONTRIBUTING.md)
|
||||
|
||||
**Pre-bundled TensorFlow models for maximum reliability with Brainy vector database**
|
||||
|
||||
</div>
|
||||
|
||||
## ✨ Overview
|
||||
|
||||
This package provides offline access to the Universal Sentence Encoder model, eliminating network dependencies and ensuring consistent performance. It's designed as an optional companion to the main `@soulcraft/brainy` package for applications requiring maximum reliability.
|
||||
|
||||
### 🚀 Key Features
|
||||
|
||||
- 🔒 **Maximum Reliability**: Fully offline model loading with zero network dependencies
|
||||
- 📦 **Pre-bundled Models**: Complete Universal Sentence Encoder model (~25MB) included
|
||||
- 🗜️ **Model Compression**: Multiple optimized variants (float16, int8) for different use cases
|
||||
- ⚡ **Performance Optimized**: Use case-specific optimizations for memory and speed
|
||||
- 🛠️ **Easy Integration**: Drop-in replacement for online model loading
|
||||
- 📊 **Comprehensive Metrics**: Detailed model information and performance statistics
|
||||
|
||||
## 🔧 Installation
|
||||
|
||||
```bash
|
||||
npm install @soulcraft/brainy-models
|
||||
```
|
||||
|
||||
### Prerequisites
|
||||
|
||||
- Node.js >= 18.0.0
|
||||
- `@soulcraft/brainy` >= 0.33.0
|
||||
|
||||
## 🏁 Quick Start
|
||||
|
||||
### Basic Usage
|
||||
|
||||
```typescript
|
||||
import { BundledUniversalSentenceEncoder } from '@soulcraft/brainy-models'
|
||||
|
||||
// Create encoder instance
|
||||
const encoder = new BundledUniversalSentenceEncoder({
|
||||
verbose: true,
|
||||
preferCompressed: false
|
||||
})
|
||||
|
||||
// Load the bundled model
|
||||
await encoder.load()
|
||||
|
||||
// Generate embeddings
|
||||
const texts = ['Hello world', 'How are you?', 'Machine learning is amazing']
|
||||
const embeddings = await encoder.embedToArrays(texts)
|
||||
|
||||
console.log(`Generated ${embeddings.length} embeddings of ${embeddings[0].length} dimensions`)
|
||||
|
||||
// Clean up
|
||||
encoder.dispose()
|
||||
```
|
||||
|
||||
### Integration with Brainy
|
||||
|
||||
```typescript
|
||||
import Brainy from '@soulcraft/brainy'
|
||||
import { BundledUniversalSentenceEncoder } from '@soulcraft/brainy-models'
|
||||
|
||||
// Create bundled encoder
|
||||
const bundledEncoder = new BundledUniversalSentenceEncoder({ verbose: true })
|
||||
await bundledEncoder.load()
|
||||
|
||||
// Use with Brainy (custom integration)
|
||||
const brainy = new Brainy({
|
||||
// Configure Brainy to use the bundled encoder
|
||||
customEmbedding: async (texts) => {
|
||||
return await bundledEncoder.embedToArrays(texts)
|
||||
}
|
||||
})
|
||||
```
|
||||
|
||||
### Using Compressed Models
|
||||
|
||||
```typescript
|
||||
import { BundledUniversalSentenceEncoder } from '@soulcraft/brainy-models'
|
||||
|
||||
// Use compressed model for memory-constrained environments
|
||||
const encoder = new BundledUniversalSentenceEncoder({
|
||||
preferCompressed: true,
|
||||
verbose: true
|
||||
})
|
||||
|
||||
await encoder.load()
|
||||
|
||||
// The encoder will automatically use the most appropriate compressed variant
|
||||
const embeddings = await encoder.embedToArrays(['Sample text'])
|
||||
```
|
||||
|
||||
## 📚 API Reference
|
||||
|
||||
### BundledUniversalSentenceEncoder
|
||||
|
||||
Main class for loading and using bundled models.
|
||||
|
||||
#### Constructor
|
||||
|
||||
```typescript
|
||||
new BundledUniversalSentenceEncoder(options)
|
||||
```
|
||||
|
||||
**Options:**
|
||||
- `verbose?: boolean` - Enable detailed logging (default: false)
|
||||
- `preferCompressed?: boolean` - Prefer compressed model variants (default: false)
|
||||
|
||||
#### Methods
|
||||
|
||||
##### `load(): Promise<void>`
|
||||
|
||||
Load the bundled model from local files.
|
||||
|
||||
```typescript
|
||||
await encoder.load()
|
||||
```
|
||||
|
||||
##### `embed(texts: string[]): Promise<tf.Tensor2D>`
|
||||
|
||||
Generate embeddings as TensorFlow tensors.
|
||||
|
||||
```typescript
|
||||
const embeddings = await encoder.embed(['Hello world'])
|
||||
// Remember to dispose of tensors when done
|
||||
embeddings.dispose()
|
||||
```
|
||||
|
||||
##### `embedToArrays(texts: string[]): Promise<number[][]>`
|
||||
|
||||
Generate embeddings as JavaScript arrays (automatically disposes tensors).
|
||||
|
||||
```typescript
|
||||
const embeddings = await encoder.embedToArrays(['Hello world'])
|
||||
console.log(embeddings[0].length) // 512
|
||||
```
|
||||
|
||||
##### `getMetadata(): ModelMetadata | null`
|
||||
|
||||
Get model metadata information.
|
||||
|
||||
```typescript
|
||||
const metadata = encoder.getMetadata()
|
||||
console.log(metadata?.dimensions) // 512
|
||||
```
|
||||
|
||||
##### `isLoaded(): boolean`
|
||||
|
||||
Check if the model is loaded.
|
||||
|
||||
```typescript
|
||||
if (encoder.isLoaded()) {
|
||||
// Model is ready to use
|
||||
}
|
||||
```
|
||||
|
||||
##### `getModelInfo(): { inputShape: number[], outputShape: number[] } | null`
|
||||
|
||||
Get model input/output shape information.
|
||||
|
||||
```typescript
|
||||
const info = encoder.getModelInfo()
|
||||
console.log(info?.outputShape) // [-1, 512]
|
||||
```
|
||||
|
||||
##### `dispose(): void`
|
||||
|
||||
Clean up model resources.
|
||||
|
||||
```typescript
|
||||
encoder.dispose()
|
||||
```
|
||||
|
||||
### ModelCompressor
|
||||
|
||||
Utility class for model compression and optimization.
|
||||
|
||||
#### Static Methods
|
||||
|
||||
##### `quantizeModel(modelPath: string, outputPath: string, options?): Promise<void>`
|
||||
|
||||
Compress a model using quantization.
|
||||
|
||||
```typescript
|
||||
import { ModelCompressor } from '@soulcraft/brainy-models'
|
||||
|
||||
await ModelCompressor.quantizeModel(
|
||||
'/path/to/model.json',
|
||||
'/path/to/compressed/model.json',
|
||||
{ dtype: 'int8' }
|
||||
)
|
||||
```
|
||||
|
||||
##### `getModelSize(modelPath: string): Promise<ModelSizeInfo>`
|
||||
|
||||
Get detailed model size information.
|
||||
|
||||
```typescript
|
||||
const sizeInfo = await ModelCompressor.getModelSize('/path/to/model.json')
|
||||
console.log(`Total size: ${sizeInfo.totalSize} bytes`)
|
||||
```
|
||||
|
||||
### Utility Functions
|
||||
|
||||
#### `utils.checkModelsAvailable(): boolean`
|
||||
|
||||
Check if bundled models are available.
|
||||
|
||||
```typescript
|
||||
import { utils } from '@soulcraft/brainy-models'
|
||||
|
||||
if (utils.checkModelsAvailable()) {
|
||||
console.log('Models are ready to use')
|
||||
}
|
||||
```
|
||||
|
||||
#### `utils.listAvailableModels(): string[]`
|
||||
|
||||
List available bundled models.
|
||||
|
||||
```typescript
|
||||
const models = utils.listAvailableModels()
|
||||
console.log('Available models:', models)
|
||||
```
|
||||
|
||||
## 🎯 Model Variants
|
||||
|
||||
The package includes multiple model variants optimized for different use cases:
|
||||
|
||||
### Original (Float32)
|
||||
- **Size**: ~25MB
|
||||
- **Use case**: Maximum accuracy
|
||||
- **Memory**: High
|
||||
- **Speed**: Fast
|
||||
|
||||
### Float16 Compressed
|
||||
- **Size**: ~12-15MB
|
||||
- **Use case**: Balanced performance
|
||||
- **Memory**: Medium
|
||||
- **Speed**: Fast
|
||||
|
||||
### Int8 Quantized
|
||||
- **Size**: ~6-8MB
|
||||
- **Use case**: Memory-constrained environments
|
||||
- **Memory**: Low
|
||||
- **Speed**: Medium
|
||||
|
||||
## ⚙️ Scripts
|
||||
|
||||
The package includes several utility scripts:
|
||||
|
||||
### Download Models
|
||||
|
||||
Download the complete Universal Sentence Encoder model:
|
||||
|
||||
```bash
|
||||
npm run download-models
|
||||
```
|
||||
|
||||
### Compress Models
|
||||
|
||||
Create optimized model variants:
|
||||
|
||||
```bash
|
||||
npm run compress-models
|
||||
```
|
||||
|
||||
### Test Models
|
||||
|
||||
Verify model functionality:
|
||||
|
||||
```bash
|
||||
npm test
|
||||
```
|
||||
|
||||
## 🔨 Development
|
||||
|
||||
### Building the Package
|
||||
|
||||
```bash
|
||||
npm run build
|
||||
```
|
||||
|
||||
### Running Tests
|
||||
|
||||
```bash
|
||||
npm test
|
||||
```
|
||||
|
||||
### Creating a Release
|
||||
|
||||
```bash
|
||||
npm run pack
|
||||
```
|
||||
|
||||
## ⚖️ Comparison with Online Loading
|
||||
|
||||
| Feature | Online Loading | Bundled Models |
|
||||
|---------|----------------|----------------|
|
||||
| **Reliability** | Network dependent | 100% offline |
|
||||
| **First load time** | 30-60 seconds | < 1 second |
|
||||
| **Subsequent loads** | Cached (~1 second) | < 1 second |
|
||||
| **Package size** | ~3KB | ~25MB |
|
||||
| **Network required** | Yes (first time) | No |
|
||||
| **Offline support** | Limited | Complete |
|
||||
|
||||
## 💡 Use Cases
|
||||
|
||||
### When to Use Bundled Models
|
||||
|
||||
- ✅ Production applications requiring maximum reliability
|
||||
- ✅ Offline or air-gapped environments
|
||||
- ✅ Applications with strict SLA requirements
|
||||
- ✅ Edge computing and IoT devices
|
||||
- ✅ Development environments with unreliable internet
|
||||
|
||||
### When to Use Online Loading
|
||||
|
||||
- ✅ Development and prototyping
|
||||
- ✅ Applications where package size matters
|
||||
- ✅ Environments with reliable internet connectivity
|
||||
- ✅ Applications that rarely use embeddings
|
||||
|
||||
## 🔧 Troubleshooting
|
||||
|
||||
### Model Not Found Error
|
||||
|
||||
```
|
||||
Error: Bundled model not found. Please run "npm run download-models"
|
||||
```
|
||||
|
||||
**Solution**: Run the download script to fetch the model files:
|
||||
```bash
|
||||
cd node_modules/@soulcraft/brainy-models
|
||||
npm run download-models
|
||||
```
|
||||
|
||||
### Memory Issues
|
||||
|
||||
If you encounter memory issues, try using compressed models:
|
||||
|
||||
```typescript
|
||||
const encoder = new BundledUniversalSentenceEncoder({
|
||||
preferCompressed: true
|
||||
})
|
||||
```
|
||||
|
||||
### Performance Optimization
|
||||
|
||||
For optimal performance:
|
||||
|
||||
1. **Memory-constrained**: Use int8 quantized models
|
||||
2. **Speed-critical**: Use original float32 models
|
||||
3. **Balanced**: Use float16 compressed models
|
||||
|
||||
## 📄 License
|
||||
|
||||
MIT
|
||||
|
||||
## 🤝 Contributing
|
||||
|
||||
Contributions are welcome! Please see the main [Brainy repository](https://github.com/soulcraft-research/brainy) for contribution guidelines.
|
||||
|
||||
## 💬 Support
|
||||
|
||||
For issues and questions:
|
||||
- [GitHub Issues](https://github.com/soulcraft-research/brainy/issues)
|
||||
- [Documentation](https://github.com/soulcraft-research/brainy)
|
||||
103
brainy-models-package/dist/index.d.ts
vendored
Normal file
103
brainy-models-package/dist/index.d.ts
vendored
Normal file
|
|
@ -0,0 +1,103 @@
|
|||
/**
|
||||
* @soulcraft/brainy-models
|
||||
*
|
||||
* Pre-bundled TensorFlow models for maximum reliability with Brainy vector database.
|
||||
* This package provides offline access to the Universal Sentence Encoder model,
|
||||
* eliminating network dependencies and ensuring consistent performance.
|
||||
*/
|
||||
import * as tf from '@tensorflow/tfjs';
|
||||
export interface ModelMetadata {
|
||||
name: string;
|
||||
version: string;
|
||||
description: string;
|
||||
dimensions: number;
|
||||
downloadDate: string;
|
||||
source: string;
|
||||
approach: string;
|
||||
modelUrl: string;
|
||||
bundledLocally: boolean;
|
||||
reliability: string;
|
||||
}
|
||||
export interface BundledModelOptions {
|
||||
verbose?: boolean;
|
||||
preferCompressed?: boolean;
|
||||
}
|
||||
/**
|
||||
* Bundled Universal Sentence Encoder for offline use
|
||||
*/
|
||||
export declare class BundledUniversalSentenceEncoder {
|
||||
private model;
|
||||
private metadata;
|
||||
private options;
|
||||
constructor(options?: BundledModelOptions);
|
||||
/**
|
||||
* Load the bundled model from local files
|
||||
*/
|
||||
load(): Promise<void>;
|
||||
/**
|
||||
* Generate embeddings for the given texts
|
||||
*/
|
||||
embed(texts: string[]): Promise<tf.Tensor2D>;
|
||||
/**
|
||||
* Generate embeddings and return as JavaScript arrays
|
||||
*/
|
||||
embedToArrays(texts: string[]): Promise<number[][]>;
|
||||
/**
|
||||
* Get model metadata
|
||||
*/
|
||||
getMetadata(): ModelMetadata | null;
|
||||
/**
|
||||
* Check if the model is loaded
|
||||
*/
|
||||
isLoaded(): boolean;
|
||||
/**
|
||||
* Get model information
|
||||
*/
|
||||
getModelInfo(): {
|
||||
inputShape: number[];
|
||||
outputShape: number[];
|
||||
} | null;
|
||||
/**
|
||||
* Dispose of the model and free memory
|
||||
*/
|
||||
dispose(): void;
|
||||
}
|
||||
/**
|
||||
* Model compression utilities
|
||||
*/
|
||||
export declare class ModelCompressor {
|
||||
/**
|
||||
* Compress model weights using quantization
|
||||
* Note: TensorFlow.js doesn't currently support model quantization
|
||||
*/
|
||||
static quantizeModel(modelPath: string, outputPath: string, options?: {
|
||||
dtype?: 'int8' | 'int16';
|
||||
}): Promise<void>;
|
||||
/**
|
||||
* Get model size information by reading files from disk
|
||||
*/
|
||||
static getModelSize(modelPath: string): Promise<{
|
||||
totalSize: number;
|
||||
weightsSize: number;
|
||||
modelJsonSize: number;
|
||||
}>;
|
||||
}
|
||||
/**
|
||||
* Utility functions
|
||||
*/
|
||||
export declare const utils: {
|
||||
/**
|
||||
* Check if bundled models are available
|
||||
*/
|
||||
checkModelsAvailable(): boolean;
|
||||
/**
|
||||
* Get bundled models directory
|
||||
*/
|
||||
getModelsDirectory(): string;
|
||||
/**
|
||||
* List available bundled models
|
||||
*/
|
||||
listAvailableModels(): string[];
|
||||
};
|
||||
export default BundledUniversalSentenceEncoder;
|
||||
//# sourceMappingURL=index.d.ts.map
|
||||
1
brainy-models-package/dist/index.d.ts.map
vendored
Normal file
1
brainy-models-package/dist/index.d.ts.map
vendored
Normal file
|
|
@ -0,0 +1 @@
|
|||
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH,OAAO,KAAK,EAAE,MAAM,kBAAkB,CAAA;AAwBtC,MAAM,WAAW,aAAa;IAC5B,IAAI,EAAE,MAAM,CAAA;IACZ,OAAO,EAAE,MAAM,CAAA;IACf,WAAW,EAAE,MAAM,CAAA;IACnB,UAAU,EAAE,MAAM,CAAA;IAClB,YAAY,EAAE,MAAM,CAAA;IACpB,MAAM,EAAE,MAAM,CAAA;IACd,QAAQ,EAAE,MAAM,CAAA;IAChB,QAAQ,EAAE,MAAM,CAAA;IAChB,cAAc,EAAE,OAAO,CAAA;IACvB,WAAW,EAAE,MAAM,CAAA;CACpB;AAED,MAAM,WAAW,mBAAmB;IAClC,OAAO,CAAC,EAAE,OAAO,CAAA;IACjB,gBAAgB,CAAC,EAAE,OAAO,CAAA;CAC3B;AAED;;GAEG;AACH,qBAAa,+BAA+B;IAC1C,OAAO,CAAC,KAAK,CAA6B;IAC1C,OAAO,CAAC,QAAQ,CAA6B;IAC7C,OAAO,CAAC,OAAO,CAAqB;gBAExB,OAAO,GAAE,mBAAwB;IAQ7C;;OAEG;IACG,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;IAwC3B;;OAEG;IACG,KAAK,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,EAAE,CAAC,QAAQ,CAAC;IAqBlD;;OAEG;IACG,aAAa,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC;IAOzD;;OAEG;IACH,WAAW,IAAI,aAAa,GAAG,IAAI;IAInC;;OAEG;IACH,QAAQ,IAAI,OAAO;IAInB;;OAEG;IACH,YAAY,IAAI;QAAE,UAAU,EAAE,MAAM,EAAE,CAAC;QAAC,WAAW,EAAE,MAAM,EAAE,CAAA;KAAE,GAAG,IAAI;IAWtE;;OAEG;IACH,OAAO,IAAI,IAAI;CAMhB;AAED;;GAEG;AACH,qBAAa,eAAe;IAC1B;;;OAGG;WACU,aAAa,CACxB,SAAS,EAAE,MAAM,EACjB,UAAU,EAAE,MAAM,EAClB,OAAO,GAAE;QAAE,KAAK,CAAC,EAAE,MAAM,GAAG,OAAO,CAAA;KAAO,GACzC,OAAO,CAAC,IAAI,CAAC;IAwBhB;;OAEG;WACU,YAAY,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC;QACpD,SAAS,EAAE,MAAM,CAAA;QACjB,WAAW,EAAE,MAAM,CAAA;QACnB,aAAa,EAAE,MAAM,CAAA;KACtB,CAAC;CAuCH;AAED;;GAEG;AACH,eAAO,MAAM,KAAK;IAChB;;OAEG;4BACqB,OAAO;IAK/B;;OAEG;0BACmB,MAAM;IAI5B;;OAEG;2BACoB,MAAM,EAAE;CAUhC,CAAA;AAGD,eAAe,+BAA+B,CAAA"}
|
||||
237
brainy-models-package/dist/index.js
vendored
Normal file
237
brainy-models-package/dist/index.js
vendored
Normal file
|
|
@ -0,0 +1,237 @@
|
|||
/**
|
||||
* @soulcraft/brainy-models
|
||||
*
|
||||
* Pre-bundled TensorFlow models for maximum reliability with Brainy vector database.
|
||||
* This package provides offline access to the Universal Sentence Encoder model,
|
||||
* eliminating network dependencies and ensuring consistent performance.
|
||||
*/
|
||||
import * as tf from '@tensorflow/tfjs';
|
||||
import { readFileSync, existsSync } from 'fs';
|
||||
import { join, dirname } from 'path';
|
||||
import { fileURLToPath } from 'url';
|
||||
/**
|
||||
* Helper function to safely extract error message from unknown error type
|
||||
*/
|
||||
function getErrorMessage(error) {
|
||||
if (error instanceof Error) {
|
||||
return error.message;
|
||||
}
|
||||
if (typeof error === 'string') {
|
||||
return error;
|
||||
}
|
||||
return String(error);
|
||||
}
|
||||
// Get the package directory
|
||||
const __filename = fileURLToPath(import.meta.url);
|
||||
const __dirname = dirname(__filename);
|
||||
const PACKAGE_ROOT = join(__dirname, '..');
|
||||
const MODELS_DIR = join(PACKAGE_ROOT, 'models');
|
||||
/**
|
||||
* Bundled Universal Sentence Encoder for offline use
|
||||
*/
|
||||
export class BundledUniversalSentenceEncoder {
|
||||
model = null;
|
||||
metadata = null;
|
||||
options;
|
||||
constructor(options = {}) {
|
||||
this.options = {
|
||||
verbose: false,
|
||||
preferCompressed: false,
|
||||
...options
|
||||
};
|
||||
}
|
||||
/**
|
||||
* Load the bundled model from local files
|
||||
*/
|
||||
async load() {
|
||||
try {
|
||||
const modelDir = join(MODELS_DIR, 'universal-sentence-encoder');
|
||||
const modelPath = join(modelDir, 'model.json');
|
||||
const metadataPath = join(modelDir, 'metadata.json');
|
||||
if (!existsSync(modelPath)) {
|
||||
throw new Error(`Bundled model not found at ${modelPath}. ` +
|
||||
'Please run "npm run download-models" to download the model files.');
|
||||
}
|
||||
if (this.options.verbose) {
|
||||
console.log('🔄 Loading bundled Universal Sentence Encoder model...');
|
||||
}
|
||||
// Load metadata
|
||||
if (existsSync(metadataPath)) {
|
||||
const metadataContent = readFileSync(metadataPath, 'utf8');
|
||||
this.metadata = JSON.parse(metadataContent);
|
||||
if (this.options.verbose) {
|
||||
console.log(`📋 Model metadata:`, this.metadata);
|
||||
}
|
||||
}
|
||||
// Load the model
|
||||
this.model = await tf.loadGraphModel(`file://${modelPath}`);
|
||||
if (this.options.verbose) {
|
||||
console.log('✅ Bundled model loaded successfully');
|
||||
console.log(`🔒 Reliability: Maximum (fully offline)`);
|
||||
}
|
||||
}
|
||||
catch (error) {
|
||||
throw new Error(`Failed to load bundled model: ${getErrorMessage(error)}`);
|
||||
}
|
||||
}
|
||||
/**
|
||||
* Generate embeddings for the given texts
|
||||
*/
|
||||
async embed(texts) {
|
||||
if (!this.model) {
|
||||
throw new Error('Model not loaded. Call load() first.');
|
||||
}
|
||||
try {
|
||||
// Convert texts to tensor
|
||||
const inputTensor = tf.tensor1d(texts, 'string');
|
||||
// Run inference
|
||||
const embeddings = this.model.predict(inputTensor);
|
||||
// Clean up input tensor
|
||||
inputTensor.dispose();
|
||||
return embeddings;
|
||||
}
|
||||
catch (error) {
|
||||
throw new Error(`Failed to generate embeddings: ${getErrorMessage(error)}`);
|
||||
}
|
||||
}
|
||||
/**
|
||||
* Generate embeddings and return as JavaScript arrays
|
||||
*/
|
||||
async embedToArrays(texts) {
|
||||
const embeddings = await this.embed(texts);
|
||||
const arrays = await embeddings.array();
|
||||
embeddings.dispose();
|
||||
return arrays;
|
||||
}
|
||||
/**
|
||||
* Get model metadata
|
||||
*/
|
||||
getMetadata() {
|
||||
return this.metadata;
|
||||
}
|
||||
/**
|
||||
* Check if the model is loaded
|
||||
*/
|
||||
isLoaded() {
|
||||
return this.model !== null;
|
||||
}
|
||||
/**
|
||||
* Get model information
|
||||
*/
|
||||
getModelInfo() {
|
||||
if (!this.model) {
|
||||
return null;
|
||||
}
|
||||
return {
|
||||
inputShape: this.model.inputs[0].shape || [],
|
||||
outputShape: this.model.outputs[0].shape || []
|
||||
};
|
||||
}
|
||||
/**
|
||||
* Dispose of the model and free memory
|
||||
*/
|
||||
dispose() {
|
||||
if (this.model) {
|
||||
this.model.dispose();
|
||||
this.model = null;
|
||||
}
|
||||
}
|
||||
}
|
||||
/**
|
||||
* Model compression utilities
|
||||
*/
|
||||
export class ModelCompressor {
|
||||
/**
|
||||
* Compress model weights using quantization
|
||||
* Note: TensorFlow.js doesn't currently support model quantization
|
||||
*/
|
||||
static async quantizeModel(modelPath, outputPath, options = {}) {
|
||||
const { dtype = 'int8' } = options;
|
||||
try {
|
||||
console.log(`🔄 Loading model for quantization: ${modelPath}`);
|
||||
const model = await tf.loadGraphModel(`file://${modelPath}`);
|
||||
console.log(`🗜️ Quantizing model to ${dtype}...`);
|
||||
// TensorFlow.js doesn't have built-in quantization or model serialization APIs yet
|
||||
// This is a placeholder implementation that acknowledges the limitation
|
||||
console.warn('⚠️ Model quantization is not yet supported in TensorFlow.js');
|
||||
console.log(`📋 Model loaded successfully from: ${modelPath}`);
|
||||
console.log(`📋 Target output path: ${outputPath}`);
|
||||
console.log(`📋 Target dtype: ${dtype}`);
|
||||
model.dispose();
|
||||
throw new Error('Model quantization is not yet supported in TensorFlow.js. This feature requires server-side processing with TensorFlow Python.');
|
||||
}
|
||||
catch (error) {
|
||||
throw new Error(`Failed to compress model: ${getErrorMessage(error)}`);
|
||||
}
|
||||
}
|
||||
/**
|
||||
* Get model size information by reading files from disk
|
||||
*/
|
||||
static async getModelSize(modelPath) {
|
||||
try {
|
||||
// Load model to verify it's valid
|
||||
const model = await tf.loadGraphModel(`file://${modelPath}`);
|
||||
model.dispose();
|
||||
// Get model.json size
|
||||
const modelJsonSize = existsSync(modelPath) ? readFileSync(modelPath).length : 0;
|
||||
// Calculate weights size by reading weight files
|
||||
let weightsSize = 0;
|
||||
const modelDir = dirname(modelPath);
|
||||
// Read model.json to get weight file names
|
||||
if (existsSync(modelPath)) {
|
||||
const modelJson = JSON.parse(readFileSync(modelPath, 'utf8'));
|
||||
if (modelJson.weightsManifest) {
|
||||
for (const manifest of modelJson.weightsManifest) {
|
||||
for (const path of manifest.paths) {
|
||||
const weightFilePath = join(modelDir, path);
|
||||
if (existsSync(weightFilePath)) {
|
||||
weightsSize += readFileSync(weightFilePath).length;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
const totalSize = weightsSize + modelJsonSize;
|
||||
return {
|
||||
totalSize,
|
||||
weightsSize,
|
||||
modelJsonSize
|
||||
};
|
||||
}
|
||||
catch (error) {
|
||||
throw new Error(`Failed to get model size: ${getErrorMessage(error)}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
/**
|
||||
* Utility functions
|
||||
*/
|
||||
export const utils = {
|
||||
/**
|
||||
* Check if bundled models are available
|
||||
*/
|
||||
checkModelsAvailable() {
|
||||
const modelPath = join(MODELS_DIR, 'universal-sentence-encoder', 'model.json');
|
||||
return existsSync(modelPath);
|
||||
},
|
||||
/**
|
||||
* Get bundled models directory
|
||||
*/
|
||||
getModelsDirectory() {
|
||||
return MODELS_DIR;
|
||||
},
|
||||
/**
|
||||
* List available bundled models
|
||||
*/
|
||||
listAvailableModels() {
|
||||
const models = [];
|
||||
const useModelPath = join(MODELS_DIR, 'universal-sentence-encoder', 'model.json');
|
||||
if (existsSync(useModelPath)) {
|
||||
models.push('universal-sentence-encoder');
|
||||
}
|
||||
return models;
|
||||
}
|
||||
};
|
||||
// Default export for convenience
|
||||
export default BundledUniversalSentenceEncoder;
|
||||
//# sourceMappingURL=index.js.map
|
||||
1
brainy-models-package/dist/index.js.map
vendored
Normal file
1
brainy-models-package/dist/index.js.map
vendored
Normal file
File diff suppressed because one or more lines are too long
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
|
|
@ -0,0 +1,13 @@
|
|||
{
|
||||
"name": "universal-sentence-encoder-lite",
|
||||
"version": "1.0.0",
|
||||
"description": "Universal Sentence Encoder Lite model with tokenizer support",
|
||||
"dimensions": 512,
|
||||
"downloadDate": "2025-08-06T00:56:00.000Z",
|
||||
"source": "https://www.kaggle.com/models/tensorflow/universal-sentence-encoder/tfJs/lite/1",
|
||||
"approach": "full-bundle",
|
||||
"modelUrl": "https://www.kaggle.com/models/tensorflow/universal-sentence-encoder/tfJs/lite/1",
|
||||
"bundledLocally": true,
|
||||
"reliability": "maximum",
|
||||
"notes": "This model requires tokenization of input text. Use with @tensorflow-models/universal-sentence-encoder for proper tokenization."
|
||||
}
|
||||
7182
brainy-models-package/models/universal-sentence-encoder/model.json
Normal file
7182
brainy-models-package/models/universal-sentence-encoder/model.json
Normal file
File diff suppressed because it is too large
Load diff
1
brainy-models-package/models/vocab.json
Normal file
1
brainy-models-package/models/vocab.json
Normal file
File diff suppressed because one or more lines are too long
78
brainy-models-package/package.json
Normal file
78
brainy-models-package/package.json
Normal file
|
|
@ -0,0 +1,78 @@
|
|||
{
|
||||
"name": "@soulcraft/brainy-models",
|
||||
"version": "0.8.0",
|
||||
"description": "Pre-bundled TensorFlow models for maximum reliability with Brainy vector database",
|
||||
"main": "dist/index.js",
|
||||
"module": "dist/index.js",
|
||||
"types": "dist/index.d.ts",
|
||||
"type": "module",
|
||||
"engines": {
|
||||
"node": ">=18.0.0"
|
||||
},
|
||||
"scripts": {
|
||||
"prebuild": "echo 'Models already downloaded'",
|
||||
"build": "tsc",
|
||||
"test": "node test/test-models.js",
|
||||
"prepare": "npm run build",
|
||||
"download-models": "node scripts/download-full-models.js",
|
||||
"compress-models": "node scripts/compress-models.js",
|
||||
"_pack": "npm pack",
|
||||
"_release": "standard-version",
|
||||
"_release:patch": "standard-version --release-as patch",
|
||||
"_release:minor": "standard-version --release-as minor",
|
||||
"_release:major": "standard-version --release-as major",
|
||||
"_release:dry-run": "standard-version --dry-run",
|
||||
"_github-release": "node scripts/create-github-release.js",
|
||||
"_workflow": "node scripts/release-workflow.js",
|
||||
"_workflow:patch": "node scripts/release-workflow.js patch",
|
||||
"_workflow:minor": "node scripts/release-workflow.js minor",
|
||||
"_workflow:major": "node scripts/release-workflow.js major",
|
||||
"_workflow:dry-run": "npm run build && npm test && npm run _release:dry-run",
|
||||
"_deploy": "npm run build && npm publish",
|
||||
"release:minor": "npm run _release:minor"
|
||||
},
|
||||
"keywords": [
|
||||
"tensorflow",
|
||||
"models",
|
||||
"universal-sentence-encoder",
|
||||
"embeddings",
|
||||
"brainy",
|
||||
"vector-database",
|
||||
"offline",
|
||||
"bundled"
|
||||
],
|
||||
"author": "David Snelling (david@soulcraft.com)",
|
||||
"license": "MIT",
|
||||
"private": false,
|
||||
"publishConfig": {
|
||||
"access": "public"
|
||||
},
|
||||
"homepage": "https://github.com/soulcraft-research/brainy",
|
||||
"bugs": {
|
||||
"url": "https://github.com/soulcraft-research/brainy/issues"
|
||||
},
|
||||
"repository": {
|
||||
"type": "git",
|
||||
"url": "git+https://github.com/soulcraft-research/brainy.git",
|
||||
"directory": "brainy-models-package"
|
||||
},
|
||||
"files": [
|
||||
"dist/",
|
||||
"models/",
|
||||
"README.md",
|
||||
"LICENSE"
|
||||
],
|
||||
"dependencies": {
|
||||
"@tensorflow-models/universal-sentence-encoder": "^1.3.3",
|
||||
"@tensorflow/tfjs": "^4.23.0-rc.0",
|
||||
"@tensorflow/tfjs-node": "^4.23.0-rc.0"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@types/node": "^20.11.30",
|
||||
"standard-version": "^9.5.0",
|
||||
"typescript": "^5.4.5"
|
||||
},
|
||||
"peerDependencies": {
|
||||
"@soulcraft/brainy": ">=0.33.0"
|
||||
}
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue