**feat(models): add pre-bundled Universal Sentence Encoder for offline use**
- Introduced `@soulcraft/brainy-models` package with pre-bundled TensorFlow models for enhanced offline reliability. - Added `index.d.ts` and `index.js` allowing offline embedding workflows with the Universal Sentence Encoder model. - Included utility scripts for model compression, size retrieval, and availability checks. - Added `metadata.json` and `model.json` defining the Universal Sentence Encoder configuration with offline bundling. - Ensured comprehensive model documentation, error handling, and robust logging for seamless integration. - Supported optional model quantization placeholders for future TensorFlow.js enhancements. **Purpose**: Enable fully offline-ready embedding workflows via pre-bundled Universal Sentence Encoder models, ensuring maximum reliability and air-gapped environment compatibility.
This commit is contained in:
parent
93483572d8
commit
e476d45fac
16 changed files with 7678 additions and 52 deletions
103
brainy-models-package/dist/index.d.ts
vendored
Normal file
103
brainy-models-package/dist/index.d.ts
vendored
Normal file
|
|
@ -0,0 +1,103 @@
|
|||
/**
|
||||
* @soulcraft/brainy-models
|
||||
*
|
||||
* Pre-bundled TensorFlow models for maximum reliability with Brainy vector database.
|
||||
* This package provides offline access to the Universal Sentence Encoder model,
|
||||
* eliminating network dependencies and ensuring consistent performance.
|
||||
*/
|
||||
import * as tf from '@tensorflow/tfjs';
|
||||
export interface ModelMetadata {
|
||||
name: string;
|
||||
version: string;
|
||||
description: string;
|
||||
dimensions: number;
|
||||
downloadDate: string;
|
||||
source: string;
|
||||
approach: string;
|
||||
modelUrl: string;
|
||||
bundledLocally: boolean;
|
||||
reliability: string;
|
||||
}
|
||||
export interface BundledModelOptions {
|
||||
verbose?: boolean;
|
||||
preferCompressed?: boolean;
|
||||
}
|
||||
/**
|
||||
* Bundled Universal Sentence Encoder for offline use
|
||||
*/
|
||||
export declare class BundledUniversalSentenceEncoder {
|
||||
private model;
|
||||
private metadata;
|
||||
private options;
|
||||
constructor(options?: BundledModelOptions);
|
||||
/**
|
||||
* Load the bundled model from local files
|
||||
*/
|
||||
load(): Promise<void>;
|
||||
/**
|
||||
* Generate embeddings for the given texts
|
||||
*/
|
||||
embed(texts: string[]): Promise<tf.Tensor2D>;
|
||||
/**
|
||||
* Generate embeddings and return as JavaScript arrays
|
||||
*/
|
||||
embedToArrays(texts: string[]): Promise<number[][]>;
|
||||
/**
|
||||
* Get model metadata
|
||||
*/
|
||||
getMetadata(): ModelMetadata | null;
|
||||
/**
|
||||
* Check if the model is loaded
|
||||
*/
|
||||
isLoaded(): boolean;
|
||||
/**
|
||||
* Get model information
|
||||
*/
|
||||
getModelInfo(): {
|
||||
inputShape: number[];
|
||||
outputShape: number[];
|
||||
} | null;
|
||||
/**
|
||||
* Dispose of the model and free memory
|
||||
*/
|
||||
dispose(): void;
|
||||
}
|
||||
/**
|
||||
* Model compression utilities
|
||||
*/
|
||||
export declare class ModelCompressor {
|
||||
/**
|
||||
* Compress model weights using quantization
|
||||
* Note: TensorFlow.js doesn't currently support model quantization
|
||||
*/
|
||||
static quantizeModel(modelPath: string, outputPath: string, options?: {
|
||||
dtype?: 'int8' | 'int16';
|
||||
}): Promise<void>;
|
||||
/**
|
||||
* Get model size information by reading files from disk
|
||||
*/
|
||||
static getModelSize(modelPath: string): Promise<{
|
||||
totalSize: number;
|
||||
weightsSize: number;
|
||||
modelJsonSize: number;
|
||||
}>;
|
||||
}
|
||||
/**
|
||||
* Utility functions
|
||||
*/
|
||||
export declare const utils: {
|
||||
/**
|
||||
* Check if bundled models are available
|
||||
*/
|
||||
checkModelsAvailable(): boolean;
|
||||
/**
|
||||
* Get bundled models directory
|
||||
*/
|
||||
getModelsDirectory(): string;
|
||||
/**
|
||||
* List available bundled models
|
||||
*/
|
||||
listAvailableModels(): string[];
|
||||
};
|
||||
export default BundledUniversalSentenceEncoder;
|
||||
//# sourceMappingURL=index.d.ts.map
|
||||
Loading…
Add table
Add a link
Reference in a new issue