feat: add brainy-models-package v0.8.0 with USE-lite model

- Add complete Universal Sentence Encoder Lite model (27MB)
- Include vocab.json for tokenization support
- Update package to work with @tensorflow-models/universal-sentence-encoder
- Ensure offline model loading capability for Docker deployments
- Published to npm as @soulcraft/brainy-models@0.8.0

brainy-models-package/dist/index.d.ts

@@ -0,0 +1,103 @@
+/**
+ * @soulcraft/brainy-models
+ *
+ * Pre-bundled TensorFlow models for maximum reliability with Brainy vector database.
+ * This package provides offline access to the Universal Sentence Encoder model,
+ * eliminating network dependencies and ensuring consistent performance.
+ */
+import * as tf from '@tensorflow/tfjs';
+export interface ModelMetadata {
+    name: string;
+    version: string;
+    description: string;
+    dimensions: number;
+    downloadDate: string;
+    source: string;
+    approach: string;
+    modelUrl: string;
+    bundledLocally: boolean;
+    reliability: string;
+}
+export interface BundledModelOptions {
+    verbose?: boolean;
+    preferCompressed?: boolean;
+}
+/**
+ * Bundled Universal Sentence Encoder for offline use
+ */
+export declare class BundledUniversalSentenceEncoder {
+    private model;
+    private metadata;
+    private options;
+    constructor(options?: BundledModelOptions);
+    /**
+     * Load the bundled model from local files
+     */
+    load(): Promise<void>;
+    /**
+     * Generate embeddings for the given texts
+     */
+    embed(texts: string[]): Promise<tf.Tensor2D>;
+    /**
+     * Generate embeddings and return as JavaScript arrays
+     */
+    embedToArrays(texts: string[]): Promise<number[][]>;
+    /**
+     * Get model metadata
+     */
+    getMetadata(): ModelMetadata | null;
+    /**
+     * Check if the model is loaded
+     */
+    isLoaded(): boolean;
+    /**
+     * Get model information
+     */
+    getModelInfo(): {
+        inputShape: number[];
+        outputShape: number[];
+    } | null;
+    /**
+     * Dispose of the model and free memory
+     */
+    dispose(): void;
+}
+/**
+ * Model compression utilities
+ */
+export declare class ModelCompressor {
+    /**
+     * Compress model weights using quantization
+     * Note: TensorFlow.js doesn't currently support model quantization
+     */
+    static quantizeModel(modelPath: string, outputPath: string, options?: {
+        dtype?: 'int8' | 'int16';
+    }): Promise<void>;
+    /**
+     * Get model size information by reading files from disk
+     */
+    static getModelSize(modelPath: string): Promise<{
+        totalSize: number;
+        weightsSize: number;
+        modelJsonSize: number;
+    }>;
+}
+/**
+ * Utility functions
+ */
+export declare const utils: {
+    /**
+     * Check if bundled models are available
+     */
+    checkModelsAvailable(): boolean;
+    /**
+     * Get bundled models directory
+     */
+    getModelsDirectory(): string;
+    /**
+     * List available bundled models
+     */
+    listAvailableModels(): string[];
+};
+export default BundledUniversalSentenceEncoder;
+//# sourceMappingURL=index.d.ts.map

brainy-models-package/dist/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH,OAAO,KAAK,EAAE,MAAM,kBAAkB,CAAA;AAwBtC,MAAM,WAAW,aAAa;IAC5B,IAAI,EAAE,MAAM,CAAA;IACZ,OAAO,EAAE,MAAM,CAAA;IACf,WAAW,EAAE,MAAM,CAAA;IACnB,UAAU,EAAE,MAAM,CAAA;IAClB,YAAY,EAAE,MAAM,CAAA;IACpB,MAAM,EAAE,MAAM,CAAA;IACd,QAAQ,EAAE,MAAM,CAAA;IAChB,QAAQ,EAAE,MAAM,CAAA;IAChB,cAAc,EAAE,OAAO,CAAA;IACvB,WAAW,EAAE,MAAM,CAAA;CACpB;AAED,MAAM,WAAW,mBAAmB;IAClC,OAAO,CAAC,EAAE,OAAO,CAAA;IACjB,gBAAgB,CAAC,EAAE,OAAO,CAAA;CAC3B;AAED;;GAEG;AACH,qBAAa,+BAA+B;IAC1C,OAAO,CAAC,KAAK,CAA6B;IAC1C,OAAO,CAAC,QAAQ,CAA6B;IAC7C,OAAO,CAAC,OAAO,CAAqB;gBAExB,OAAO,GAAE,mBAAwB;IAQ7C;;OAEG;IACG,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;IAwC3B;;OAEG;IACG,KAAK,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,EAAE,CAAC,QAAQ,CAAC;IAqBlD;;OAEG;IACG,aAAa,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC;IAOzD;;OAEG;IACH,WAAW,IAAI,aAAa,GAAG,IAAI;IAInC;;OAEG;IACH,QAAQ,IAAI,OAAO;IAInB;;OAEG;IACH,YAAY,IAAI;QAAE,UAAU,EAAE,MAAM,EAAE,CAAC;QAAC,WAAW,EAAE,MAAM,EAAE,CAAA;KAAE,GAAG,IAAI;IAWtE;;OAEG;IACH,OAAO,IAAI,IAAI;CAMhB;AAED;;GAEG;AACH,qBAAa,eAAe;IAC1B;;;OAGG;WACU,aAAa,CACxB,SAAS,EAAE,MAAM,EACjB,UAAU,EAAE,MAAM,EAClB,OAAO,GAAE;QAAE,KAAK,CAAC,EAAE,MAAM,GAAG,OAAO,CAAA;KAAO,GACzC,OAAO,CAAC,IAAI,CAAC;IAwBhB;;OAEG;WACU,YAAY,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC;QACpD,SAAS,EAAE,MAAM,CAAA;QACjB,WAAW,EAAE,MAAM,CAAA;QACnB,aAAa,EAAE,MAAM,CAAA;KACtB,CAAC;CAuCH;AAED;;GAEG;AACH,eAAO,MAAM,KAAK;IAChB;;OAEG;4BACqB,OAAO;IAK/B;;OAEG;0BACmB,MAAM;IAI5B;;OAEG;2BACoB,MAAM,EAAE;CAUhC,CAAA;AAGD,eAAe,+BAA+B,CAAA"}

brainy-models-package/dist/index.js

@@ -0,0 +1,237 @@
+/**
+ * @soulcraft/brainy-models
+ *
+ * Pre-bundled TensorFlow models for maximum reliability with Brainy vector database.
+ * This package provides offline access to the Universal Sentence Encoder model,
+ * eliminating network dependencies and ensuring consistent performance.
+ */
+import * as tf from '@tensorflow/tfjs';
+import { readFileSync, existsSync } from 'fs';
+import { join, dirname } from 'path';
+import { fileURLToPath } from 'url';
+/**
+ * Helper function to safely extract error message from unknown error type
+ */
+function getErrorMessage(error) {
+    if (error instanceof Error) {
+        return error.message;
+    }
+    if (typeof error === 'string') {
+        return error;
+    }
+    return String(error);
+}
+// Get the package directory
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = dirname(__filename);
+const PACKAGE_ROOT = join(__dirname, '..');
+const MODELS_DIR = join(PACKAGE_ROOT, 'models');
+/**
+ * Bundled Universal Sentence Encoder for offline use
+ */
+export class BundledUniversalSentenceEncoder {
+    model = null;
+    metadata = null;
+    options;
+    constructor(options = {}) {
+        this.options = {
+            verbose: false,
+            preferCompressed: false,
+            ...options
+        };
+    }
+    /**
+     * Load the bundled model from local files
+     */
+    async load() {
+        try {
+            const modelDir = join(MODELS_DIR, 'universal-sentence-encoder');
+            const modelPath = join(modelDir, 'model.json');
+            const metadataPath = join(modelDir, 'metadata.json');
+            if (!existsSync(modelPath)) {
+                throw new Error(`Bundled model not found at ${modelPath}. ` +
+                    'Please run "npm run download-models" to download the model files.');
+            }
+            if (this.options.verbose) {
+                console.log('🔄 Loading bundled Universal Sentence Encoder model...');
+            }
+            // Load metadata
+            if (existsSync(metadataPath)) {
+                const metadataContent = readFileSync(metadataPath, 'utf8');
+                this.metadata = JSON.parse(metadataContent);
+                if (this.options.verbose) {
+                    console.log(`📋 Model metadata:`, this.metadata);
+                }
+            }
+            // Load the model
+            this.model = await tf.loadGraphModel(`file://${modelPath}`);
+            if (this.options.verbose) {
+                console.log('✅ Bundled model loaded successfully');
+                console.log(`🔒 Reliability: Maximum (fully offline)`);
+            }
+        }
+        catch (error) {
+            throw new Error(`Failed to load bundled model: ${getErrorMessage(error)}`);
+        }
+    }
+    /**
+     * Generate embeddings for the given texts
+     */
+    async embed(texts) {
+        if (!this.model) {
+            throw new Error('Model not loaded. Call load() first.');
+        }
+        try {
+            // Convert texts to tensor
+            const inputTensor = tf.tensor1d(texts, 'string');
+            // Run inference
+            const embeddings = this.model.predict(inputTensor);
+            // Clean up input tensor
+            inputTensor.dispose();
+            return embeddings;
+        }
+        catch (error) {
+            throw new Error(`Failed to generate embeddings: ${getErrorMessage(error)}`);
+        }
+    }
+    /**
+     * Generate embeddings and return as JavaScript arrays
+     */
+    async embedToArrays(texts) {
+        const embeddings = await this.embed(texts);
+        const arrays = await embeddings.array();
+        embeddings.dispose();
+        return arrays;
+    }
+    /**
+     * Get model metadata
+     */
+    getMetadata() {
+        return this.metadata;
+    }
+    /**
+     * Check if the model is loaded
+     */
+    isLoaded() {
+        return this.model !== null;
+    }
+    /**
+     * Get model information
+     */
+    getModelInfo() {
+        if (!this.model) {
+            return null;
+        }
+        return {
+            inputShape: this.model.inputs[0].shape || [],
+            outputShape: this.model.outputs[0].shape || []
+        };
+    }
+    /**
+     * Dispose of the model and free memory
+     */
+    dispose() {
+        if (this.model) {
+            this.model.dispose();
+            this.model = null;
+        }
+    }
+}
+/**
+ * Model compression utilities
+ */
+export class ModelCompressor {
+    /**
+     * Compress model weights using quantization
+     * Note: TensorFlow.js doesn't currently support model quantization
+     */
+    static async quantizeModel(modelPath, outputPath, options = {}) {
+        const { dtype = 'int8' } = options;
+        try {
+            console.log(`🔄 Loading model for quantization: ${modelPath}`);
+            const model = await tf.loadGraphModel(`file://${modelPath}`);
+            console.log(`🗜️ Quantizing model to ${dtype}...`);
+            // TensorFlow.js doesn't have built-in quantization or model serialization APIs yet
+            // This is a placeholder implementation that acknowledges the limitation
+            console.warn('⚠️ Model quantization is not yet supported in TensorFlow.js');
+            console.log(`📋 Model loaded successfully from: ${modelPath}`);
+            console.log(`📋 Target output path: ${outputPath}`);
+            console.log(`📋 Target dtype: ${dtype}`);
+            model.dispose();
+            throw new Error('Model quantization is not yet supported in TensorFlow.js. This feature requires server-side processing with TensorFlow Python.');
+        }
+        catch (error) {
+            throw new Error(`Failed to compress model: ${getErrorMessage(error)}`);
+        }
+    }
+    /**
+     * Get model size information by reading files from disk
+     */
+    static async getModelSize(modelPath) {
+        try {
+            // Load model to verify it's valid
+            const model = await tf.loadGraphModel(`file://${modelPath}`);
+            model.dispose();
+            // Get model.json size
+            const modelJsonSize = existsSync(modelPath) ? readFileSync(modelPath).length : 0;
+            // Calculate weights size by reading weight files
+            let weightsSize = 0;
+            const modelDir = dirname(modelPath);
+            // Read model.json to get weight file names
+            if (existsSync(modelPath)) {
+                const modelJson = JSON.parse(readFileSync(modelPath, 'utf8'));
+                if (modelJson.weightsManifest) {
+                    for (const manifest of modelJson.weightsManifest) {
+                        for (const path of manifest.paths) {
+                            const weightFilePath = join(modelDir, path);
+                            if (existsSync(weightFilePath)) {
+                                weightsSize += readFileSync(weightFilePath).length;
+                            }
+                        }
+                    }
+                }
+            }
+            const totalSize = weightsSize + modelJsonSize;
+            return {
+                totalSize,
+                weightsSize,
+                modelJsonSize
+            };
+        }
+        catch (error) {
+            throw new Error(`Failed to get model size: ${getErrorMessage(error)}`);
+        }
+    }
+}
+/**
+ * Utility functions
+ */
+export const utils = {
+    /**
+     * Check if bundled models are available
+     */
+    checkModelsAvailable() {
+        const modelPath = join(MODELS_DIR, 'universal-sentence-encoder', 'model.json');
+        return existsSync(modelPath);
+    },
+    /**
+     * Get bundled models directory
+     */
+    getModelsDirectory() {
+        return MODELS_DIR;
+    },
+    /**
+     * List available bundled models
+     */
+    listAvailableModels() {
+        const models = [];
+        const useModelPath = join(MODELS_DIR, 'universal-sentence-encoder', 'model.json');
+        if (existsSync(useModelPath)) {
+            models.push('universal-sentence-encoder');
+        }
+        return models;
+    }
+};
+// Default export for convenience
+export default BundledUniversalSentenceEncoder;
+//# sourceMappingURL=index.js.map