/** * Embedding functions for converting data to vectors */ import { EmbeddingFunction, EmbeddingModel, Vector } from '../coreTypes.js' /** * Simple character-based embedding function * This is a very basic implementation for demo purposes */ export class SimpleEmbedding implements EmbeddingModel { private initialized = false /** * Initialize the embedding model */ public async init(): Promise { this.initialized = true return Promise.resolve() } /** * Embed text into a vector using character frequencies * @param data Text to embed */ public async embed(data: string): Promise { if (!this.initialized) { await this.init() } // Only handle string data if (typeof data !== 'string') { throw new Error('SimpleEmbedding only supports string data') } // Normalize the text const normalizedText = data.toLowerCase().trim() // Create a simple 4-dimensional vector based on character frequencies const vector: Vector = [0, 0, 0, 0] // Count vowels, consonants, numbers, and special characters for (let i = 0; i < normalizedText.length; i++) { const char = normalizedText[i] if ('aeiou'.includes(char)) { vector[0] += 0.1 // Vowels affect first dimension } else if ('bcdfghjklmnpqrstvwxyz'.includes(char)) { vector[1] += 0.1 // Consonants affect second dimension } else if ('0123456789'.includes(char)) { vector[2] += 0.1 // Numbers affect third dimension } else { vector[3] += 0.1 // Special chars affect fourth dimension } } // Normalize the vector const magnitude = Math.sqrt( vector.reduce((sum, val) => sum + val * val, 0) ) if (magnitude > 0) { return vector.map((val) => val / magnitude) } return vector } /** * Dispose of the model resources */ public async dispose(): Promise { this.initialized = false return Promise.resolve() } } /** * TensorFlow Universal Sentence Encoder embedding model * This model provides high-quality text embeddings using TensorFlow.js * The required TensorFlow.js dependencies are automatically installed with this package */ export class UniversalSentenceEncoder implements EmbeddingModel { private model: any = null private initialized = false private tf: any = null private use: any = null /** * Initialize the embedding model */ public async init(): Promise { try { // Dynamically import TensorFlow.js and Universal Sentence Encoder // Use type assertions to tell TypeScript these modules exist this.tf = await import('@tensorflow/tfjs/dist/tf.esm.js' as any) this.use = await import('@tensorflow-models/universal-sentence-encoder/dist/universal-sentence-encoder.esm.js' as any) // Load the model this.model = await this.use.load() this.initialized = true } catch (error) { console.error('Failed to initialize Universal Sentence Encoder:', error) throw new Error( `Failed to initialize Universal Sentence Encoder: ${error}` ) } } /** * Embed text into a vector using Universal Sentence Encoder * @param data Text to embed */ public async embed(data: string | string[]): Promise { if (!this.initialized) { await this.init() } try { // Handle different input types let textToEmbed: string[] if (typeof data === 'string') { textToEmbed = [data] } else if (Array.isArray(data) && data.every(item => typeof item === 'string')) { textToEmbed = data } else { throw new Error('UniversalSentenceEncoder only supports string or string[] data') } // Get embeddings const embeddings = await this.model.embed(textToEmbed) // Convert to array and return the first embedding const embeddingArray = await embeddings.array() return embeddingArray[0] } catch (error) { console.error('Failed to embed text with Universal Sentence Encoder:', error) throw new Error(`Failed to embed text with Universal Sentence Encoder: ${error}`) } } /** * Dispose of the model resources */ public async dispose(): Promise { if (this.model && this.tf) { try { // Dispose of the model and tensors this.model.dispose() this.tf.disposeVariables() this.initialized = false } catch (error) { console.error('Failed to dispose Universal Sentence Encoder:', error) } } return Promise.resolve() } } /** * Create an embedding function from an embedding model * @param model Embedding model to use */ export function createEmbeddingFunction(model: EmbeddingModel): EmbeddingFunction { return async (data: any): Promise => { return await model.embed(data) } } /** * Creates a TensorFlow-based Universal Sentence Encoder embedding function * This is the recommended embedding function for high-quality text embeddings */ export function createTensorFlowEmbeddingFunction(): EmbeddingFunction { return createEmbeddingFunction(new UniversalSentenceEncoder()) } /** * Simple embedding function using character-based embedding * This is a basic implementation that doesn't use TensorFlow */ export function createSimpleEmbeddingFunction(): EmbeddingFunction { return createEmbeddingFunction(new SimpleEmbedding()) } /** * Default embedding function using UniversalSentenceEncoder * This provides high-quality text embeddings using TensorFlow.js */ export const defaultEmbeddingFunction: EmbeddingFunction = createTensorFlowEmbeddingFunction()