Initial commit

src/utils/distance.ts

@@ -0,0 +1,86 @@
+/**
+ * Distance functions for vector similarity calculations
+ * Optimized for Node.js 23.11+ using enhanced array methods
+ */
+
+import { DistanceFunction, Vector } from '../coreTypes.js'
+
+/**
+ * Calculates the Euclidean distance between two vectors
+ * Lower values indicate higher similarity
+ * Optimized using array methods for Node.js 23.11+
+ */
+export const euclideanDistance: DistanceFunction = (a: Vector, b: Vector): number => {
+  if (a.length !== b.length) {
+    throw new Error('Vectors must have the same dimensions')
+  }
+
+  // Use array.reduce for better performance in Node.js 23.11+
+  const sum = a.reduce((acc, val, i) => {
+    const diff = val - b[i]
+    return acc + (diff * diff)
+  }, 0)
+
+  return Math.sqrt(sum)
+}
+
+/**
+ * Calculates the cosine distance between two vectors
+ * Lower values indicate higher similarity
+ * Range: 0 (identical) to 2 (opposite)
+ * Optimized using array methods for Node.js 23.11+
+ */
+export const cosineDistance: DistanceFunction = (a: Vector, b: Vector): number => {
+  if (a.length !== b.length) {
+    throw new Error('Vectors must have the same dimensions')
+  }
+
+  // Use array.reduce to calculate all values in a single pass
+  const { dotProduct, normA, normB } = a.reduce((acc, val, i) => {
+    return {
+      dotProduct: acc.dotProduct + (val * b[i]),
+      normA: acc.normA + (val * val),
+      normB: acc.normB + (b[i] * b[i])
+    };
+  }, { dotProduct: 0, normA: 0, normB: 0 });
+
+  if (normA === 0 || normB === 0) {
+    return 2 // Maximum distance for zero vectors
+  }
+
+  const similarity = dotProduct / (Math.sqrt(normA) * Math.sqrt(normB))
+  // Convert cosine similarity (-1 to 1) to distance (0 to 2)
+  return 1 - similarity
+}
+
+/**
+ * Calculates the Manhattan (L1) distance between two vectors
+ * Lower values indicate higher similarity
+ * Optimized using array methods for Node.js 23.11+
+ */
+export const manhattanDistance: DistanceFunction = (a: Vector, b: Vector): number => {
+  if (a.length !== b.length) {
+    throw new Error('Vectors must have the same dimensions')
+  }
+
+  // Use array.reduce for better performance in Node.js 23.11+
+  return a.reduce((sum, val, i) => sum + Math.abs(val - b[i]), 0)
+}
+
+/**
+ * Calculates the dot product similarity between two vectors
+ * Higher values indicate higher similarity
+ * Converted to a distance metric (lower is better)
+ * Optimized using array methods for Node.js 23.11+
+ */
+export const dotProductDistance: DistanceFunction = (a: Vector, b: Vector): number => {
+  if (a.length !== b.length) {
+    throw new Error('Vectors must have the same dimensions')
+  }
+
+  // Use array.reduce for better performance in Node.js 23.11+
+  const dotProduct = a.reduce((sum, val, i) => sum + (val * b[i]), 0)
+
+  // Convert to a distance metric (lower is better)
+  return -dotProduct
+}

src/utils/embedding.ts

@@ -0,0 +1,293 @@
+/**
+ * Embedding functions for converting data to vectors
+ */
+
+import { EmbeddingFunction, EmbeddingModel, Vector } from '../coreTypes.js'
+import { executeInThread } from './workerUtils.js'
+
+/**
+ * TensorFlow Universal Sentence Encoder embedding model
+ * This model provides high-quality text embeddings using TensorFlow.js
+ * The required TensorFlow.js dependencies are automatically installed with this package
+ */
+export class UniversalSentenceEncoder implements EmbeddingModel {
+  private model: any = null
+  private initialized = false
+  private tf: any = null
+  private use: any = null
+
+  /**
+   * Initialize the embedding model
+   */
+  public async init(): Promise<void> {
+    try {
+      // Save original console.warn
+      const originalWarn = console.warn
+
+      // Override console.warn to suppress TensorFlow.js Node.js backend message
+      console.warn = function (message?: any, ...optionalParams: any[]) {
+        if (
+          message &&
+          typeof message === 'string' &&
+          message.includes(
+            'Hi, looks like you are running TensorFlow.js in Node.js'
+          )
+        ) {
+          return // Suppress the specific warning
+        }
+        originalWarn(message, ...optionalParams)
+      }
+
+      // Define EPSILON flag before TensorFlow.js is loaded
+      // This prevents the "Cannot evaluate flag 'EPSILON': no evaluation function found" error
+      if (typeof window !== 'undefined') {
+        ;(window as any).EPSILON = 1e-7
+        // Define the flag with an evaluation function for TensorFlow.js
+        ;(window as any).ENV = (window as any).ENV || {}
+        ;(window as any).ENV.flagRegistry =
+          (window as any).ENV.flagRegistry || {}
+        ;(window as any).ENV.flagRegistry.EPSILON = {
+          evaluationFn: () => 1e-7
+        }
+      } else if (typeof global !== 'undefined') {
+        ;(global as any).EPSILON = 1e-7
+        // Define the flag with an evaluation function for TensorFlow.js
+        ;(global as any).ENV = (global as any).ENV || {}
+        ;(global as any).ENV.flagRegistry =
+          (global as any).ENV.flagRegistry || {}
+        ;(global as any).ENV.flagRegistry.EPSILON = {
+          evaluationFn: () => 1e-7
+        }
+      }
+
+      // Dynamically import TensorFlow.js and Universal Sentence Encoder
+      // Use type assertions to tell TypeScript these modules exist
+      this.tf = await import('@tensorflow/tfjs')
+      this.use = await import('@tensorflow-models/universal-sentence-encoder')
+
+      // Load the model
+      this.model = await this.use.load()
+      this.initialized = true
+
+      // Restore original console.warn
+      console.warn = originalWarn
+    } catch (error) {
+      console.error('Failed to initialize Universal Sentence Encoder:', error)
+      throw new Error(
+        `Failed to initialize Universal Sentence Encoder: ${error}`
+      )
+    }
+  }
+
+  /**
+   * Embed text into a vector using Universal Sentence Encoder
+   * @param data Text to embed
+   */
+  public async embed(data: string | string[]): Promise<Vector> {
+    if (!this.initialized) {
+      await this.init()
+    }
+
+    try {
+      // Handle different input types
+      let textToEmbed: string[]
+      if (typeof data === 'string') {
+        // Handle empty string case
+        if (data.trim() === '') {
+          // Return a zero vector of appropriate dimension (512 is the default for USE)
+          return new Array(512).fill(0)
+        }
+        textToEmbed = [data]
+      } else if (
+        Array.isArray(data) &&
+        data.every((item) => typeof item === 'string')
+      ) {
+        // Handle empty array or array with empty strings
+        if (data.length === 0 || data.every((item) => item.trim() === '')) {
+          return new Array(512).fill(0)
+        }
+        // Filter out empty strings
+        textToEmbed = data.filter((item) => item.trim() !== '')
+        if (textToEmbed.length === 0) {
+          return new Array(512).fill(0)
+        }
+      } else {
+        throw new Error(
+          'UniversalSentenceEncoder only supports string or string[] data'
+        )
+      }
+
+      // Get embeddings
+      const embeddings = await this.model.embed(textToEmbed)
+
+      // Convert to array and return the first embedding
+      const embeddingArray = await embeddings.array()
+      return embeddingArray[0]
+    } catch (error) {
+      console.error(
+        'Failed to embed text with Universal Sentence Encoder:',
+        error
+      )
+      throw new Error(
+        `Failed to embed text with Universal Sentence Encoder: ${error}`
+      )
+    }
+  }
+
+  /**
+   * Dispose of the model resources
+   */
+  public async dispose(): Promise<void> {
+    if (this.model && this.tf) {
+      try {
+        // Dispose of the model and tensors
+        this.model.dispose()
+        this.tf.disposeVariables()
+        this.initialized = false
+      } catch (error) {
+        console.error('Failed to dispose Universal Sentence Encoder:', error)
+      }
+    }
+    return Promise.resolve()
+  }
+}
+
+/**
+ * Create an embedding function from an embedding model
+ * @param model Embedding model to use
+ */
+export function createEmbeddingFunction(
+  model: EmbeddingModel
+): EmbeddingFunction {
+  return async (data: any): Promise<Vector> => {
+    return await model.embed(data)
+  }
+}
+
+/**
+ * Creates a TensorFlow-based Universal Sentence Encoder embedding function
+ * This is the required embedding function for all text embeddings
+ */
+export function createTensorFlowEmbeddingFunction(): EmbeddingFunction {
+  // Create a single shared instance of the model
+  const model = new UniversalSentenceEncoder()
+  let modelInitialized = false
+
+  return async (data: any): Promise<Vector> => {
+    try {
+      // Initialize the model if it hasn't been initialized yet
+      if (!modelInitialized) {
+        await model.init()
+        modelInitialized = true
+      }
+
+      return await model.embed(data)
+    } catch (error) {
+      console.error('Failed to use TensorFlow embedding:', error)
+      throw new Error(
+        `Universal Sentence Encoder is required but failed: ${error}`
+      )
+    }
+  }
+}
+
+/**
+ * Creates a TensorFlow-based Universal Sentence Encoder embedding function that runs in a separate thread
+ * This provides better performance for CPU-intensive embedding operations
+ * @param options Configuration options
+ * @returns An embedding function that runs in a separate thread
+ */
+export function createThreadedEmbeddingFunction(
+  options: { fallbackToMain?: boolean } = {}
+): EmbeddingFunction {
+  // Create a standard embedding function to use as fallback
+  const standardEmbedding = createTensorFlowEmbeddingFunction()
+
+  // Flag to track if we've fallen back to main thread
+  let useFallback = false
+
+  return async (data: any): Promise<Vector> => {
+    // If we've already determined that threading doesn't work, use the fallback
+    if (useFallback) {
+      return standardEmbedding(data)
+    }
+
+    try {
+      // Function to be executed in a worker thread
+      // This must be a regular function (not async) to avoid Promise cloning issues
+      const embedInWorker = (inputData: any) => {
+        // Return a plain object with the input data
+        // All async operations will be performed inside the worker
+        return { data: inputData }
+      }
+
+      // Worker implementation function that will be stringified and run in the worker
+      const workerImplementation = async ({ data }: { data: any }) => {
+        // We need to dynamically import TensorFlow.js and USE in the worker
+        const tf = await import('@tensorflow/tfjs')
+        const use = await import('@tensorflow-models/universal-sentence-encoder')
+
+        // Load the model
+        const model = await use.load()
+
+        // Handle different input types
+        let textToEmbed: string[]
+        if (typeof data === 'string') {
+          if (data.trim() === '') {
+            return new Array(512).fill(0)
+          }
+          textToEmbed = [data]
+        } else if (
+          Array.isArray(data) &&
+          data.every((item) => typeof item === 'string')
+        ) {
+          if (data.length === 0 || data.every((item) => item.trim() === '')) {
+            return new Array(512).fill(0)
+          }
+          textToEmbed = data.filter((item) => item.trim() !== '')
+          if (textToEmbed.length === 0) {
+            return new Array(512).fill(0)
+          }
+        } else {
+          throw new Error(
+            'UniversalSentenceEncoder only supports string or string[] data'
+          )
+        }
+
+        // Get embeddings
+        const embeddings = await model.embed(textToEmbed)
+
+        // Convert to array and return the first embedding
+        const embeddingArray = await embeddings.array()
+
+        // Dispose of the tensor to free memory
+        embeddings.dispose()
+
+        return embeddingArray[0]
+      }
+
+      // Execute the embedding function in a separate thread
+      // Pass the worker implementation as a string to avoid Promise cloning issues
+      return await executeInThread<Vector>(workerImplementation.toString(), embedInWorker(data))
+    } catch (error) {
+      // If threading fails and fallback is enabled, use the standard embedding function
+      if (options.fallbackToMain) {
+        console.warn('Threaded embedding failed, falling back to main thread:', error)
+        useFallback = true
+        return standardEmbedding(data)
+      }
+
+      // Otherwise, propagate the error
+      throw new Error(`Threaded embedding failed: ${error}`)
+    }
+  }
+}
+
+/**
+ * Default embedding function
+ * Uses UniversalSentenceEncoder for all text embeddings
+ * TensorFlow.js is required for this to work
+ * Uses threading when available for better performance
+ */
+export const defaultEmbeddingFunction: EmbeddingFunction =
+  createThreadedEmbeddingFunction({ fallbackToMain: true })

src/utils/environment.ts

@@ -0,0 +1,57 @@
+/**
+ * Utility functions for environment detection
+ */
+
+/**
+ * Check if code is running in a browser environment
+ */
+export function isBrowser(): boolean {
+  return typeof window !== 'undefined' && typeof document !== 'undefined';
+}
+
+/**
+ * Check if code is running in a Node.js environment
+ */
+export function isNode(): boolean {
+  return typeof process !== 'undefined' && 
+    process.versions != null && 
+    process.versions.node != null;
+}
+
+/**
+ * Check if code is running in a Web Worker environment
+ */
+export function isWebWorker(): boolean {
+  return typeof self === 'object' && 
+    self.constructor && 
+    self.constructor.name === 'DedicatedWorkerGlobalScope';
+}
+
+/**
+ * Check if Web Workers are available in the current environment
+ */
+export function areWebWorkersAvailable(): boolean {
+  return isBrowser() && typeof Worker !== 'undefined';
+}
+
+/**
+ * Check if Worker Threads are available in the current environment (Node.js)
+ */
+export function areWorkerThreadsAvailable(): boolean {
+  if (!isNode()) return false;
+  
+  try {
+    // Dynamic import to avoid errors in browser environments
+    require('worker_threads');
+    return true;
+  } catch (e) {
+    return false;
+  }
+}
+
+/**
+ * Determine if threading is available in the current environment
+ */
+export function isThreadingAvailable(): boolean {
+  return areWebWorkersAvailable() || areWorkerThreadsAvailable();
+}

src/utils/index.ts

@@ -0,0 +1,2 @@
+export * from './distance.js'
+export * from './embedding.js'

src/utils/version.ts

@@ -0,0 +1,6 @@
+/**
+ * This file is auto-generated during the build process.
+ * Do not modify this file directly.
+ */
+
+export const VERSION = '0.9.10';

src/utils/workerUtils.ts

@@ -0,0 +1,145 @@
+/**
+ * Utility functions for working with Web Workers and Worker Threads
+ */
+
+import { isNode, isBrowser } from './environment.js'
+
+/**
+ * Execute a function in a Web Worker (browser environment)
+ *
+ * @param fnString The function to execute as a string
+ * @param args The arguments to pass to the function
+ * @returns A promise that resolves with the result of the function
+ */
+export function executeInWebWorker<T>(fnString: string, args: any): Promise<T> {
+  return new Promise((resolve, reject) => {
+    // Create a blob URL for the worker script
+    const workerScript = `
+      self.onmessage = async function(e) {
+        try {
+          const fn = ${fnString};
+          const result = await fn(e.data);
+          self.postMessage({ success: true, data: result });
+        } catch (error) {
+          self.postMessage({ 
+            success: false, 
+            error: error instanceof Error ? error.message : String(error) 
+          });
+        }
+      };
+    `
+
+    const blob = new Blob([workerScript], { type: 'application/javascript' })
+    const blobURL = URL.createObjectURL(blob)
+
+    // Create a worker
+    const worker = new Worker(blobURL)
+
+    // Set up message handling
+    worker.onmessage = function (
+      e: MessageEvent<{ success: boolean; data: T; error?: string }>
+    ) {
+      URL.revokeObjectURL(blobURL) // Clean up
+      worker.terminate() // Terminate the worker
+
+      if (e.data.success) {
+        resolve(e.data.data)
+      } else {
+        reject(new Error(e.data.error))
+      }
+    }
+
+    worker.onerror = function (error: ErrorEvent) {
+      URL.revokeObjectURL(blobURL) // Clean up
+      worker.terminate() // Terminate the worker
+      reject(error)
+    }
+
+    // Start the worker
+    worker.postMessage(args)
+  })
+}
+
+/**
+ * Execute a function in a Worker Thread (Node.js environment)
+ *
+ * @param fnString The function to execute as a string
+ * @param args The arguments to pass to the function
+ * @returns A promise that resolves with the result of the function
+ */
+export function executeInWorkerThread<T>(
+  fnString: string,
+  args: any
+): Promise<T> {
+  return new Promise((resolve, reject) => {
+    try {
+      // Dynamic import to avoid errors in browser environments
+      const { Worker } = require('worker_threads')
+
+      // Create a worker script
+      const workerScript = `
+        const { parentPort } = require('worker_threads');
+
+        parentPort.once('message', async (data) => {
+          try {
+            const fn = ${fnString};
+            const result = await fn(data);
+            parentPort.postMessage({ success: true, data: result });
+          } catch (error) {
+            parentPort.postMessage({ 
+              success: false, 
+              error: error instanceof Error ? error.message : String(error) 
+            });
+          }
+        });
+      `
+
+      // Create a worker
+      const worker = new Worker(workerScript, { eval: true })
+
+      // Set up message handling
+      worker.on(
+        'message',
+        (data: { success: boolean; data: T; error?: string }) => {
+          worker.terminate() // Terminate the worker
+
+          if (data.success) {
+            resolve(data.data)
+          } else {
+            reject(new Error(data.error))
+          }
+        }
+      )
+
+      worker.on('error', (error: Error) => {
+        worker.terminate() // Terminate the worker
+        reject(error)
+      })
+
+      // Start the worker
+      worker.postMessage(args)
+    } catch (error) {
+      reject(error)
+    }
+  })
+}
+
+/**
+ * Execute a function in a separate thread based on the environment
+ *
+ * @param fnString The function to execute as a string
+ * @param args The arguments to pass to the function
+ * @returns A promise that resolves with the result of the function
+ */
+export function executeInThread<T>(fnString: string, args: any): Promise<T> {
+  if (isBrowser()) {
+    return executeInWebWorker<T>(fnString, args)
+  } else if (isNode()) {
+    return executeInWorkerThread<T>(fnString, args)
+  } else {
+    // Fall back to executing in the main thread
+    // Parse the function from string and execute it
+    const fn = new Function('return ' + fnString)()
+    return Promise.resolve(fn(args) as T)
+  }
+}