Initial commit of Brainy vector database v0.1.0

src/utils/distance.ts

@@ -0,0 +1,88 @@
+/**
+ * Distance functions for vector similarity calculations
+ */
+
+import { DistanceFunction, Vector } from '../coreTypes.js'
+
+/**
+ * Calculates the Euclidean distance between two vectors
+ * Lower values indicate higher similarity
+ */
+export const euclideanDistance: DistanceFunction = (a: Vector, b: Vector): number => {
+  if (a.length !== b.length) {
+    throw new Error('Vectors must have the same dimensions')
+  }
+
+  let sum = 0
+  for (let i = 0; i < a.length; i++) {
+    const diff = a[i] - b[i]
+    sum += diff * diff
+  }
+
+  return Math.sqrt(sum)
+}
+
+/**
+ * Calculates the cosine distance between two vectors
+ * Lower values indicate higher similarity
+ * Range: 0 (identical) to 2 (opposite)
+ */
+export const cosineDistance: DistanceFunction = (a: Vector, b: Vector): number => {
+  if (a.length !== b.length) {
+    throw new Error('Vectors must have the same dimensions')
+  }
+
+  let dotProduct = 0
+  let normA = 0
+  let normB = 0
+
+  for (let i = 0; i < a.length; i++) {
+    dotProduct += a[i] * b[i]
+    normA += a[i] * a[i]
+    normB += b[i] * b[i]
+  }
+
+  if (normA === 0 || normB === 0) {
+    return 2 // Maximum distance for zero vectors
+  }
+
+  const similarity = dotProduct / (Math.sqrt(normA) * Math.sqrt(normB))
+  // Convert cosine similarity (-1 to 1) to distance (0 to 2)
+  return 1 - similarity
+}
+
+/**
+ * Calculates the Manhattan (L1) distance between two vectors
+ * Lower values indicate higher similarity
+ */
+export const manhattanDistance: DistanceFunction = (a: Vector, b: Vector): number => {
+  if (a.length !== b.length) {
+    throw new Error('Vectors must have the same dimensions')
+  }
+
+  let sum = 0
+  for (let i = 0; i < a.length; i++) {
+    sum += Math.abs(a[i] - b[i])
+  }
+
+  return sum
+}
+
+/**
+ * Calculates the dot product similarity between two vectors
+ * Higher values indicate higher similarity
+ * Converted to a distance metric (lower is better)
+ */
+export const dotProductDistance: DistanceFunction = (a: Vector, b: Vector): number => {
+  if (a.length !== b.length) {
+    throw new Error('Vectors must have the same dimensions')
+  }
+
+  let dotProduct = 0
+  for (let i = 0; i < a.length; i++) {
+    dotProduct += a[i] * b[i]
+  }
+
+  // Convert to a distance metric (lower is better)
+  return -dotProduct
+}

src/utils/embedding.ts

@@ -0,0 +1,168 @@
+/**
+ * Embedding functions for converting data to vectors
+ */
+
+import { EmbeddingFunction, EmbeddingModel, Vector } from '../coreTypes.js'
+
+/**
+ * Simple character-based embedding function
+ * This is a very basic implementation for demo purposes
+ */
+export class SimpleEmbedding implements EmbeddingModel {
+  private initialized = false
+
+  /**
+   * Initialize the embedding model
+   */
+  public async init(): Promise<void> {
+    this.initialized = true
+    return Promise.resolve()
+  }
+
+  /**
+   * Embed text into a vector using character frequencies
+   * @param data Text to embed
+   */
+  public async embed(data: string): Promise<Vector> {
+    if (!this.initialized) {
+      await this.init()
+    }
+
+    // Only handle string data
+    if (typeof data !== 'string') {
+      throw new Error('SimpleEmbedding only supports string data')
+    }
+
+    // Normalize the text
+    const normalizedText = data.toLowerCase().trim()
+
+    // Create a simple 4-dimensional vector based on character frequencies
+    const vector: Vector = [0, 0, 0, 0]
+
+    // Count vowels, consonants, numbers, and special characters
+    for (let i = 0; i < normalizedText.length; i++) {
+      const char = normalizedText[i]
+      if ('aeiou'.includes(char)) {
+        vector[0] += 0.1 // Vowels affect first dimension
+      } else if ('bcdfghjklmnpqrstvwxyz'.includes(char)) {
+        vector[1] += 0.1 // Consonants affect second dimension
+      } else if ('0123456789'.includes(char)) {
+        vector[2] += 0.1 // Numbers affect third dimension
+      } else {
+        vector[3] += 0.1 // Special chars affect fourth dimension
+      }
+    }
+
+    // Normalize the vector
+    const magnitude = Math.sqrt(
+      vector.reduce((sum, val) => sum + val * val, 0)
+    )
+    if (magnitude > 0) {
+      return vector.map((val) => val / magnitude)
+    }
+
+    return vector
+  }
+
+  /**
+   * Dispose of the model resources
+   */
+  public async dispose(): Promise<void> {
+    this.initialized = false
+    return Promise.resolve()
+  }
+}
+
+/**
+ * TensorFlow Universal Sentence Encoder embedding model
+ * Requires @tensorflow/tfjs and @tensorflow-models/universal-sentence-encoder to be installed
+ */
+export class UniversalSentenceEncoder implements EmbeddingModel {
+  private model: any = null
+  private initialized = false
+  private tf: any = null
+  private use: any = null
+
+  /**
+   * Initialize the embedding model
+   */
+  public async init(): Promise<void> {
+    try {
+      // Dynamically import TensorFlow.js and Universal Sentence Encoder
+      // Use type assertions to tell TypeScript these modules exist
+      this.tf = await import('@tensorflow/tfjs/dist/tf.esm.js' as any)
+      this.use = await import('@tensorflow-models/universal-sentence-encoder/dist/universal-sentence-encoder.esm.js' as any)
+
+      // Load the model
+      this.model = await this.use.load()
+      this.initialized = true
+    } catch (error) {
+      console.error('Failed to initialize Universal Sentence Encoder:', error)
+      throw new Error(`Failed to initialize Universal Sentence Encoder: ${error}`)
+    }
+  }
+
+  /**
+   * Embed text into a vector using Universal Sentence Encoder
+   * @param data Text to embed
+   */
+  public async embed(data: string | string[]): Promise<Vector> {
+    if (!this.initialized) {
+      await this.init()
+    }
+
+    try {
+      // Handle different input types
+      let textToEmbed: string[]
+      if (typeof data === 'string') {
+        textToEmbed = [data]
+      } else if (Array.isArray(data) && data.every(item => typeof item === 'string')) {
+        textToEmbed = data
+      } else {
+        throw new Error('UniversalSentenceEncoder only supports string or string[] data')
+      }
+
+      // Get embeddings
+      const embeddings = await this.model.embed(textToEmbed)
+
+      // Convert to array and return the first embedding
+      const embeddingArray = await embeddings.array()
+      return embeddingArray[0]
+    } catch (error) {
+      console.error('Failed to embed text with Universal Sentence Encoder:', error)
+      throw new Error(`Failed to embed text with Universal Sentence Encoder: ${error}`)
+    }
+  }
+
+  /**
+   * Dispose of the model resources
+   */
+  public async dispose(): Promise<void> {
+    if (this.model && this.tf) {
+      try {
+        // Dispose of the model and tensors
+        this.model.dispose()
+        this.tf.disposeVariables()
+        this.initialized = false
+      } catch (error) {
+        console.error('Failed to dispose Universal Sentence Encoder:', error)
+      }
+    }
+    return Promise.resolve()
+  }
+}
+
+/**
+ * Create an embedding function from an embedding model
+ * @param model Embedding model to use
+ */
+export function createEmbeddingFunction(model: EmbeddingModel): EmbeddingFunction {
+  return async (data: any): Promise<Vector> => {
+    return await model.embed(data)
+  }
+}
+
+/**
+ * Default embedding function using UniversalSentenceEncoder
+ */
+export const defaultEmbeddingFunction: EmbeddingFunction = createEmbeddingFunction(new UniversalSentenceEncoder())

src/utils/index.ts

@@ -0,0 +1,2 @@
+export * from './distance.js'
+export * from './embedding.js'