perf: pre-compute type embeddings at build time (zero runtime cost)

Major optimization - all type embeddings now built into package: Build-time generation: - Created scripts/buildTypeEmbeddings.ts to generate all type embeddings - Generates embeddings for 31 NounTypes + 40 VerbTypes at build time - Stores as base64-encoded binary data in embeddedTypeEmbeddings.ts - Added check script to rebuild only when needed Updated all consumers: - NeuralEntityExtractor: loads pre-computed embeddings (instant) - BrainyTypes: loads pre-computed embeddings (instant init) - NaturalLanguageProcessor: loads pre-computed embeddings (instant init) Build process: - Added npm run build:types to generate embeddings - Added npm run build:types:if-needed for conditional rebuild - Integrated into main build pipeline - Auto-rebuilds only when types or build script change Benefits: - Zero runtime cost - embeddings loaded instantly - Survives all container restarts - All 71 types always available (31 nouns + 40 verbs) - ~100KB memory overhead for permanent performance gain - Eliminates 5-10 second initialization delay This completes the type embedding optimization started in v3.32.5
2025-10-09 18:08:57 -07:00 · 2025-10-09 18:08:57 -07:00 · 0d649b8a79
commit 0d649b8a79
parent 87eb60d527
9 changed files with 643 additions and 111 deletions
--- a/scripts/buildTypeEmbeddings.ts
+++ b/scripts/buildTypeEmbeddings.ts
@ -0,0 +1,361 @@
+#!/usr/bin/env node
+
+/**
+ * Build embedded type embeddings with pre-computed vectors
+ * Generates embeddings for all 31 NounTypes + 40 VerbTypes
+ * NO runtime computation, NO external files needed!
+ */
+
+import { TransformerEmbedding } from '../src/utils/embedding.js'
+import * as fs from 'fs/promises'
+import * as path from 'path'
+import { fileURLToPath } from 'url'
+import { NounType, VerbType } from '../src/types/graphTypes.js'
+
+const __dirname = path.dirname(fileURLToPath(import.meta.url))
+
+/**
+ * Type descriptions for semantic matching
+ * Copied from BrainyTypes for consistency
+ */
+const NOUN_TYPE_DESCRIPTIONS: Record<string, string> = {
+  // Core Entity Types
+  [NounType.Person]: 'person human individual user employee customer citizen member author creator agent actor participant',
+  [NounType.Organization]: 'organization company business corporation institution agency department team group committee board',
+  [NounType.Location]: 'location place address city country region area zone coordinate position site venue building',
+  [NounType.Thing]: 'thing object item product device equipment tool instrument asset artifact material physical tangible',
+  [NounType.Concept]: 'concept idea theory principle philosophy belief value abstract intangible notion thought',
+  [NounType.Event]: 'event occurrence incident activity happening meeting conference celebration milestone timestamp date',
+
+  // Digital/Content Types
+  [NounType.Document]: 'document file report article paper text pdf word contract agreement record documentation',
+  [NounType.Media]: 'media image photo video audio music podcast multimedia graphic visualization animation',
+  [NounType.File]: 'file digital data binary code script program software archive package bundle',
+  [NounType.Message]: 'message email chat communication notification alert announcement broadcast transmission',
+  [NounType.Content]: 'content information data text material resource publication post blog webpage',
+
+  // Collection Types
+  [NounType.Collection]: 'collection group set list array category folder directory catalog inventory database',
+  [NounType.Dataset]: 'dataset data table spreadsheet database records statistics metrics measurements analysis',
+
+  // Business/Application Types
+  [NounType.Product]: 'product item merchandise offering service feature application software solution package',
+  [NounType.Service]: 'service offering subscription support maintenance utility function capability',
+  [NounType.User]: 'user account profile member subscriber customer client participant identity credentials',
+  [NounType.Task]: 'task action todo item job assignment duty responsibility activity step procedure',
+  [NounType.Project]: 'project initiative program campaign effort endeavor plan scheme venture undertaking',
+
+  // Descriptive Types
+  [NounType.Process]: 'process workflow procedure method algorithm sequence pipeline operation routine protocol',
+  [NounType.State]: 'state status condition phase stage mode situation circumstance configuration setting',
+  [NounType.Role]: 'role position title function responsibility duty job capacity designation authority',
+  [NounType.Topic]: 'subject topic theme category tag keyword area domain field discipline specialty',
+  [NounType.Language]: 'language dialect locale tongue vernacular communication speech linguistics vocabulary',
+  [NounType.Currency]: 'currency money dollar euro pound yen bitcoin payment financial monetary unit',
+  [NounType.Measurement]: 'measurement metric quantity value amount size dimension weight height volume distance',
+
+  // Scientific/Research Types
+  [NounType.Hypothesis]: 'hypothesis theory proposition thesis assumption premise conjecture speculation prediction',
+  [NounType.Experiment]: 'experiment test trial study research investigation analysis observation examination',
+
+  // Legal/Regulatory Types
+  [NounType.Contract]: 'contract agreement deal treaty pact covenant license terms conditions policy',
+  [NounType.Regulation]: 'regulation law rule policy standard compliance requirement guideline ordinance statute',
+
+  // Technical Infrastructure Types
+  [NounType.Interface]: 'interface API endpoint protocol specification contract schema definition connection',
+  [NounType.Resource]: 'resource infrastructure server database storage compute memory bandwidth capacity asset'
+}
+
+const VERB_TYPE_DESCRIPTIONS: Record<string, string> = {
+  // Core Relationship Types
+  [VerbType.RelatedTo]: 'related connected associated linked correlated relevant pertinent applicable',
+  [VerbType.Contains]: 'contains includes holds stores encompasses comprises consists incorporates',
+  [VerbType.PartOf]: 'part component element member piece portion section segment constituent',
+  [VerbType.LocatedAt]: 'located situated positioned placed found exists resides occupies',
+  [VerbType.References]: 'references cites mentions points links refers quotes sources',
+
+  // Temporal/Causal Types
+  [VerbType.Precedes]: 'precedes before earlier prior previous antecedent preliminary foregoing',
+  [VerbType.Succeeds]: 'succeeds follows after later subsequent next ensuing succeeding',
+  [VerbType.Causes]: 'causes triggers induces produces generates results influences affects',
+  [VerbType.DependsOn]: 'depends requires needs relies necessitates contingent prerequisite',
+  [VerbType.Requires]: 'requires needs demands necessitates mandates obliges compels entails',
+
+  // Creation/Transformation Types
+  [VerbType.Creates]: 'creates makes produces generates builds constructs forms establishes',
+  [VerbType.Transforms]: 'transforms converts changes modifies alters transitions morphs evolves',
+  [VerbType.Becomes]: 'becomes turns evolves transforms changes transitions develops grows',
+  [VerbType.Modifies]: 'modifies changes updates alters edits revises adjusts adapts',
+  [VerbType.Consumes]: 'consumes uses utilizes depletes expends absorbs takes processes',
+
+  // Ownership/Attribution Types
+  [VerbType.Owns]: 'owns possesses holds controls manages administers governs maintains',
+  [VerbType.AttributedTo]: 'attributed credited assigned ascribed authored written composed',
+  [VerbType.CreatedBy]: 'created made produced generated built developed authored written',
+  [VerbType.BelongsTo]: 'belongs property possession part member affiliate associated owned',
+
+  // Social/Organizational Types
+  [VerbType.MemberOf]: 'member participant affiliate associate belongs joined enrolled registered',
+  [VerbType.WorksWith]: 'works collaborates cooperates partners teams assists helps supports',
+  [VerbType.FriendOf]: 'friend companion buddy pal acquaintance associate connection relationship',
+  [VerbType.Follows]: 'follows subscribes tracks monitors watches observes trails pursues',
+  [VerbType.Likes]: 'likes enjoys appreciates favors prefers admires values endorses',
+  [VerbType.ReportsTo]: 'reports answers subordinate accountable responsible supervised managed',
+  [VerbType.Supervises]: 'supervises manages oversees directs leads controls guides administers',
+  [VerbType.Mentors]: 'mentors teaches guides coaches instructs trains advises counsels',
+  [VerbType.Communicates]: 'communicates talks speaks messages contacts interacts corresponds exchanges',
+
+  // Descriptive/Functional Types
+  [VerbType.Describes]: 'describes explains details documents specifies outlines depicts characterizes',
+  [VerbType.Defines]: 'defines specifies establishes determines sets declares identifies designates',
+  [VerbType.Categorizes]: 'categorizes classifies groups sorts organizes arranges labels tags',
+  [VerbType.Measures]: 'measures quantifies gauges assesses evaluates calculates determines counts',
+  [VerbType.Evaluates]: 'evaluates assesses analyzes reviews examines appraises judges rates',
+  [VerbType.Uses]: 'uses utilizes employs applies operates handles manipulates exploits',
+  [VerbType.Implements]: 'implements executes realizes performs accomplishes carries delivers completes',
+  [VerbType.Extends]: 'extends expands enhances augments amplifies broadens enlarges develops',
+
+  // Enhanced Relationships
+  [VerbType.Inherits]: 'inherits derives extends receives obtains acquires succeeds legacy',
+  [VerbType.Conflicts]: 'conflicts contradicts opposes clashes disputes disagrees incompatible inconsistent',
+  [VerbType.Synchronizes]: 'synchronizes coordinates aligns harmonizes matches corresponds parallels coincides',
+  [VerbType.Competes]: 'competes rivals contends contests challenges opposes vies struggles'
+}
+
+async function buildTypeEmbeddings() {
+  console.log('🧠 Building embedded type embeddings for Brainy...')
+
+  // Count types
+  const nounTypes = Object.keys(NOUN_TYPE_DESCRIPTIONS)
+  const verbTypes = Object.keys(VERB_TYPE_DESCRIPTIONS)
+  console.log(`📊 Processing ${nounTypes.length} noun types and ${verbTypes.length} verb types...`)
+
+  // Initialize TransformerEmbedding for embedding (one-time only!)
+  const embedder = new TransformerEmbedding({
+    verbose: true,
+    localFilesOnly: false // Allow downloading models during build
+  })
+
+  await embedder.init()
+  console.log('✅ TransformerEmbedding initialized')
+
+  // Generate noun type embeddings
+  const nounEmbeddings = new Map<string, number[]>()
+  console.log('📝 Generating noun type embeddings...')
+
+  for (const [type, description] of Object.entries(NOUN_TYPE_DESCRIPTIONS)) {
+    try {
+      const embedding = await embedder.embed(description)
+      if (embedding && Array.isArray(embedding)) {
+        nounEmbeddings.set(type, embedding)
+        console.log(`  ✓ ${type}`)
+      }
+    } catch (error) {
+      console.warn(`  ⚠️  Failed to embed noun type: ${type}`)
+    }
+  }
+
+  // Generate verb type embeddings
+  const verbEmbeddings = new Map<string, number[]>()
+  console.log('📝 Generating verb type embeddings...')
+
+  for (const [type, description] of Object.entries(VERB_TYPE_DESCRIPTIONS)) {
+    try {
+      const embedding = await embedder.embed(description)
+      if (embedding && Array.isArray(embedding)) {
+        verbEmbeddings.set(type, embedding)
+        console.log(`  ✓ ${type}`)
+      }
+    } catch (error) {
+      console.warn(`  ⚠️  Failed to embed verb type: ${type}`)
+    }
+  }
+
+  console.log(`✅ Generated ${nounEmbeddings.size} noun embeddings and ${verbEmbeddings.size} verb embeddings`)
+
+  // Get embedding dimension
+  const embeddingDim = nounEmbeddings.size > 0 ?
+    Array.from(nounEmbeddings.values())[0]?.length ?? 384 :
+    384
+
+  // Convert to compact binary format
+  const totalTypes = nounTypes.length + verbTypes.length
+  const totalFloats = totalTypes * embeddingDim
+  const buffer = new ArrayBuffer(totalFloats * 4)
+  const view = new DataView(buffer)
+
+  let offset = 0
+
+  // Pack noun embeddings
+  for (const type of nounTypes) {
+    const embedding = nounEmbeddings.get(type) || new Array(embeddingDim).fill(0)
+    for (let i = 0; i < embeddingDim; i++) {
+      view.setFloat32(offset, embedding[i], true) // little-endian
+      offset += 4
+    }
+  }
+
+  // Pack verb embeddings
+  for (const type of verbTypes) {
+    const embedding = verbEmbeddings.get(type) || new Array(embeddingDim).fill(0)
+    for (let i = 0; i < embeddingDim; i++) {
+      view.setFloat32(offset, embedding[i], true) // little-endian
+      offset += 4
+    }
+  }
+
+  // Convert to base64
+  const uint8 = new Uint8Array(buffer)
+  const base64 = Buffer.from(uint8).toString('base64')
+
+  // Generate TypeScript file
+  const tsContent = `/**
+ * 🧠 BRAINY EMBEDDED TYPE EMBEDDINGS
+ *
+ * AUTO-GENERATED - DO NOT EDIT
+ * Generated: ${new Date().toISOString()}
+ * Noun Types: ${nounTypes.length}
+ * Verb Types: ${verbTypes.length}
+ *
+ * This file contains pre-computed embeddings for all NounTypes and VerbTypes.
+ * No runtime computation needed, instant availability!
+ */
+
+import { NounType, VerbType } from '../types/graphTypes.js'
+import { Vector } from '../coreTypes.js'
+
+// Type metadata
+export const TYPE_METADATA = {
+  nounTypes: ${nounTypes.length},
+  verbTypes: ${verbTypes.length},
+  totalTypes: ${totalTypes},
+  embeddingDimensions: ${embeddingDim},
+  generatedAt: "${new Date().toISOString()}",
+  sizeBytes: {
+    embeddings: ${buffer.byteLength},
+    base64: ${base64.length}
+  }
+}
+
+// All noun types in order
+const NOUN_TYPE_ORDER: NounType[] = ${JSON.stringify(nounTypes)}
+
+// All verb types in order
+const VERB_TYPE_ORDER: VerbType[] = ${JSON.stringify(verbTypes)}
+
+// Pre-computed embeddings (${(base64.length / 1024).toFixed(1)}KB base64)
+const EMBEDDINGS_BASE64 = "${base64}"
+
+// Decode embeddings at startup (happens once, <10ms)
+function decodeEmbeddings(): Uint8Array {
+  if (typeof Buffer !== 'undefined') {
+    // Node.js environment
+    return Buffer.from(EMBEDDINGS_BASE64, 'base64')
+  } else if (typeof atob !== 'undefined') {
+    // Browser environment
+    const binaryString = atob(EMBEDDINGS_BASE64)
+    const bytes = new Uint8Array(binaryString.length)
+    for (let i = 0; i < binaryString.length; i++) {
+      bytes[i] = binaryString.charCodeAt(i)
+    }
+    return bytes
+  }
+  return new Uint8Array(0)
+}
+
+// Cached decoded embeddings
+let decodedEmbeddings: Uint8Array | null = null
+
+/**
+ * Get noun type embeddings as a Map for fast lookup
+ * This is called once and cached
+ */
+export function getNounTypeEmbeddings(): Map<NounType, Vector> {
+  if (!decodedEmbeddings) {
+    decodedEmbeddings = decodeEmbeddings()
+  }
+
+  const embeddings = new Map<NounType, Vector>()
+  const view = new DataView(decodedEmbeddings.buffer)
+  const embeddingSize = ${embeddingDim}
+
+  NOUN_TYPE_ORDER.forEach((type, index) => {
+    const offset = index * embeddingSize * 4
+    const embedding = new Float32Array(embeddingSize)
+
+    for (let i = 0; i < embeddingSize; i++) {
+      embedding[i] = view.getFloat32(offset + i * 4, true)
+    }
+
+    embeddings.set(type, Array.from(embedding))
+  })
+
+  return embeddings
+}
+
+/**
+ * Get verb type embeddings as a Map for fast lookup
+ * This is called once and cached
+ */
+export function getVerbTypeEmbeddings(): Map<VerbType, Vector> {
+  if (!decodedEmbeddings) {
+    decodedEmbeddings = decodeEmbeddings()
+  }
+
+  const embeddings = new Map<VerbType, Vector>()
+  const view = new DataView(decodedEmbeddings.buffer)
+  const embeddingSize = ${embeddingDim}
+
+  // Verb embeddings start after noun embeddings
+  const verbStartOffset = ${nounTypes.length} * embeddingSize * 4
+
+  VERB_TYPE_ORDER.forEach((type, index) => {
+    const offset = verbStartOffset + index * embeddingSize * 4
+    const embedding = new Float32Array(embeddingSize)
+
+    for (let i = 0; i < embeddingSize; i++) {
+      embedding[i] = view.getFloat32(offset + i * 4, true)
+    }
+
+    embeddings.set(type, Array.from(embedding))
+  })
+
+  return embeddings
+}
+
+// Import logging
+import { prodLog } from '../utils/logger.js'
+prodLog.info(\`🧠 Brainy Type Embeddings loaded: \${TYPE_METADATA.nounTypes} nouns, \${TYPE_METADATA.verbTypes} verbs, \${(TYPE_METADATA.sizeBytes.embeddings / 1024).toFixed(1)}KB\`)
+`
+
+  // Write the TypeScript file
+  const outputPath = path.join(__dirname, '..', 'src', 'neural', 'embeddedTypeEmbeddings.ts')
+  await fs.writeFile(outputPath, tsContent)
+
+  // Report statistics
+  console.log(`
+✅ EMBEDDED TYPE EMBEDDINGS BUILT SUCCESSFULLY!
+================================================
+Noun Types: ${nounTypes.length}
+Verb Types: ${verbTypes.length}
+Total Types: ${totalTypes}
+Embedding Dimensions: ${embeddingDim}
+
+File sizes:
+  Embeddings binary: ${(buffer.byteLength / 1024).toFixed(1)} KB
+  Base64 encoded: ${(base64.length / 1024).toFixed(1)} KB
+
+Output: ${outputPath}
+
+Type embeddings are now embedded directly in Brainy!
+No runtime computation needed, instant availability.
+`)
+}
+
+// Run if called directly
+if (import.meta.url === `file://${process.argv[1]}`) {
+  buildTypeEmbeddings().catch(console.error)
+}
+
+export { buildTypeEmbeddings }
--- a/scripts/check-type-embeddings.cjs
+++ b/scripts/check-type-embeddings.cjs
@ -0,0 +1,43 @@
+#!/usr/bin/env node
+
+/**
+ * Check if type embeddings need rebuilding
+ * Only rebuild if:
+ * 1. embeddedTypeEmbeddings.ts doesn't exist
+ * 2. Type definitions have changed
+ * 3. Build script has changed
+ */
+
+const fs = require('fs');
+const path = require('path');
+
+const EMBEDDED_FILE = path.join(__dirname, '../src/neural/embeddedTypeEmbeddings.ts');
+const BUILD_SCRIPT = path.join(__dirname, 'buildTypeEmbeddings.ts');
+const GRAPH_TYPES = path.join(__dirname, '../src/types/graphTypes.ts');
+
+// Check if embedded type embeddings exist
+if (!fs.existsSync(EMBEDDED_FILE)) {
+  console.log('❌ Embedded type embeddings not found. Building...');
+  process.exit(1); // Signal need to rebuild
+}
+
+// Check if build script is newer than embedded embeddings
+const embeddedStats = fs.statSync(EMBEDDED_FILE);
+const buildScriptStats = fs.statSync(BUILD_SCRIPT);
+
+if (buildScriptStats.mtime > embeddedStats.mtime) {
+  console.log('🔄 Build script has changed. Rebuilding type embeddings...');
+  process.exit(1); // Signal need to rebuild
+}
+
+// Check if type definitions are newer than embedded embeddings
+if (fs.existsSync(GRAPH_TYPES)) {
+  const graphTypesStats = fs.statSync(GRAPH_TYPES);
+  if (graphTypesStats.mtime > embeddedStats.mtime) {
+    console.log('🔄 Type definitions have changed. Rebuilding type embeddings...');
+    process.exit(1); // Signal need to rebuild
+  }
+}
+
+console.log('✅ Embedded type embeddings are up-to-date. Skipping rebuild.');
+process.exit(0); // No rebuild needed