#!/usr/bin/env node /** * Build embedded type embeddings with pre-computed vectors * Generates embeddings for all 31 NounTypes + 40 VerbTypes * NO runtime computation, NO external files needed! */ import { TransformerEmbedding } from '../src/utils/embedding.js' import * as fs from 'fs/promises' import * as path from 'path' import { fileURLToPath } from 'url' import { NounType, VerbType } from '../src/types/graphTypes.js' const __dirname = path.dirname(fileURLToPath(import.meta.url)) /** * Type descriptions for semantic matching * Copied from BrainyTypes for consistency */ const NOUN_TYPE_DESCRIPTIONS: Record = { // Core Entity Types [NounType.Person]: 'person human individual user employee customer citizen member author creator agent actor participant', [NounType.Organization]: 'organization company business corporation institution agency department team group committee board', [NounType.Location]: 'location place address city country region area zone coordinate position site venue building', [NounType.Thing]: 'thing object item product device equipment tool instrument asset artifact material physical tangible', [NounType.Concept]: 'concept idea theory principle philosophy belief value abstract intangible notion thought', [NounType.Event]: 'event occurrence incident activity happening meeting conference celebration milestone timestamp date', // Digital/Content Types [NounType.Document]: 'document file report article paper text pdf word contract agreement record documentation', [NounType.Media]: 'media image photo video audio music podcast multimedia graphic visualization animation', [NounType.File]: 'file digital data binary code script program software archive package bundle', [NounType.Message]: 'message email chat communication notification alert announcement broadcast transmission', [NounType.Content]: 'content information data text material resource publication post blog webpage', // Collection Types [NounType.Collection]: 'collection group set list array category folder directory catalog inventory database', [NounType.Dataset]: 'dataset data table spreadsheet database records statistics metrics measurements analysis', // Business/Application Types [NounType.Product]: 'product item merchandise offering service feature application software solution package', [NounType.Service]: 'service offering subscription support maintenance utility function capability', [NounType.User]: 'user account profile member subscriber customer client participant identity credentials', [NounType.Task]: 'task action todo item job assignment duty responsibility activity step procedure', [NounType.Project]: 'project initiative program campaign effort endeavor plan scheme venture undertaking', // Descriptive Types [NounType.Process]: 'process workflow procedure method algorithm sequence pipeline operation routine protocol', [NounType.State]: 'state status condition phase stage mode situation circumstance configuration setting', [NounType.Role]: 'role position title function responsibility duty job capacity designation authority', [NounType.Topic]: 'subject topic theme category tag keyword area domain field discipline specialty', [NounType.Language]: 'language dialect locale tongue vernacular communication speech linguistics vocabulary', [NounType.Currency]: 'currency money dollar euro pound yen bitcoin payment financial monetary unit', [NounType.Measurement]: 'measurement metric quantity value amount size dimension weight height volume distance', // Scientific/Research Types [NounType.Hypothesis]: 'hypothesis theory proposition thesis assumption premise conjecture speculation prediction', [NounType.Experiment]: 'experiment test trial study research investigation analysis observation examination', // Legal/Regulatory Types [NounType.Contract]: 'contract agreement deal treaty pact covenant license terms conditions policy', [NounType.Regulation]: 'regulation law rule policy standard compliance requirement guideline ordinance statute', // Technical Infrastructure Types [NounType.Interface]: 'interface API endpoint protocol specification contract schema definition connection', [NounType.Resource]: 'resource infrastructure server database storage compute memory bandwidth capacity asset' } const VERB_TYPE_DESCRIPTIONS: Record = { // Core Relationship Types [VerbType.RelatedTo]: 'related connected associated linked correlated relevant pertinent applicable', [VerbType.Contains]: 'contains includes holds stores encompasses comprises consists incorporates', [VerbType.PartOf]: 'part component element member piece portion section segment constituent', [VerbType.LocatedAt]: 'located situated positioned placed found exists resides occupies', [VerbType.References]: 'references cites mentions points links refers quotes sources', // Temporal/Causal Types [VerbType.Precedes]: 'precedes before earlier prior previous antecedent preliminary foregoing', [VerbType.Succeeds]: 'succeeds follows after later subsequent next ensuing succeeding', [VerbType.Causes]: 'causes triggers induces produces generates results influences affects', [VerbType.DependsOn]: 'depends requires needs relies necessitates contingent prerequisite', [VerbType.Requires]: 'requires needs demands necessitates mandates obliges compels entails', // Creation/Transformation Types [VerbType.Creates]: 'creates makes produces generates builds constructs forms establishes', [VerbType.Transforms]: 'transforms converts changes modifies alters transitions morphs evolves', [VerbType.Becomes]: 'becomes turns evolves transforms changes transitions develops grows', [VerbType.Modifies]: 'modifies changes updates alters edits revises adjusts adapts', [VerbType.Consumes]: 'consumes uses utilizes depletes expends absorbs takes processes', // Ownership/Attribution Types [VerbType.Owns]: 'owns possesses holds controls manages administers governs maintains', [VerbType.AttributedTo]: 'attributed credited assigned ascribed authored written composed', [VerbType.CreatedBy]: 'created made produced generated built developed authored written', [VerbType.BelongsTo]: 'belongs property possession part member affiliate associated owned', // Social/Organizational Types [VerbType.MemberOf]: 'member participant affiliate associate belongs joined enrolled registered', [VerbType.WorksWith]: 'works collaborates cooperates partners teams assists helps supports', [VerbType.FriendOf]: 'friend companion buddy pal acquaintance associate connection relationship', [VerbType.Follows]: 'follows subscribes tracks monitors watches observes trails pursues', [VerbType.Likes]: 'likes enjoys appreciates favors prefers admires values endorses', [VerbType.ReportsTo]: 'reports answers subordinate accountable responsible supervised managed', [VerbType.Supervises]: 'supervises manages oversees directs leads controls guides administers', [VerbType.Mentors]: 'mentors teaches guides coaches instructs trains advises counsels', [VerbType.Communicates]: 'communicates talks speaks messages contacts interacts corresponds exchanges', // Descriptive/Functional Types [VerbType.Describes]: 'describes explains details documents specifies outlines depicts characterizes', [VerbType.Defines]: 'defines specifies establishes determines sets declares identifies designates', [VerbType.Categorizes]: 'categorizes classifies groups sorts organizes arranges labels tags', [VerbType.Measures]: 'measures quantifies gauges assesses evaluates calculates determines counts', [VerbType.Evaluates]: 'evaluates assesses analyzes reviews examines appraises judges rates', [VerbType.Uses]: 'uses utilizes employs applies operates handles manipulates exploits', [VerbType.Implements]: 'implements executes realizes performs accomplishes carries delivers completes', [VerbType.Extends]: 'extends expands enhances augments amplifies broadens enlarges develops', // Enhanced Relationships [VerbType.Inherits]: 'inherits derives extends receives obtains acquires succeeds legacy', [VerbType.Conflicts]: 'conflicts contradicts opposes clashes disputes disagrees incompatible inconsistent', [VerbType.Synchronizes]: 'synchronizes coordinates aligns harmonizes matches corresponds parallels coincides', [VerbType.Competes]: 'competes rivals contends contests challenges opposes vies struggles' } async function buildTypeEmbeddings() { console.log('🧠 Building embedded type embeddings for Brainy...') // Count types const nounTypes = Object.keys(NOUN_TYPE_DESCRIPTIONS) const verbTypes = Object.keys(VERB_TYPE_DESCRIPTIONS) console.log(`📊 Processing ${nounTypes.length} noun types and ${verbTypes.length} verb types...`) // Initialize TransformerEmbedding for embedding (one-time only!) const embedder = new TransformerEmbedding({ verbose: true, localFilesOnly: false // Allow downloading models during build }) await embedder.init() console.log('✅ TransformerEmbedding initialized') // Generate noun type embeddings const nounEmbeddings = new Map() console.log('📝 Generating noun type embeddings...') for (const [type, description] of Object.entries(NOUN_TYPE_DESCRIPTIONS)) { try { const embedding = await embedder.embed(description) if (embedding && Array.isArray(embedding)) { nounEmbeddings.set(type, embedding) console.log(` ✓ ${type}`) } } catch (error) { console.warn(` ⚠️ Failed to embed noun type: ${type}`) } } // Generate verb type embeddings const verbEmbeddings = new Map() console.log('📝 Generating verb type embeddings...') for (const [type, description] of Object.entries(VERB_TYPE_DESCRIPTIONS)) { try { const embedding = await embedder.embed(description) if (embedding && Array.isArray(embedding)) { verbEmbeddings.set(type, embedding) console.log(` ✓ ${type}`) } } catch (error) { console.warn(` ⚠️ Failed to embed verb type: ${type}`) } } console.log(`✅ Generated ${nounEmbeddings.size} noun embeddings and ${verbEmbeddings.size} verb embeddings`) // Get embedding dimension const embeddingDim = nounEmbeddings.size > 0 ? Array.from(nounEmbeddings.values())[0]?.length ?? 384 : 384 // Convert to compact binary format const totalTypes = nounTypes.length + verbTypes.length const totalFloats = totalTypes * embeddingDim const buffer = new ArrayBuffer(totalFloats * 4) const view = new DataView(buffer) let offset = 0 // Pack noun embeddings for (const type of nounTypes) { const embedding = nounEmbeddings.get(type) || new Array(embeddingDim).fill(0) for (let i = 0; i < embeddingDim; i++) { view.setFloat32(offset, embedding[i], true) // little-endian offset += 4 } } // Pack verb embeddings for (const type of verbTypes) { const embedding = verbEmbeddings.get(type) || new Array(embeddingDim).fill(0) for (let i = 0; i < embeddingDim; i++) { view.setFloat32(offset, embedding[i], true) // little-endian offset += 4 } } // Convert to base64 const uint8 = new Uint8Array(buffer) const base64 = Buffer.from(uint8).toString('base64') // Generate TypeScript file const tsContent = `/** * 🧠 BRAINY EMBEDDED TYPE EMBEDDINGS * * AUTO-GENERATED - DO NOT EDIT * Generated: ${new Date().toISOString()} * Noun Types: ${nounTypes.length} * Verb Types: ${verbTypes.length} * * This file contains pre-computed embeddings for all NounTypes and VerbTypes. * No runtime computation needed, instant availability! */ import { NounType, VerbType } from '../types/graphTypes.js' import { Vector } from '../coreTypes.js' // Type metadata export const TYPE_METADATA = { nounTypes: ${nounTypes.length}, verbTypes: ${verbTypes.length}, totalTypes: ${totalTypes}, embeddingDimensions: ${embeddingDim}, generatedAt: "${new Date().toISOString()}", sizeBytes: { embeddings: ${buffer.byteLength}, base64: ${base64.length} } } // All noun types in order const NOUN_TYPE_ORDER: NounType[] = ${JSON.stringify(nounTypes)} // All verb types in order const VERB_TYPE_ORDER: VerbType[] = ${JSON.stringify(verbTypes)} // Pre-computed embeddings (${(base64.length / 1024).toFixed(1)}KB base64) const EMBEDDINGS_BASE64 = "${base64}" // Decode embeddings at startup (happens once, <10ms) function decodeEmbeddings(): Uint8Array { if (typeof Buffer !== 'undefined') { // Node.js environment return Buffer.from(EMBEDDINGS_BASE64, 'base64') } else if (typeof atob !== 'undefined') { // Browser environment const binaryString = atob(EMBEDDINGS_BASE64) const bytes = new Uint8Array(binaryString.length) for (let i = 0; i < binaryString.length; i++) { bytes[i] = binaryString.charCodeAt(i) } return bytes } return new Uint8Array(0) } // Cached decoded embeddings let decodedEmbeddings: Uint8Array | null = null /** * Get noun type embeddings as a Map for fast lookup * This is called once and cached */ export function getNounTypeEmbeddings(): Map { if (!decodedEmbeddings) { decodedEmbeddings = decodeEmbeddings() } const embeddings = new Map() const view = new DataView(decodedEmbeddings.buffer) const embeddingSize = ${embeddingDim} NOUN_TYPE_ORDER.forEach((type, index) => { const offset = index * embeddingSize * 4 const embedding = new Float32Array(embeddingSize) for (let i = 0; i < embeddingSize; i++) { embedding[i] = view.getFloat32(offset + i * 4, true) } embeddings.set(type, Array.from(embedding)) }) return embeddings } /** * Get verb type embeddings as a Map for fast lookup * This is called once and cached */ export function getVerbTypeEmbeddings(): Map { if (!decodedEmbeddings) { decodedEmbeddings = decodeEmbeddings() } const embeddings = new Map() const view = new DataView(decodedEmbeddings.buffer) const embeddingSize = ${embeddingDim} // Verb embeddings start after noun embeddings const verbStartOffset = ${nounTypes.length} * embeddingSize * 4 VERB_TYPE_ORDER.forEach((type, index) => { const offset = verbStartOffset + index * embeddingSize * 4 const embedding = new Float32Array(embeddingSize) for (let i = 0; i < embeddingSize; i++) { embedding[i] = view.getFloat32(offset + i * 4, true) } embeddings.set(type, Array.from(embedding)) }) return embeddings } // Import logging import { prodLog } from '../utils/logger.js' prodLog.info(\`🧠 Brainy Type Embeddings loaded: \${TYPE_METADATA.nounTypes} nouns, \${TYPE_METADATA.verbTypes} verbs, \${(TYPE_METADATA.sizeBytes.embeddings / 1024).toFixed(1)}KB\`) ` // Write the TypeScript file const outputPath = path.join(__dirname, '..', 'src', 'neural', 'embeddedTypeEmbeddings.ts') await fs.writeFile(outputPath, tsContent) // Report statistics console.log(` ✅ EMBEDDED TYPE EMBEDDINGS BUILT SUCCESSFULLY! ================================================ Noun Types: ${nounTypes.length} Verb Types: ${verbTypes.length} Total Types: ${totalTypes} Embedding Dimensions: ${embeddingDim} File sizes: Embeddings binary: ${(buffer.byteLength / 1024).toFixed(1)} KB Base64 encoded: ${(base64.length / 1024).toFixed(1)} KB Output: ${outputPath} Type embeddings are now embedded directly in Brainy! No runtime computation needed, instant availability. `) } // Run if called directly if (import.meta.url === `file://${process.argv[1]}`) { buildTypeEmbeddings().catch(console.error) } export { buildTypeEmbeddings }