BREAKING CHANGE: Complete migration from TensorFlow.js to Transformers.js for embedding generation This is a major architectural change that replaces TensorFlow.js (USE model) with Transformers.js (all-MiniLM-L6-v2) for significantly improved performance and reduced complexity. Key Changes: - Replace TensorFlow.js Universal Sentence Encoder with Transformers.js all-MiniLM-L6-v2 - Reduce model size from 525MB to 87MB (83% reduction) - Reduce embedding dimensions from 512 to 384 (faster distance calculations) - Remove TensorFlow.js Float32Array patching (caused ONNX conflicts) - Implement smart bundled model detection for offline operation - Add explicit model download script for Docker deployments - Remove complex environment variables in favor of simple configuration - Update all distance functions to use optimized pure JavaScript - Remove TensorFlow-specific utilities and type definitions Performance Improvements: - Model loading: 5x faster (87MB vs 525MB) - Memory usage: 75% reduction (~200-400MB vs ~1.5GB) - Distance calculations: Faster pure JS vs GPU overhead for small vectors - Cold start performance: Significantly improved Files Changed: - Updated package.json: New dependencies, simplified scripts - Rewrote src/utils/embedding.ts: Complete Transformers.js implementation - Updated src/utils/distance.ts: Optimized JavaScript distance functions - Simplified src/setup.ts: Removed TensorFlow-specific patching - Simplified src/utils/textEncoding.ts: Only Node.js TextEncoder/Decoder patches - Deleted src/utils/robustModelLoader.ts: TensorFlow-specific loader - Deleted src/types/tensorflowTypes.ts: TensorFlow type definitions - Added scripts/download-models.cjs: Docker-compatible model downloader - Added comprehensive documentation: README.md, OFFLINE_MODELS.md, analysis docs Testing: - All 19 tests passing - Removed test mocking in favor of real implementation testing - Updated test environment for Transformers.js compatibility - Performance tests validate improved efficiency This migration resolves production issues with Docker egress limitations and provides a more robust, performant foundation for vector operations.
444 lines
8.9 KiB
TypeScript
444 lines
8.9 KiB
TypeScript
/**
|
|
* Brainy
|
|
* A vector and graph database using HNSW
|
|
*/
|
|
|
|
// No setup needed - using clean ONNX Runtime with Transformers.js
|
|
|
|
// Export main BrainyData class and related types
|
|
import { BrainyData, BrainyDataConfig } from './brainyData.js'
|
|
|
|
export { BrainyData }
|
|
export type { BrainyDataConfig }
|
|
|
|
// Export distance functions for convenience
|
|
import {
|
|
euclideanDistance,
|
|
cosineDistance,
|
|
manhattanDistance,
|
|
dotProductDistance,
|
|
getStatistics
|
|
} from './utils/index.js'
|
|
|
|
export {
|
|
euclideanDistance,
|
|
cosineDistance,
|
|
manhattanDistance,
|
|
dotProductDistance,
|
|
getStatistics
|
|
}
|
|
|
|
// Export embedding functionality
|
|
import {
|
|
UniversalSentenceEncoder,
|
|
TransformerEmbedding,
|
|
createEmbeddingFunction,
|
|
defaultEmbeddingFunction,
|
|
batchEmbed,
|
|
embeddingFunctions
|
|
} from './utils/embedding.js'
|
|
|
|
// Export worker utilities
|
|
import { executeInThread, cleanupWorkerPools } from './utils/workerUtils.js'
|
|
|
|
// Export logging utilities
|
|
import {
|
|
logger,
|
|
LogLevel,
|
|
configureLogger,
|
|
createModuleLogger
|
|
} from './utils/logger.js'
|
|
|
|
// Export environment utilities
|
|
import {
|
|
isBrowser,
|
|
isNode,
|
|
isWebWorker,
|
|
areWebWorkersAvailable,
|
|
areWorkerThreadsAvailable,
|
|
areWorkerThreadsAvailableSync,
|
|
isThreadingAvailable,
|
|
isThreadingAvailableAsync
|
|
} from './utils/environment.js'
|
|
|
|
export {
|
|
UniversalSentenceEncoder,
|
|
TransformerEmbedding,
|
|
createEmbeddingFunction,
|
|
defaultEmbeddingFunction,
|
|
batchEmbed,
|
|
embeddingFunctions,
|
|
|
|
// Worker utilities
|
|
executeInThread,
|
|
cleanupWorkerPools,
|
|
|
|
// Environment utilities
|
|
isBrowser,
|
|
isNode,
|
|
isWebWorker,
|
|
areWebWorkersAvailable,
|
|
areWorkerThreadsAvailable,
|
|
areWorkerThreadsAvailableSync,
|
|
isThreadingAvailable,
|
|
isThreadingAvailableAsync,
|
|
|
|
// Logging utilities
|
|
logger,
|
|
LogLevel,
|
|
configureLogger,
|
|
createModuleLogger
|
|
}
|
|
|
|
// Export storage adapters
|
|
import {
|
|
OPFSStorage,
|
|
MemoryStorage,
|
|
R2Storage,
|
|
S3CompatibleStorage,
|
|
createStorage
|
|
} from './storage/storageFactory.js'
|
|
|
|
export {
|
|
OPFSStorage,
|
|
MemoryStorage,
|
|
R2Storage,
|
|
S3CompatibleStorage,
|
|
createStorage
|
|
}
|
|
|
|
// FileSystemStorage is exported separately to avoid browser build issues
|
|
export { FileSystemStorage } from './storage/adapters/fileSystemStorage.js'
|
|
|
|
// Export unified pipeline
|
|
import {
|
|
Pipeline,
|
|
pipeline,
|
|
augmentationPipeline,
|
|
ExecutionMode,
|
|
PipelineOptions,
|
|
PipelineResult,
|
|
executeStreamlined,
|
|
executeByType,
|
|
executeSingle,
|
|
processStaticData,
|
|
processStreamingData,
|
|
createPipeline,
|
|
createStreamingPipeline,
|
|
StreamlinedExecutionMode,
|
|
StreamlinedPipelineOptions,
|
|
StreamlinedPipelineResult
|
|
} from './pipeline.js'
|
|
|
|
// Export sequential pipeline (for backward compatibility)
|
|
import {
|
|
SequentialPipeline,
|
|
sequentialPipeline,
|
|
SequentialPipelineOptions
|
|
} from './sequentialPipeline.js'
|
|
|
|
// Export augmentation factory
|
|
import {
|
|
createSenseAugmentation,
|
|
addWebSocketSupport,
|
|
executeAugmentation,
|
|
loadAugmentationModule,
|
|
AugmentationOptions
|
|
} from './augmentationFactory.js'
|
|
|
|
export {
|
|
// Unified pipeline exports
|
|
Pipeline,
|
|
pipeline,
|
|
augmentationPipeline,
|
|
ExecutionMode,
|
|
SequentialPipeline,
|
|
sequentialPipeline,
|
|
|
|
// Streamlined pipeline exports (now part of unified pipeline)
|
|
executeStreamlined,
|
|
executeByType,
|
|
executeSingle,
|
|
processStaticData,
|
|
processStreamingData,
|
|
createPipeline,
|
|
createStreamingPipeline,
|
|
StreamlinedExecutionMode,
|
|
|
|
// Augmentation factory exports
|
|
createSenseAugmentation,
|
|
addWebSocketSupport,
|
|
executeAugmentation,
|
|
loadAugmentationModule
|
|
}
|
|
export type {
|
|
PipelineOptions,
|
|
PipelineResult,
|
|
SequentialPipelineOptions,
|
|
StreamlinedPipelineOptions,
|
|
StreamlinedPipelineResult,
|
|
AugmentationOptions
|
|
}
|
|
|
|
// Export augmentation registry for build-time loading
|
|
import {
|
|
availableAugmentations,
|
|
registerAugmentation,
|
|
initializeAugmentationPipeline,
|
|
setAugmentationEnabled,
|
|
getAugmentationsByType
|
|
} from './augmentationRegistry.js'
|
|
|
|
export {
|
|
availableAugmentations,
|
|
registerAugmentation,
|
|
initializeAugmentationPipeline,
|
|
setAugmentationEnabled,
|
|
getAugmentationsByType
|
|
}
|
|
|
|
// Export augmentation registry loader for build tools
|
|
import {
|
|
loadAugmentationsFromModules,
|
|
createAugmentationRegistryPlugin,
|
|
createAugmentationRegistryRollupPlugin
|
|
} from './augmentationRegistryLoader.js'
|
|
import type {
|
|
AugmentationRegistryLoaderOptions,
|
|
AugmentationLoadResult
|
|
} from './augmentationRegistryLoader.js'
|
|
|
|
export {
|
|
loadAugmentationsFromModules,
|
|
createAugmentationRegistryPlugin,
|
|
createAugmentationRegistryRollupPlugin
|
|
}
|
|
export type { AugmentationRegistryLoaderOptions, AugmentationLoadResult }
|
|
|
|
// Export augmentation implementations
|
|
import {
|
|
MemoryStorageAugmentation,
|
|
FileSystemStorageAugmentation,
|
|
OPFSStorageAugmentation,
|
|
createMemoryAugmentation
|
|
} from './augmentations/memoryAugmentations.js'
|
|
import {
|
|
WebSocketConduitAugmentation,
|
|
WebRTCConduitAugmentation,
|
|
createConduitAugmentation
|
|
} from './augmentations/conduitAugmentations.js'
|
|
import {
|
|
ServerSearchConduitAugmentation,
|
|
ServerSearchActivationAugmentation,
|
|
createServerSearchAugmentations
|
|
} from './augmentations/serverSearchAugmentations.js'
|
|
|
|
// Non-LLM exports
|
|
export {
|
|
MemoryStorageAugmentation,
|
|
FileSystemStorageAugmentation,
|
|
OPFSStorageAugmentation,
|
|
createMemoryAugmentation,
|
|
WebSocketConduitAugmentation,
|
|
WebRTCConduitAugmentation,
|
|
createConduitAugmentation,
|
|
ServerSearchConduitAugmentation,
|
|
ServerSearchActivationAugmentation,
|
|
createServerSearchAugmentations
|
|
}
|
|
|
|
// LLM augmentations are optional and not imported by default
|
|
// They can be imported directly from their module if needed:
|
|
// import { LLMCognitionAugmentation, LLMActivationAugmentation, createLLMAugmentations } from './augmentations/llmAugmentations.js'
|
|
|
|
// Export types
|
|
import type {
|
|
Vector,
|
|
VectorDocument,
|
|
SearchResult,
|
|
DistanceFunction,
|
|
EmbeddingFunction,
|
|
EmbeddingModel,
|
|
HNSWNoun,
|
|
HNSWVerb,
|
|
HNSWConfig,
|
|
StorageAdapter
|
|
} from './coreTypes.js'
|
|
|
|
// Export HNSW index and optimized version
|
|
import { HNSWIndex } from './hnsw/hnswIndex.js'
|
|
import {
|
|
HNSWIndexOptimized,
|
|
HNSWOptimizedConfig
|
|
} from './hnsw/hnswIndexOptimized.js'
|
|
|
|
export { HNSWIndex, HNSWIndexOptimized }
|
|
|
|
export type {
|
|
Vector,
|
|
VectorDocument,
|
|
SearchResult,
|
|
DistanceFunction,
|
|
EmbeddingFunction,
|
|
EmbeddingModel,
|
|
HNSWNoun,
|
|
HNSWVerb,
|
|
HNSWConfig,
|
|
HNSWOptimizedConfig,
|
|
StorageAdapter
|
|
}
|
|
|
|
// Export augmentation types
|
|
import type {
|
|
IAugmentation,
|
|
AugmentationResponse,
|
|
IWebSocketSupport,
|
|
ISenseAugmentation,
|
|
IConduitAugmentation,
|
|
ICognitionAugmentation,
|
|
IMemoryAugmentation,
|
|
IPerceptionAugmentation,
|
|
IDialogAugmentation,
|
|
IActivationAugmentation
|
|
} from './types/augmentations.js'
|
|
import { AugmentationType, BrainyAugmentations } from './types/augmentations.js'
|
|
|
|
export type { IAugmentation, AugmentationResponse, IWebSocketSupport }
|
|
export {
|
|
AugmentationType,
|
|
BrainyAugmentations,
|
|
ISenseAugmentation,
|
|
IConduitAugmentation,
|
|
ICognitionAugmentation,
|
|
IMemoryAugmentation,
|
|
IPerceptionAugmentation,
|
|
IDialogAugmentation,
|
|
IActivationAugmentation
|
|
}
|
|
|
|
// Export combined WebSocket augmentation interfaces
|
|
export type {
|
|
IWebSocketCognitionAugmentation,
|
|
IWebSocketSenseAugmentation,
|
|
IWebSocketPerceptionAugmentation,
|
|
IWebSocketActivationAugmentation,
|
|
IWebSocketDialogAugmentation,
|
|
IWebSocketConduitAugmentation,
|
|
IWebSocketMemoryAugmentation
|
|
} from './types/augmentations.js'
|
|
|
|
// Export graph types
|
|
import type {
|
|
GraphNoun,
|
|
GraphVerb,
|
|
EmbeddedGraphVerb,
|
|
Person,
|
|
Location,
|
|
Thing,
|
|
Event,
|
|
Concept,
|
|
Content,
|
|
Collection,
|
|
Organization,
|
|
Document,
|
|
Media,
|
|
File,
|
|
Message,
|
|
Dataset,
|
|
Product,
|
|
Service,
|
|
User,
|
|
Task,
|
|
Project,
|
|
Process,
|
|
State,
|
|
Role,
|
|
Topic,
|
|
Language,
|
|
Currency,
|
|
Measurement
|
|
} from './types/graphTypes.js'
|
|
import { NounType, VerbType } from './types/graphTypes.js'
|
|
|
|
export type {
|
|
GraphNoun,
|
|
GraphVerb,
|
|
EmbeddedGraphVerb,
|
|
Person,
|
|
Location,
|
|
Thing,
|
|
Event,
|
|
Concept,
|
|
Content,
|
|
Collection,
|
|
Organization,
|
|
Document,
|
|
Media,
|
|
File,
|
|
Message,
|
|
Dataset,
|
|
Product,
|
|
Service,
|
|
User,
|
|
Task,
|
|
Project,
|
|
Process,
|
|
State,
|
|
Role,
|
|
Topic,
|
|
Language,
|
|
Currency,
|
|
Measurement
|
|
}
|
|
// Export type utility functions
|
|
import { getNounTypes, getVerbTypes, getNounTypeMap, getVerbTypeMap } from './utils/typeUtils.js'
|
|
|
|
export {
|
|
NounType,
|
|
VerbType,
|
|
getNounTypes,
|
|
getVerbTypes,
|
|
getNounTypeMap,
|
|
getVerbTypeMap
|
|
}
|
|
|
|
// Export MCP (Model Control Protocol) components
|
|
import {
|
|
BrainyMCPAdapter,
|
|
MCPAugmentationToolset,
|
|
BrainyMCPService
|
|
} from './mcp/index.js' // Import from mcp/index.js
|
|
import {
|
|
MCPRequest,
|
|
MCPResponse,
|
|
MCPDataAccessRequest,
|
|
MCPToolExecutionRequest,
|
|
MCPSystemInfoRequest,
|
|
MCPAuthenticationRequest,
|
|
MCPRequestType,
|
|
MCPServiceOptions,
|
|
MCPTool,
|
|
MCP_VERSION
|
|
} from './types/mcpTypes.js'
|
|
|
|
export {
|
|
// MCP classes
|
|
BrainyMCPAdapter,
|
|
MCPAugmentationToolset,
|
|
BrainyMCPService,
|
|
|
|
// MCP types
|
|
MCPRequestType,
|
|
MCP_VERSION
|
|
}
|
|
|
|
export type {
|
|
MCPRequest,
|
|
MCPResponse,
|
|
MCPDataAccessRequest,
|
|
MCPToolExecutionRequest,
|
|
MCPSystemInfoRequest,
|
|
MCPAuthenticationRequest,
|
|
MCPServiceOptions,
|
|
MCPTool
|
|
}
|