brainy/src/index.ts

445 lines
8.9 KiB
TypeScript
Raw Normal View History

2025-06-24 11:41:30 -07:00
/**
* Brainy
* A vector and graph database using HNSW
2025-06-24 11:41:30 -07:00
*/
feat\!: migrate from TensorFlow.js to Transformers.js with ONNX Runtime BREAKING CHANGE: Complete migration from TensorFlow.js to Transformers.js for embedding generation This is a major architectural change that replaces TensorFlow.js (USE model) with Transformers.js (all-MiniLM-L6-v2) for significantly improved performance and reduced complexity. Key Changes: - Replace TensorFlow.js Universal Sentence Encoder with Transformers.js all-MiniLM-L6-v2 - Reduce model size from 525MB to 87MB (83% reduction) - Reduce embedding dimensions from 512 to 384 (faster distance calculations) - Remove TensorFlow.js Float32Array patching (caused ONNX conflicts) - Implement smart bundled model detection for offline operation - Add explicit model download script for Docker deployments - Remove complex environment variables in favor of simple configuration - Update all distance functions to use optimized pure JavaScript - Remove TensorFlow-specific utilities and type definitions Performance Improvements: - Model loading: 5x faster (87MB vs 525MB) - Memory usage: 75% reduction (~200-400MB vs ~1.5GB) - Distance calculations: Faster pure JS vs GPU overhead for small vectors - Cold start performance: Significantly improved Files Changed: - Updated package.json: New dependencies, simplified scripts - Rewrote src/utils/embedding.ts: Complete Transformers.js implementation - Updated src/utils/distance.ts: Optimized JavaScript distance functions - Simplified src/setup.ts: Removed TensorFlow-specific patching - Simplified src/utils/textEncoding.ts: Only Node.js TextEncoder/Decoder patches - Deleted src/utils/robustModelLoader.ts: TensorFlow-specific loader - Deleted src/types/tensorflowTypes.ts: TensorFlow type definitions - Added scripts/download-models.cjs: Docker-compatible model downloader - Added comprehensive documentation: README.md, OFFLINE_MODELS.md, analysis docs Testing: - All 19 tests passing - Removed test mocking in favor of real implementation testing - Updated test environment for Transformers.js compatibility - Performance tests validate improved efficiency This migration resolves production issues with Docker egress limitations and provides a more robust, performant foundation for vector operations.
2025-08-05 19:29:59 -07:00
// No setup needed - using clean ONNX Runtime with Transformers.js
2025-06-24 11:41:30 -07:00
// Export main BrainyData class and related types
import { BrainyData, BrainyDataConfig } from './brainyData.js'
2025-06-24 11:41:30 -07:00
export { BrainyData }
export type { BrainyDataConfig }
2025-06-24 11:41:30 -07:00
// Export distance functions for convenience
import {
euclideanDistance,
cosineDistance,
manhattanDistance,
dotProductDistance,
getStatistics
2025-06-24 11:41:30 -07:00
} from './utils/index.js'
export {
euclideanDistance,
cosineDistance,
manhattanDistance,
dotProductDistance,
getStatistics
2025-06-24 11:41:30 -07:00
}
// Export embedding functionality
import {
UniversalSentenceEncoder,
feat\!: migrate from TensorFlow.js to Transformers.js with ONNX Runtime BREAKING CHANGE: Complete migration from TensorFlow.js to Transformers.js for embedding generation This is a major architectural change that replaces TensorFlow.js (USE model) with Transformers.js (all-MiniLM-L6-v2) for significantly improved performance and reduced complexity. Key Changes: - Replace TensorFlow.js Universal Sentence Encoder with Transformers.js all-MiniLM-L6-v2 - Reduce model size from 525MB to 87MB (83% reduction) - Reduce embedding dimensions from 512 to 384 (faster distance calculations) - Remove TensorFlow.js Float32Array patching (caused ONNX conflicts) - Implement smart bundled model detection for offline operation - Add explicit model download script for Docker deployments - Remove complex environment variables in favor of simple configuration - Update all distance functions to use optimized pure JavaScript - Remove TensorFlow-specific utilities and type definitions Performance Improvements: - Model loading: 5x faster (87MB vs 525MB) - Memory usage: 75% reduction (~200-400MB vs ~1.5GB) - Distance calculations: Faster pure JS vs GPU overhead for small vectors - Cold start performance: Significantly improved Files Changed: - Updated package.json: New dependencies, simplified scripts - Rewrote src/utils/embedding.ts: Complete Transformers.js implementation - Updated src/utils/distance.ts: Optimized JavaScript distance functions - Simplified src/setup.ts: Removed TensorFlow-specific patching - Simplified src/utils/textEncoding.ts: Only Node.js TextEncoder/Decoder patches - Deleted src/utils/robustModelLoader.ts: TensorFlow-specific loader - Deleted src/types/tensorflowTypes.ts: TensorFlow type definitions - Added scripts/download-models.cjs: Docker-compatible model downloader - Added comprehensive documentation: README.md, OFFLINE_MODELS.md, analysis docs Testing: - All 19 tests passing - Removed test mocking in favor of real implementation testing - Updated test environment for Transformers.js compatibility - Performance tests validate improved efficiency This migration resolves production issues with Docker egress limitations and provides a more robust, performant foundation for vector operations.
2025-08-05 19:29:59 -07:00
TransformerEmbedding,
createEmbeddingFunction,
feat\!: migrate from TensorFlow.js to Transformers.js with ONNX Runtime BREAKING CHANGE: Complete migration from TensorFlow.js to Transformers.js for embedding generation This is a major architectural change that replaces TensorFlow.js (USE model) with Transformers.js (all-MiniLM-L6-v2) for significantly improved performance and reduced complexity. Key Changes: - Replace TensorFlow.js Universal Sentence Encoder with Transformers.js all-MiniLM-L6-v2 - Reduce model size from 525MB to 87MB (83% reduction) - Reduce embedding dimensions from 512 to 384 (faster distance calculations) - Remove TensorFlow.js Float32Array patching (caused ONNX conflicts) - Implement smart bundled model detection for offline operation - Add explicit model download script for Docker deployments - Remove complex environment variables in favor of simple configuration - Update all distance functions to use optimized pure JavaScript - Remove TensorFlow-specific utilities and type definitions Performance Improvements: - Model loading: 5x faster (87MB vs 525MB) - Memory usage: 75% reduction (~200-400MB vs ~1.5GB) - Distance calculations: Faster pure JS vs GPU overhead for small vectors - Cold start performance: Significantly improved Files Changed: - Updated package.json: New dependencies, simplified scripts - Rewrote src/utils/embedding.ts: Complete Transformers.js implementation - Updated src/utils/distance.ts: Optimized JavaScript distance functions - Simplified src/setup.ts: Removed TensorFlow-specific patching - Simplified src/utils/textEncoding.ts: Only Node.js TextEncoder/Decoder patches - Deleted src/utils/robustModelLoader.ts: TensorFlow-specific loader - Deleted src/types/tensorflowTypes.ts: TensorFlow type definitions - Added scripts/download-models.cjs: Docker-compatible model downloader - Added comprehensive documentation: README.md, OFFLINE_MODELS.md, analysis docs Testing: - All 19 tests passing - Removed test mocking in favor of real implementation testing - Updated test environment for Transformers.js compatibility - Performance tests validate improved efficiency This migration resolves production issues with Docker egress limitations and provides a more robust, performant foundation for vector operations.
2025-08-05 19:29:59 -07:00
defaultEmbeddingFunction,
batchEmbed,
embeddingFunctions
2025-06-24 11:41:30 -07:00
} from './utils/embedding.js'
// Export worker utilities
import { executeInThread, cleanupWorkerPools } from './utils/workerUtils.js'
// Export logging utilities
import {
logger,
LogLevel,
configureLogger,
createModuleLogger
} from './utils/logger.js'
// Export environment utilities
import {
isBrowser,
isNode,
isWebWorker,
areWebWorkersAvailable,
areWorkerThreadsAvailable,
areWorkerThreadsAvailableSync,
isThreadingAvailable,
isThreadingAvailableAsync
} from './utils/environment.js'
2025-06-24 11:41:30 -07:00
export {
UniversalSentenceEncoder,
feat\!: migrate from TensorFlow.js to Transformers.js with ONNX Runtime BREAKING CHANGE: Complete migration from TensorFlow.js to Transformers.js for embedding generation This is a major architectural change that replaces TensorFlow.js (USE model) with Transformers.js (all-MiniLM-L6-v2) for significantly improved performance and reduced complexity. Key Changes: - Replace TensorFlow.js Universal Sentence Encoder with Transformers.js all-MiniLM-L6-v2 - Reduce model size from 525MB to 87MB (83% reduction) - Reduce embedding dimensions from 512 to 384 (faster distance calculations) - Remove TensorFlow.js Float32Array patching (caused ONNX conflicts) - Implement smart bundled model detection for offline operation - Add explicit model download script for Docker deployments - Remove complex environment variables in favor of simple configuration - Update all distance functions to use optimized pure JavaScript - Remove TensorFlow-specific utilities and type definitions Performance Improvements: - Model loading: 5x faster (87MB vs 525MB) - Memory usage: 75% reduction (~200-400MB vs ~1.5GB) - Distance calculations: Faster pure JS vs GPU overhead for small vectors - Cold start performance: Significantly improved Files Changed: - Updated package.json: New dependencies, simplified scripts - Rewrote src/utils/embedding.ts: Complete Transformers.js implementation - Updated src/utils/distance.ts: Optimized JavaScript distance functions - Simplified src/setup.ts: Removed TensorFlow-specific patching - Simplified src/utils/textEncoding.ts: Only Node.js TextEncoder/Decoder patches - Deleted src/utils/robustModelLoader.ts: TensorFlow-specific loader - Deleted src/types/tensorflowTypes.ts: TensorFlow type definitions - Added scripts/download-models.cjs: Docker-compatible model downloader - Added comprehensive documentation: README.md, OFFLINE_MODELS.md, analysis docs Testing: - All 19 tests passing - Removed test mocking in favor of real implementation testing - Updated test environment for Transformers.js compatibility - Performance tests validate improved efficiency This migration resolves production issues with Docker egress limitations and provides a more robust, performant foundation for vector operations.
2025-08-05 19:29:59 -07:00
TransformerEmbedding,
createEmbeddingFunction,
defaultEmbeddingFunction,
feat\!: migrate from TensorFlow.js to Transformers.js with ONNX Runtime BREAKING CHANGE: Complete migration from TensorFlow.js to Transformers.js for embedding generation This is a major architectural change that replaces TensorFlow.js (USE model) with Transformers.js (all-MiniLM-L6-v2) for significantly improved performance and reduced complexity. Key Changes: - Replace TensorFlow.js Universal Sentence Encoder with Transformers.js all-MiniLM-L6-v2 - Reduce model size from 525MB to 87MB (83% reduction) - Reduce embedding dimensions from 512 to 384 (faster distance calculations) - Remove TensorFlow.js Float32Array patching (caused ONNX conflicts) - Implement smart bundled model detection for offline operation - Add explicit model download script for Docker deployments - Remove complex environment variables in favor of simple configuration - Update all distance functions to use optimized pure JavaScript - Remove TensorFlow-specific utilities and type definitions Performance Improvements: - Model loading: 5x faster (87MB vs 525MB) - Memory usage: 75% reduction (~200-400MB vs ~1.5GB) - Distance calculations: Faster pure JS vs GPU overhead for small vectors - Cold start performance: Significantly improved Files Changed: - Updated package.json: New dependencies, simplified scripts - Rewrote src/utils/embedding.ts: Complete Transformers.js implementation - Updated src/utils/distance.ts: Optimized JavaScript distance functions - Simplified src/setup.ts: Removed TensorFlow-specific patching - Simplified src/utils/textEncoding.ts: Only Node.js TextEncoder/Decoder patches - Deleted src/utils/robustModelLoader.ts: TensorFlow-specific loader - Deleted src/types/tensorflowTypes.ts: TensorFlow type definitions - Added scripts/download-models.cjs: Docker-compatible model downloader - Added comprehensive documentation: README.md, OFFLINE_MODELS.md, analysis docs Testing: - All 19 tests passing - Removed test mocking in favor of real implementation testing - Updated test environment for Transformers.js compatibility - Performance tests validate improved efficiency This migration resolves production issues with Docker egress limitations and provides a more robust, performant foundation for vector operations.
2025-08-05 19:29:59 -07:00
batchEmbed,
embeddingFunctions,
// Worker utilities
executeInThread,
cleanupWorkerPools,
// Environment utilities
isBrowser,
isNode,
isWebWorker,
areWebWorkersAvailable,
areWorkerThreadsAvailable,
areWorkerThreadsAvailableSync,
isThreadingAvailable,
isThreadingAvailableAsync,
// Logging utilities
logger,
LogLevel,
configureLogger,
createModuleLogger
2025-06-24 11:41:30 -07:00
}
// Export storage adapters
import {
OPFSStorage,
MemoryStorage,
R2Storage,
S3CompatibleStorage,
createStorage
} from './storage/storageFactory.js'
2025-06-24 11:41:30 -07:00
export {
OPFSStorage,
MemoryStorage,
R2Storage,
S3CompatibleStorage,
createStorage
2025-06-24 11:41:30 -07:00
}
// FileSystemStorage is exported separately to avoid browser build issues
export { FileSystemStorage } from './storage/adapters/fileSystemStorage.js'
2025-06-24 11:41:30 -07:00
// Export unified pipeline
import {
Pipeline,
pipeline,
augmentationPipeline,
ExecutionMode,
PipelineOptions,
PipelineResult,
executeStreamlined,
executeByType,
executeSingle,
processStaticData,
processStreamingData,
createPipeline,
createStreamingPipeline,
StreamlinedExecutionMode,
StreamlinedPipelineOptions,
StreamlinedPipelineResult
2025-06-24 11:41:30 -07:00
} from './pipeline.js'
// Export sequential pipeline (for backward compatibility)
import {
SequentialPipeline,
sequentialPipeline,
SequentialPipelineOptions
2025-06-24 11:41:30 -07:00
} from './sequentialPipeline.js'
// Export augmentation factory
import {
createSenseAugmentation,
addWebSocketSupport,
executeAugmentation,
loadAugmentationModule,
AugmentationOptions
2025-06-24 11:41:30 -07:00
} from './augmentationFactory.js'
export {
// Unified pipeline exports
Pipeline,
pipeline,
augmentationPipeline,
ExecutionMode,
SequentialPipeline,
sequentialPipeline,
// Streamlined pipeline exports (now part of unified pipeline)
executeStreamlined,
executeByType,
executeSingle,
processStaticData,
processStreamingData,
createPipeline,
createStreamingPipeline,
StreamlinedExecutionMode,
// Augmentation factory exports
createSenseAugmentation,
addWebSocketSupport,
executeAugmentation,
loadAugmentationModule
2025-06-24 11:41:30 -07:00
}
export type {
PipelineOptions,
PipelineResult,
SequentialPipelineOptions,
StreamlinedPipelineOptions,
StreamlinedPipelineResult,
AugmentationOptions
2025-06-24 11:41:30 -07:00
}
// Export augmentation registry for build-time loading
import {
availableAugmentations,
registerAugmentation,
initializeAugmentationPipeline,
setAugmentationEnabled,
getAugmentationsByType
2025-06-24 11:41:30 -07:00
} from './augmentationRegistry.js'
export {
availableAugmentations,
registerAugmentation,
initializeAugmentationPipeline,
setAugmentationEnabled,
getAugmentationsByType
2025-06-24 11:41:30 -07:00
}
// Export augmentation registry loader for build tools
import {
loadAugmentationsFromModules,
createAugmentationRegistryPlugin,
createAugmentationRegistryRollupPlugin
2025-06-24 11:41:30 -07:00
} from './augmentationRegistryLoader.js'
import type {
AugmentationRegistryLoaderOptions,
AugmentationLoadResult
2025-06-24 11:41:30 -07:00
} from './augmentationRegistryLoader.js'
export {
loadAugmentationsFromModules,
createAugmentationRegistryPlugin,
createAugmentationRegistryRollupPlugin
2025-06-24 11:41:30 -07:00
}
export type { AugmentationRegistryLoaderOptions, AugmentationLoadResult }
2025-06-24 11:41:30 -07:00
// Export augmentation implementations
import {
MemoryStorageAugmentation,
FileSystemStorageAugmentation,
OPFSStorageAugmentation,
createMemoryAugmentation
2025-06-24 11:41:30 -07:00
} from './augmentations/memoryAugmentations.js'
import {
WebSocketConduitAugmentation,
WebRTCConduitAugmentation,
createConduitAugmentation
2025-06-24 11:41:30 -07:00
} from './augmentations/conduitAugmentations.js'
import {
ServerSearchConduitAugmentation,
ServerSearchActivationAugmentation,
createServerSearchAugmentations
2025-06-24 11:41:30 -07:00
} from './augmentations/serverSearchAugmentations.js'
// Non-LLM exports
export {
MemoryStorageAugmentation,
FileSystemStorageAugmentation,
OPFSStorageAugmentation,
createMemoryAugmentation,
WebSocketConduitAugmentation,
WebRTCConduitAugmentation,
createConduitAugmentation,
ServerSearchConduitAugmentation,
ServerSearchActivationAugmentation,
createServerSearchAugmentations
2025-06-24 11:41:30 -07:00
}
// LLM augmentations are optional and not imported by default
// They can be imported directly from their module if needed:
// import { LLMCognitionAugmentation, LLMActivationAugmentation, createLLMAugmentations } from './augmentations/llmAugmentations.js'
// Export types
import type {
Vector,
VectorDocument,
SearchResult,
DistanceFunction,
EmbeddingFunction,
EmbeddingModel,
HNSWNoun,
HNSWVerb,
HNSWConfig,
StorageAdapter
2025-06-24 11:41:30 -07:00
} from './coreTypes.js'
// Export HNSW index and optimized version
import { HNSWIndex } from './hnsw/hnswIndex.js'
import {
HNSWIndexOptimized,
HNSWOptimizedConfig
} from './hnsw/hnswIndexOptimized.js'
2025-06-24 11:41:30 -07:00
export { HNSWIndex, HNSWIndexOptimized }
2025-06-24 11:41:30 -07:00
export type {
Vector,
VectorDocument,
SearchResult,
DistanceFunction,
EmbeddingFunction,
EmbeddingModel,
HNSWNoun,
HNSWVerb,
HNSWConfig,
HNSWOptimizedConfig,
StorageAdapter
2025-06-24 11:41:30 -07:00
}
// Export augmentation types
import type {
IAugmentation,
AugmentationResponse,
IWebSocketSupport,
ISenseAugmentation,
IConduitAugmentation,
ICognitionAugmentation,
IMemoryAugmentation,
IPerceptionAugmentation,
IDialogAugmentation,
IActivationAugmentation
2025-06-24 11:41:30 -07:00
} from './types/augmentations.js'
import { AugmentationType, BrainyAugmentations } from './types/augmentations.js'
2025-06-24 11:41:30 -07:00
export type { IAugmentation, AugmentationResponse, IWebSocketSupport }
2025-06-24 11:41:30 -07:00
export {
AugmentationType,
BrainyAugmentations,
ISenseAugmentation,
IConduitAugmentation,
ICognitionAugmentation,
IMemoryAugmentation,
IPerceptionAugmentation,
IDialogAugmentation,
IActivationAugmentation
2025-06-24 11:41:30 -07:00
}
// Export combined WebSocket augmentation interfaces
export type {
IWebSocketCognitionAugmentation,
IWebSocketSenseAugmentation,
IWebSocketPerceptionAugmentation,
IWebSocketActivationAugmentation,
IWebSocketDialogAugmentation,
IWebSocketConduitAugmentation,
IWebSocketMemoryAugmentation
2025-06-24 11:41:30 -07:00
} from './types/augmentations.js'
// Export graph types
import type {
GraphNoun,
GraphVerb,
EmbeddedGraphVerb,
Person,
Location,
Thing,
Event,
Concept,
Content,
Collection,
Organization,
Document,
Media,
File,
Message,
Dataset,
Product,
Service,
User,
Task,
Project,
Process,
State,
Role,
Topic,
Language,
Currency,
Measurement
2025-06-24 11:41:30 -07:00
} from './types/graphTypes.js'
import { NounType, VerbType } from './types/graphTypes.js'
2025-06-24 11:41:30 -07:00
export type {
GraphNoun,
GraphVerb,
EmbeddedGraphVerb,
Person,
Location,
Thing,
Event,
Concept,
Content,
Collection,
Organization,
Document,
Media,
File,
Message,
Dataset,
Product,
Service,
User,
Task,
Project,
Process,
State,
Role,
Topic,
Language,
Currency,
Measurement
2025-06-24 11:41:30 -07:00
}
// Export type utility functions
import { getNounTypes, getVerbTypes, getNounTypeMap, getVerbTypeMap } from './utils/typeUtils.js'
export {
NounType,
VerbType,
getNounTypes,
getVerbTypes,
getNounTypeMap,
getVerbTypeMap
}
2025-06-24 11:41:30 -07:00
// Export MCP (Model Control Protocol) components
import {
BrainyMCPAdapter,
MCPAugmentationToolset,
BrainyMCPService
2025-06-24 11:41:30 -07:00
} from './mcp/index.js' // Import from mcp/index.js
import {
MCPRequest,
MCPResponse,
MCPDataAccessRequest,
MCPToolExecutionRequest,
MCPSystemInfoRequest,
MCPAuthenticationRequest,
MCPRequestType,
MCPServiceOptions,
MCPTool,
MCP_VERSION
2025-06-24 11:41:30 -07:00
} from './types/mcpTypes.js'
export {
// MCP classes
BrainyMCPAdapter,
MCPAugmentationToolset,
BrainyMCPService,
// MCP types
MCPRequestType,
MCP_VERSION
2025-06-24 11:41:30 -07:00
}
export type {
MCPRequest,
MCPResponse,
MCPDataAccessRequest,
MCPToolExecutionRequest,
MCPSystemInfoRequest,
MCPAuthenticationRequest,
MCPServiceOptions,
MCPTool
2025-06-24 11:41:30 -07:00
}