brainy/src/index.ts

500 lines
10 KiB
TypeScript
Raw Normal View History

2025-06-24 11:41:30 -07:00
/**
feat: Simplify architecture with Cortex orchestrator and clear augmentation tiers ## Major Architecture Improvements ### Cortex Refactoring - Renamed AugmentationPipeline → Cortex for clarity - Cortex is now the central orchestrator (not an augmentation) - NeuralImport remains as the AI-powered SENSE augmentation - Clean brain metaphor: BrainyData → Cortex → Augmentations ### Four-Tier Augmentation System 1. **Built-in** (Free, MIT): Neural Import, basic storage, search 2. **Community** (Free, npm): Community-created augmentations 3. **Premium** ($49-299/mo): AI Memory, Agent Coordinator, Enterprise connectors 4. **Brain Cloud** ($19-99/mo): Managed service with all features ### Zero Configuration Philosophy - Everything works out of the box - no config needed - Automatic model detection and loading - Seamless integration between tiers - Brain Cloud connects with one command: `brainy cloud` ### Documentation Updates - Added PHILOSOPHY.md outlining design principles - Created AUGMENTATION_ARCHITECTURE.md with tier system - Added CLI_AUGMENTATION_GUIDE.md for augmentation management - Updated README to "sell first" with concrete use cases - Improved documentation organization in /docs ### Developer Experience - Backward compatibility maintained with exports - Clean, simple API surface - Interactive-by-default approach - Premium features integrate seamlessly ### Removed - Deleted demo directory and deploy workflow (moved to website) - Removed test wrapper scripts (bash 2>&1 bug workaround) This refactor makes Brainy incredibly powerful yet easy to use, with everything automated and no configuration required. The Brain Cloud augmentations (AI memory, sync, coordination) integrate seamlessly as our killer features.
2025-08-11 09:40:37 -07:00
* Brainy - Your AI-Powered Second Brain
* 🧠 A multi-dimensional database with vector, graph, and facet storage
*
* Core Components:
* - BrainyData: The brain (core database)
* - Cortex: The orchestrator (manages augmentations)
* - NeuralImport: AI-powered data understanding
* - Augmentations: Brain capabilities (plugins)
2025-06-24 11:41:30 -07:00
*/
// Export main BrainyData class and related types
import { BrainyData, BrainyDataConfig } from './brainyData.js'
2025-06-24 11:41:30 -07:00
export { BrainyData }
export type { BrainyDataConfig }
2025-06-24 11:41:30 -07:00
feat: Simplify architecture with Cortex orchestrator and clear augmentation tiers ## Major Architecture Improvements ### Cortex Refactoring - Renamed AugmentationPipeline → Cortex for clarity - Cortex is now the central orchestrator (not an augmentation) - NeuralImport remains as the AI-powered SENSE augmentation - Clean brain metaphor: BrainyData → Cortex → Augmentations ### Four-Tier Augmentation System 1. **Built-in** (Free, MIT): Neural Import, basic storage, search 2. **Community** (Free, npm): Community-created augmentations 3. **Premium** ($49-299/mo): AI Memory, Agent Coordinator, Enterprise connectors 4. **Brain Cloud** ($19-99/mo): Managed service with all features ### Zero Configuration Philosophy - Everything works out of the box - no config needed - Automatic model detection and loading - Seamless integration between tiers - Brain Cloud connects with one command: `brainy cloud` ### Documentation Updates - Added PHILOSOPHY.md outlining design principles - Created AUGMENTATION_ARCHITECTURE.md with tier system - Added CLI_AUGMENTATION_GUIDE.md for augmentation management - Updated README to "sell first" with concrete use cases - Improved documentation organization in /docs ### Developer Experience - Backward compatibility maintained with exports - Clean, simple API surface - Interactive-by-default approach - Premium features integrate seamlessly ### Removed - Deleted demo directory and deploy workflow (moved to website) - Removed test wrapper scripts (bash 2>&1 bug workaround) This refactor makes Brainy incredibly powerful yet easy to use, with everything automated and no configuration required. The Brain Cloud augmentations (AI memory, sync, coordination) integrate seamlessly as our killer features.
2025-08-11 09:40:37 -07:00
// Export Cortex (the orchestrator)
export {
Cortex,
cortex
} from './cortex.js'
// Export Neural Import (AI data understanding)
export { NeuralImport } from './cortex/neuralImport.js'
export type {
NeuralAnalysisResult,
DetectedEntity,
DetectedRelationship,
NeuralInsight,
NeuralImportOptions
} from './cortex/neuralImport.js'
// Augmentation types are already exported later in the file
2025-06-24 11:41:30 -07:00
// Export distance functions for convenience
import {
euclideanDistance,
cosineDistance,
manhattanDistance,
dotProductDistance,
getStatistics
2025-06-24 11:41:30 -07:00
} from './utils/index.js'
export {
euclideanDistance,
cosineDistance,
manhattanDistance,
dotProductDistance,
getStatistics
2025-06-24 11:41:30 -07:00
}
// Export embedding functionality
import {
UniversalSentenceEncoder,
feat\!: migrate from TensorFlow.js to Transformers.js with ONNX Runtime BREAKING CHANGE: Complete migration from TensorFlow.js to Transformers.js for embedding generation This is a major architectural change that replaces TensorFlow.js (USE model) with Transformers.js (all-MiniLM-L6-v2) for significantly improved performance and reduced complexity. Key Changes: - Replace TensorFlow.js Universal Sentence Encoder with Transformers.js all-MiniLM-L6-v2 - Reduce model size from 525MB to 87MB (83% reduction) - Reduce embedding dimensions from 512 to 384 (faster distance calculations) - Remove TensorFlow.js Float32Array patching (caused ONNX conflicts) - Implement smart bundled model detection for offline operation - Add explicit model download script for Docker deployments - Remove complex environment variables in favor of simple configuration - Update all distance functions to use optimized pure JavaScript - Remove TensorFlow-specific utilities and type definitions Performance Improvements: - Model loading: 5x faster (87MB vs 525MB) - Memory usage: 75% reduction (~200-400MB vs ~1.5GB) - Distance calculations: Faster pure JS vs GPU overhead for small vectors - Cold start performance: Significantly improved Files Changed: - Updated package.json: New dependencies, simplified scripts - Rewrote src/utils/embedding.ts: Complete Transformers.js implementation - Updated src/utils/distance.ts: Optimized JavaScript distance functions - Simplified src/setup.ts: Removed TensorFlow-specific patching - Simplified src/utils/textEncoding.ts: Only Node.js TextEncoder/Decoder patches - Deleted src/utils/robustModelLoader.ts: TensorFlow-specific loader - Deleted src/types/tensorflowTypes.ts: TensorFlow type definitions - Added scripts/download-models.cjs: Docker-compatible model downloader - Added comprehensive documentation: README.md, OFFLINE_MODELS.md, analysis docs Testing: - All 19 tests passing - Removed test mocking in favor of real implementation testing - Updated test environment for Transformers.js compatibility - Performance tests validate improved efficiency This migration resolves production issues with Docker egress limitations and provides a more robust, performant foundation for vector operations.
2025-08-05 19:29:59 -07:00
TransformerEmbedding,
createEmbeddingFunction,
feat\!: migrate from TensorFlow.js to Transformers.js with ONNX Runtime BREAKING CHANGE: Complete migration from TensorFlow.js to Transformers.js for embedding generation This is a major architectural change that replaces TensorFlow.js (USE model) with Transformers.js (all-MiniLM-L6-v2) for significantly improved performance and reduced complexity. Key Changes: - Replace TensorFlow.js Universal Sentence Encoder with Transformers.js all-MiniLM-L6-v2 - Reduce model size from 525MB to 87MB (83% reduction) - Reduce embedding dimensions from 512 to 384 (faster distance calculations) - Remove TensorFlow.js Float32Array patching (caused ONNX conflicts) - Implement smart bundled model detection for offline operation - Add explicit model download script for Docker deployments - Remove complex environment variables in favor of simple configuration - Update all distance functions to use optimized pure JavaScript - Remove TensorFlow-specific utilities and type definitions Performance Improvements: - Model loading: 5x faster (87MB vs 525MB) - Memory usage: 75% reduction (~200-400MB vs ~1.5GB) - Distance calculations: Faster pure JS vs GPU overhead for small vectors - Cold start performance: Significantly improved Files Changed: - Updated package.json: New dependencies, simplified scripts - Rewrote src/utils/embedding.ts: Complete Transformers.js implementation - Updated src/utils/distance.ts: Optimized JavaScript distance functions - Simplified src/setup.ts: Removed TensorFlow-specific patching - Simplified src/utils/textEncoding.ts: Only Node.js TextEncoder/Decoder patches - Deleted src/utils/robustModelLoader.ts: TensorFlow-specific loader - Deleted src/types/tensorflowTypes.ts: TensorFlow type definitions - Added scripts/download-models.cjs: Docker-compatible model downloader - Added comprehensive documentation: README.md, OFFLINE_MODELS.md, analysis docs Testing: - All 19 tests passing - Removed test mocking in favor of real implementation testing - Updated test environment for Transformers.js compatibility - Performance tests validate improved efficiency This migration resolves production issues with Docker egress limitations and provides a more robust, performant foundation for vector operations.
2025-08-05 19:29:59 -07:00
defaultEmbeddingFunction,
batchEmbed,
embeddingFunctions
2025-06-24 11:41:30 -07:00
} from './utils/embedding.js'
// Export worker utilities
import { executeInThread, cleanupWorkerPools } from './utils/workerUtils.js'
// Export logging utilities
import {
logger,
LogLevel,
configureLogger,
createModuleLogger
} from './utils/logger.js'
// Export BrainyChat for conversational AI
import { BrainyChat, ChatOptions } from './chat/brainyChat.js'
export { BrainyChat }
export type { ChatOptions }
// Export Cortex CLI functionality - commented out for core MIT build
// export { Cortex } from './cortex/cortex.js'
// Export performance and optimization utilities
import {
getGlobalSocketManager,
AdaptiveSocketManager
} from './utils/adaptiveSocketManager.js'
import {
getGlobalBackpressure,
AdaptiveBackpressure
} from './utils/adaptiveBackpressure.js'
import {
getGlobalPerformanceMonitor,
PerformanceMonitor
} from './utils/performanceMonitor.js'
// Export environment utilities
import {
isBrowser,
isNode,
isWebWorker,
areWebWorkersAvailable,
areWorkerThreadsAvailable,
areWorkerThreadsAvailableSync,
isThreadingAvailable,
isThreadingAvailableAsync
} from './utils/environment.js'
2025-06-24 11:41:30 -07:00
export {
UniversalSentenceEncoder,
feat\!: migrate from TensorFlow.js to Transformers.js with ONNX Runtime BREAKING CHANGE: Complete migration from TensorFlow.js to Transformers.js for embedding generation This is a major architectural change that replaces TensorFlow.js (USE model) with Transformers.js (all-MiniLM-L6-v2) for significantly improved performance and reduced complexity. Key Changes: - Replace TensorFlow.js Universal Sentence Encoder with Transformers.js all-MiniLM-L6-v2 - Reduce model size from 525MB to 87MB (83% reduction) - Reduce embedding dimensions from 512 to 384 (faster distance calculations) - Remove TensorFlow.js Float32Array patching (caused ONNX conflicts) - Implement smart bundled model detection for offline operation - Add explicit model download script for Docker deployments - Remove complex environment variables in favor of simple configuration - Update all distance functions to use optimized pure JavaScript - Remove TensorFlow-specific utilities and type definitions Performance Improvements: - Model loading: 5x faster (87MB vs 525MB) - Memory usage: 75% reduction (~200-400MB vs ~1.5GB) - Distance calculations: Faster pure JS vs GPU overhead for small vectors - Cold start performance: Significantly improved Files Changed: - Updated package.json: New dependencies, simplified scripts - Rewrote src/utils/embedding.ts: Complete Transformers.js implementation - Updated src/utils/distance.ts: Optimized JavaScript distance functions - Simplified src/setup.ts: Removed TensorFlow-specific patching - Simplified src/utils/textEncoding.ts: Only Node.js TextEncoder/Decoder patches - Deleted src/utils/robustModelLoader.ts: TensorFlow-specific loader - Deleted src/types/tensorflowTypes.ts: TensorFlow type definitions - Added scripts/download-models.cjs: Docker-compatible model downloader - Added comprehensive documentation: README.md, OFFLINE_MODELS.md, analysis docs Testing: - All 19 tests passing - Removed test mocking in favor of real implementation testing - Updated test environment for Transformers.js compatibility - Performance tests validate improved efficiency This migration resolves production issues with Docker egress limitations and provides a more robust, performant foundation for vector operations.
2025-08-05 19:29:59 -07:00
TransformerEmbedding,
createEmbeddingFunction,
defaultEmbeddingFunction,
feat\!: migrate from TensorFlow.js to Transformers.js with ONNX Runtime BREAKING CHANGE: Complete migration from TensorFlow.js to Transformers.js for embedding generation This is a major architectural change that replaces TensorFlow.js (USE model) with Transformers.js (all-MiniLM-L6-v2) for significantly improved performance and reduced complexity. Key Changes: - Replace TensorFlow.js Universal Sentence Encoder with Transformers.js all-MiniLM-L6-v2 - Reduce model size from 525MB to 87MB (83% reduction) - Reduce embedding dimensions from 512 to 384 (faster distance calculations) - Remove TensorFlow.js Float32Array patching (caused ONNX conflicts) - Implement smart bundled model detection for offline operation - Add explicit model download script for Docker deployments - Remove complex environment variables in favor of simple configuration - Update all distance functions to use optimized pure JavaScript - Remove TensorFlow-specific utilities and type definitions Performance Improvements: - Model loading: 5x faster (87MB vs 525MB) - Memory usage: 75% reduction (~200-400MB vs ~1.5GB) - Distance calculations: Faster pure JS vs GPU overhead for small vectors - Cold start performance: Significantly improved Files Changed: - Updated package.json: New dependencies, simplified scripts - Rewrote src/utils/embedding.ts: Complete Transformers.js implementation - Updated src/utils/distance.ts: Optimized JavaScript distance functions - Simplified src/setup.ts: Removed TensorFlow-specific patching - Simplified src/utils/textEncoding.ts: Only Node.js TextEncoder/Decoder patches - Deleted src/utils/robustModelLoader.ts: TensorFlow-specific loader - Deleted src/types/tensorflowTypes.ts: TensorFlow type definitions - Added scripts/download-models.cjs: Docker-compatible model downloader - Added comprehensive documentation: README.md, OFFLINE_MODELS.md, analysis docs Testing: - All 19 tests passing - Removed test mocking in favor of real implementation testing - Updated test environment for Transformers.js compatibility - Performance tests validate improved efficiency This migration resolves production issues with Docker egress limitations and provides a more robust, performant foundation for vector operations.
2025-08-05 19:29:59 -07:00
batchEmbed,
embeddingFunctions,
// Worker utilities
executeInThread,
cleanupWorkerPools,
// Environment utilities
isBrowser,
isNode,
isWebWorker,
areWebWorkersAvailable,
areWorkerThreadsAvailable,
areWorkerThreadsAvailableSync,
isThreadingAvailable,
isThreadingAvailableAsync,
// Logging utilities
logger,
LogLevel,
configureLogger,
createModuleLogger,
// Performance and optimization utilities
getGlobalSocketManager,
AdaptiveSocketManager,
getGlobalBackpressure,
AdaptiveBackpressure,
getGlobalPerformanceMonitor,
PerformanceMonitor
2025-06-24 11:41:30 -07:00
}
// Export storage adapters
import {
OPFSStorage,
MemoryStorage,
R2Storage,
S3CompatibleStorage,
createStorage
} from './storage/storageFactory.js'
2025-06-24 11:41:30 -07:00
export {
OPFSStorage,
MemoryStorage,
R2Storage,
S3CompatibleStorage,
createStorage
2025-06-24 11:41:30 -07:00
}
// FileSystemStorage is exported separately to avoid browser build issues
export { FileSystemStorage } from './storage/adapters/fileSystemStorage.js'
2025-06-24 11:41:30 -07:00
// Export unified pipeline
import {
Pipeline,
pipeline,
augmentationPipeline,
ExecutionMode,
PipelineOptions,
PipelineResult,
executeStreamlined,
executeByType,
executeSingle,
processStaticData,
processStreamingData,
createPipeline,
createStreamingPipeline,
StreamlinedExecutionMode,
StreamlinedPipelineOptions,
StreamlinedPipelineResult
2025-06-24 11:41:30 -07:00
} from './pipeline.js'
// Export sequential pipeline (for backward compatibility)
import {
SequentialPipeline,
sequentialPipeline,
SequentialPipelineOptions
2025-06-24 11:41:30 -07:00
} from './sequentialPipeline.js'
// Export augmentation factory
import {
createSenseAugmentation,
addWebSocketSupport,
executeAugmentation,
loadAugmentationModule,
AugmentationOptions
2025-06-24 11:41:30 -07:00
} from './augmentationFactory.js'
export {
// Unified pipeline exports
Pipeline,
pipeline,
augmentationPipeline,
ExecutionMode,
SequentialPipeline,
sequentialPipeline,
// Streamlined pipeline exports (now part of unified pipeline)
executeStreamlined,
executeByType,
executeSingle,
processStaticData,
processStreamingData,
createPipeline,
createStreamingPipeline,
StreamlinedExecutionMode,
// Augmentation factory exports
createSenseAugmentation,
addWebSocketSupport,
executeAugmentation,
loadAugmentationModule
2025-06-24 11:41:30 -07:00
}
export type {
PipelineOptions,
PipelineResult,
SequentialPipelineOptions,
StreamlinedPipelineOptions,
StreamlinedPipelineResult,
AugmentationOptions
2025-06-24 11:41:30 -07:00
}
// Export augmentation registry for build-time loading
import {
availableAugmentations,
registerAugmentation,
initializeAugmentationPipeline,
setAugmentationEnabled,
getAugmentationsByType
2025-06-24 11:41:30 -07:00
} from './augmentationRegistry.js'
export {
availableAugmentations,
registerAugmentation,
initializeAugmentationPipeline,
setAugmentationEnabled,
getAugmentationsByType
2025-06-24 11:41:30 -07:00
}
// Export augmentation registry loader for build tools
import {
loadAugmentationsFromModules,
createAugmentationRegistryPlugin,
createAugmentationRegistryRollupPlugin
2025-06-24 11:41:30 -07:00
} from './augmentationRegistryLoader.js'
import type {
AugmentationRegistryLoaderOptions,
AugmentationLoadResult
2025-06-24 11:41:30 -07:00
} from './augmentationRegistryLoader.js'
export {
loadAugmentationsFromModules,
createAugmentationRegistryPlugin,
createAugmentationRegistryRollupPlugin
2025-06-24 11:41:30 -07:00
}
export type { AugmentationRegistryLoaderOptions, AugmentationLoadResult }
2025-06-24 11:41:30 -07:00
2025-06-24 11:41:30 -07:00
// Export augmentation implementations
import {
MemoryStorageAugmentation,
FileSystemStorageAugmentation,
OPFSStorageAugmentation,
createMemoryAugmentation
2025-06-24 11:41:30 -07:00
} from './augmentations/memoryAugmentations.js'
import {
WebSocketConduitAugmentation,
WebRTCConduitAugmentation,
createConduitAugmentation
2025-06-24 11:41:30 -07:00
} from './augmentations/conduitAugmentations.js'
import {
ServerSearchConduitAugmentation,
ServerSearchActivationAugmentation,
createServerSearchAugmentations
2025-06-24 11:41:30 -07:00
} from './augmentations/serverSearchAugmentations.js'
// Non-LLM exports
export {
MemoryStorageAugmentation,
FileSystemStorageAugmentation,
OPFSStorageAugmentation,
createMemoryAugmentation,
WebSocketConduitAugmentation,
WebRTCConduitAugmentation,
createConduitAugmentation,
ServerSearchConduitAugmentation,
ServerSearchActivationAugmentation,
createServerSearchAugmentations
2025-06-24 11:41:30 -07:00
}
// LLM augmentations are optional and not imported by default
// They can be imported directly from their module if needed:
// import { LLMCognitionAugmentation, LLMActivationAugmentation, createLLMAugmentations } from './augmentations/llmAugmentations.js'
// Export types
import type {
Vector,
VectorDocument,
SearchResult,
DistanceFunction,
EmbeddingFunction,
EmbeddingModel,
HNSWNoun,
HNSWVerb,
HNSWConfig,
StorageAdapter
2025-06-24 11:41:30 -07:00
} from './coreTypes.js'
// Export HNSW index and optimized version
import { HNSWIndex } from './hnsw/hnswIndex.js'
import {
HNSWIndexOptimized,
HNSWOptimizedConfig
} from './hnsw/hnswIndexOptimized.js'
2025-06-24 11:41:30 -07:00
export { HNSWIndex, HNSWIndexOptimized }
2025-06-24 11:41:30 -07:00
export type {
Vector,
VectorDocument,
SearchResult,
DistanceFunction,
EmbeddingFunction,
EmbeddingModel,
HNSWNoun,
HNSWVerb,
HNSWConfig,
HNSWOptimizedConfig,
StorageAdapter
2025-06-24 11:41:30 -07:00
}
// Export augmentation types
import type {
IAugmentation,
AugmentationResponse,
IWebSocketSupport,
ISenseAugmentation,
IConduitAugmentation,
ICognitionAugmentation,
IMemoryAugmentation,
IPerceptionAugmentation,
IDialogAugmentation,
IActivationAugmentation
2025-06-24 11:41:30 -07:00
} from './types/augmentations.js'
import { AugmentationType, BrainyAugmentations } from './types/augmentations.js'
2025-06-24 11:41:30 -07:00
export type { IAugmentation, AugmentationResponse, IWebSocketSupport }
2025-06-24 11:41:30 -07:00
export {
AugmentationType,
BrainyAugmentations,
ISenseAugmentation,
IConduitAugmentation,
ICognitionAugmentation,
IMemoryAugmentation,
IPerceptionAugmentation,
IDialogAugmentation,
IActivationAugmentation
2025-06-24 11:41:30 -07:00
}
// Export combined WebSocket augmentation interfaces
export type {
IWebSocketCognitionAugmentation,
IWebSocketSenseAugmentation,
IWebSocketPerceptionAugmentation,
IWebSocketActivationAugmentation,
IWebSocketDialogAugmentation,
IWebSocketConduitAugmentation,
IWebSocketMemoryAugmentation
2025-06-24 11:41:30 -07:00
} from './types/augmentations.js'
// Export graph types
import type {
GraphNoun,
GraphVerb,
EmbeddedGraphVerb,
Person,
Location,
Thing,
Event,
Concept,
Content,
Collection,
Organization,
Document,
Media,
File,
Message,
Dataset,
Product,
Service,
User,
Task,
Project,
Process,
State,
Role,
Topic,
Language,
Currency,
Measurement
2025-06-24 11:41:30 -07:00
} from './types/graphTypes.js'
import { NounType, VerbType } from './types/graphTypes.js'
2025-06-24 11:41:30 -07:00
export type {
GraphNoun,
GraphVerb,
EmbeddedGraphVerb,
Person,
Location,
Thing,
Event,
Concept,
Content,
Collection,
Organization,
Document,
Media,
File,
Message,
Dataset,
Product,
Service,
User,
Task,
Project,
Process,
State,
Role,
Topic,
Language,
Currency,
Measurement
2025-06-24 11:41:30 -07:00
}
// Export type utility functions
import { getNounTypes, getVerbTypes, getNounTypeMap, getVerbTypeMap } from './utils/typeUtils.js'
export {
NounType,
VerbType,
getNounTypes,
getVerbTypes,
getNounTypeMap,
getVerbTypeMap
}
2025-06-24 11:41:30 -07:00
// Export MCP (Model Control Protocol) components
import {
BrainyMCPAdapter,
MCPAugmentationToolset,
BrainyMCPService
2025-06-24 11:41:30 -07:00
} from './mcp/index.js' // Import from mcp/index.js
import {
MCPRequest,
MCPResponse,
MCPDataAccessRequest,
MCPToolExecutionRequest,
MCPSystemInfoRequest,
MCPAuthenticationRequest,
MCPRequestType,
MCPServiceOptions,
MCPTool,
MCP_VERSION
2025-06-24 11:41:30 -07:00
} from './types/mcpTypes.js'
export {
// MCP classes
BrainyMCPAdapter,
MCPAugmentationToolset,
BrainyMCPService,
// MCP types
MCPRequestType,
MCP_VERSION
2025-06-24 11:41:30 -07:00
}
export type {
MCPRequest,
MCPResponse,
MCPDataAccessRequest,
MCPToolExecutionRequest,
MCPSystemInfoRequest,
MCPAuthenticationRequest,
MCPServiceOptions,
MCPTool
2025-06-24 11:41:30 -07:00
}