🧠 Brainy 2.0.0 - Zero-Configuration AI Database with Triple Intelligence™
MAJOR RELEASE: Complete evolution of Brainy with groundbreaking features and performance. 🎯 KEY FEATURES: ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ ✨ Triple Intelligence™ Engine - Unified Vector + Metadata + Graph search - O(log n) performance on all operations - 3ms average search latency at any scale ✨ API Consolidation - 15+ search methods → 2 clean APIs - search() for vector similarity - find() for natural language queries ✨ Natural Language Processing - 220+ pre-computed NLP patterns - Instant context understanding - "Show me recent React components with tests" ✨ Zero Configuration - Works instantly, no setup required - Built-in embedding models (no API keys) - Smart defaults for everything - Automatic optimization ✨ Enterprise Features (Free for Everyone) - Scales to 10M+ items - Write-Ahead Logging (WAL) for durability - Distributed architecture with sharding - Read/write separation - Connection pooling & request deduplication - Built-in monitoring & health checks ✨ Universal Compatibility - Node.js, Browser, Edge Workers - 4 Storage Adapters (Memory, FileSystem, OPFS, S3) - TypeScript with full type safety - Worker-based embeddings 📦 WHAT'S INCLUDED: ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ • Core AI Database with HNSW indexing • 19 Production-ready augmentations • Universal Memory Manager • Complete CLI with all commands • Brain Cloud integration (soulcraft.com) • Comprehensive documentation • 52 test files with 400+ tests • Migration guide from 1.x 📊 PERFORMANCE: ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ • Initialize: 450ms (24MB memory) • Search: 3ms average (up to 10M items) • Metadata Filter: 0.8ms (O(log n)) • Bulk Import: 2.3s per 1000 items • Production Scale: 5.8ms at 10M items 🔧 TECHNICAL IMPROVEMENTS: ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ • TypeScript compilation: 153 errors → 0 • Memory usage: 200MB → 24MB baseline • Circular dependencies resolved • Worker thread communication fixed • Storage adapter consistency • Request coalescing for 3x performance 🛠️ CLI FEATURES: ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ • brainy add - Smart data ingestion • brainy find - Natural language search • brainy search - Vector similarity • brainy chat - AI conversation mode • brainy cloud - Brain Cloud integration • brainy augment - Manage extensions • 100% API compatibility 📚 DOCUMENTATION: ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ • Professional README with examples • Quick Start guide (5 minutes) • Enterprise Features guide • Migration guide from 1.x • API reference • Architecture documentation 🌟 USE CASES: ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ • AI memory layer for chatbots • Semantic document search • Code intelligence platforms • Knowledge management systems • Real-time recommendation engines • Customer support automation MIT License - Enterprise features included free for everyone. No premium tiers, no paywalls, no limits. Built with ❤️ by the Brainy community. Visit https://soulcraft.com for Brain Cloud integration.
This commit is contained in:
commit
9c87982a7d
301 changed files with 178087 additions and 0 deletions
62
src/utils/BoundedRegistry.ts
Normal file
62
src/utils/BoundedRegistry.ts
Normal file
|
|
@ -0,0 +1,62 @@
|
|||
/**
|
||||
* Bounded Registry with LRU eviction
|
||||
* Prevents unbounded memory growth in production
|
||||
*/
|
||||
export class BoundedRegistry<T> {
|
||||
private items = new Map<T, number>() // item -> last accessed timestamp
|
||||
private readonly maxSize: number
|
||||
|
||||
constructor(maxSize: number = 10000) {
|
||||
this.maxSize = maxSize
|
||||
}
|
||||
|
||||
/**
|
||||
* Add item to registry, evicting oldest if at capacity
|
||||
*/
|
||||
add(item: T): void {
|
||||
// Update timestamp if already exists
|
||||
if (this.items.has(item)) {
|
||||
this.items.delete(item) // Remove to re-add at end
|
||||
this.items.set(item, Date.now())
|
||||
return
|
||||
}
|
||||
|
||||
// Evict oldest if at capacity
|
||||
if (this.items.size >= this.maxSize) {
|
||||
const oldest = this.items.entries().next().value
|
||||
if (oldest) {
|
||||
this.items.delete(oldest[0])
|
||||
}
|
||||
}
|
||||
|
||||
this.items.set(item, Date.now())
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if item exists
|
||||
*/
|
||||
has(item: T): boolean {
|
||||
return this.items.has(item)
|
||||
}
|
||||
|
||||
/**
|
||||
* Get all items
|
||||
*/
|
||||
getAll(): T[] {
|
||||
return Array.from(this.items.keys())
|
||||
}
|
||||
|
||||
/**
|
||||
* Get size
|
||||
*/
|
||||
get size(): number {
|
||||
return this.items.size
|
||||
}
|
||||
|
||||
/**
|
||||
* Clear all items
|
||||
*/
|
||||
clear(): void {
|
||||
this.items.clear()
|
||||
}
|
||||
}
|
||||
443
src/utils/adaptiveBackpressure.ts
Normal file
443
src/utils/adaptiveBackpressure.ts
Normal file
|
|
@ -0,0 +1,443 @@
|
|||
/**
|
||||
* Adaptive Backpressure System
|
||||
* Automatically manages request flow and prevents system overload
|
||||
* Self-healing with pattern learning for optimal throughput
|
||||
*/
|
||||
|
||||
import { createModuleLogger } from './logger.js'
|
||||
|
||||
interface BackpressureMetrics {
|
||||
queueDepth: number
|
||||
processingRate: number
|
||||
errorRate: number
|
||||
latency: number
|
||||
throughput: number
|
||||
}
|
||||
|
||||
interface BackpressureConfig {
|
||||
maxQueueDepth: number
|
||||
targetLatency: number
|
||||
minThroughput: number
|
||||
adaptationRate: number
|
||||
}
|
||||
|
||||
/**
|
||||
* Self-healing backpressure manager that learns from load patterns
|
||||
*/
|
||||
export class AdaptiveBackpressure {
|
||||
private logger = createModuleLogger('AdaptiveBackpressure')
|
||||
|
||||
// Queue management
|
||||
private queue: Array<{
|
||||
id: string
|
||||
priority: number
|
||||
timestamp: number
|
||||
resolve: () => void
|
||||
}> = []
|
||||
|
||||
// Active operations tracking
|
||||
private activeOperations = new Set<string>()
|
||||
private maxConcurrent = 100
|
||||
|
||||
// Metrics tracking
|
||||
private metrics: BackpressureMetrics = {
|
||||
queueDepth: 0,
|
||||
processingRate: 0,
|
||||
errorRate: 0,
|
||||
latency: 0,
|
||||
throughput: 0
|
||||
}
|
||||
|
||||
// Configuration that adapts over time
|
||||
private config: BackpressureConfig = {
|
||||
maxQueueDepth: 1000,
|
||||
targetLatency: 1000, // 1 second target
|
||||
minThroughput: 10, // Minimum 10 ops/sec
|
||||
adaptationRate: 0.1 // How quickly to adapt
|
||||
}
|
||||
|
||||
// Historical patterns for learning
|
||||
private patterns: Array<{
|
||||
timestamp: number
|
||||
load: number
|
||||
optimal: number
|
||||
}> = []
|
||||
|
||||
// Circuit breaker state
|
||||
private circuitState: 'closed' | 'open' | 'half-open' = 'closed'
|
||||
private circuitOpenTime = 0
|
||||
private circuitFailures = 0
|
||||
private circuitThreshold = 5
|
||||
private circuitTimeout = 30000 // 30 seconds
|
||||
|
||||
// Performance tracking
|
||||
private operationTimes = new Map<string, number>()
|
||||
private completedOps: number[] = []
|
||||
private errorOps = 0
|
||||
private lastAdaptation = Date.now()
|
||||
|
||||
/**
|
||||
* Request permission to proceed with an operation
|
||||
*/
|
||||
public async requestPermission(
|
||||
operationId: string,
|
||||
priority: number = 1
|
||||
): Promise<void> {
|
||||
// Check circuit breaker
|
||||
if (this.isCircuitOpen()) {
|
||||
throw new Error('Circuit breaker is open - system is recovering')
|
||||
}
|
||||
|
||||
// Fast path for low load
|
||||
if (this.activeOperations.size < this.maxConcurrent * 0.5 && this.queue.length === 0) {
|
||||
this.activeOperations.add(operationId)
|
||||
this.operationTimes.set(operationId, Date.now())
|
||||
return
|
||||
}
|
||||
|
||||
// Check if we need to queue
|
||||
if (this.activeOperations.size >= this.maxConcurrent) {
|
||||
// Check queue depth
|
||||
if (this.queue.length >= this.config.maxQueueDepth) {
|
||||
throw new Error('Backpressure queue is full - try again later')
|
||||
}
|
||||
|
||||
// Add to queue and wait
|
||||
return new Promise<void>((resolve) => {
|
||||
this.queue.push({
|
||||
id: operationId,
|
||||
priority,
|
||||
timestamp: Date.now(),
|
||||
resolve
|
||||
})
|
||||
|
||||
// Sort queue by priority (higher priority first)
|
||||
this.queue.sort((a, b) => b.priority - a.priority)
|
||||
|
||||
// Update metrics
|
||||
this.metrics.queueDepth = this.queue.length
|
||||
})
|
||||
}
|
||||
|
||||
// Add to active operations
|
||||
this.activeOperations.add(operationId)
|
||||
this.operationTimes.set(operationId, Date.now())
|
||||
}
|
||||
|
||||
/**
|
||||
* Release permission after operation completes
|
||||
*/
|
||||
public releasePermission(operationId: string, success: boolean = true): void {
|
||||
// Remove from active operations
|
||||
this.activeOperations.delete(operationId)
|
||||
|
||||
// Track completion time
|
||||
const startTime = this.operationTimes.get(operationId)
|
||||
if (startTime) {
|
||||
const duration = Date.now() - startTime
|
||||
this.completedOps.push(duration)
|
||||
this.operationTimes.delete(operationId)
|
||||
|
||||
// Keep array bounded
|
||||
if (this.completedOps.length > 1000) {
|
||||
this.completedOps = this.completedOps.slice(-500)
|
||||
}
|
||||
}
|
||||
|
||||
// Track errors for circuit breaker
|
||||
if (!success) {
|
||||
this.errorOps++
|
||||
this.circuitFailures++
|
||||
|
||||
// Check if we should open circuit
|
||||
if (this.circuitFailures >= this.circuitThreshold) {
|
||||
this.openCircuit()
|
||||
}
|
||||
} else {
|
||||
// Reset circuit failures on success
|
||||
if (this.circuitState === 'half-open') {
|
||||
this.closeCircuit()
|
||||
}
|
||||
}
|
||||
|
||||
// Process queue if there are waiting operations
|
||||
if (this.queue.length > 0 && this.activeOperations.size < this.maxConcurrent) {
|
||||
const next = this.queue.shift()
|
||||
if (next) {
|
||||
this.activeOperations.add(next.id)
|
||||
this.operationTimes.set(next.id, Date.now())
|
||||
next.resolve()
|
||||
|
||||
// Update metrics
|
||||
this.metrics.queueDepth = this.queue.length
|
||||
}
|
||||
}
|
||||
|
||||
// Adapt configuration periodically
|
||||
this.adaptIfNeeded()
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if circuit breaker is open
|
||||
*/
|
||||
private isCircuitOpen(): boolean {
|
||||
if (this.circuitState === 'open') {
|
||||
// Check if timeout has passed
|
||||
if (Date.now() - this.circuitOpenTime > this.circuitTimeout) {
|
||||
this.circuitState = 'half-open'
|
||||
this.logger.info('Circuit breaker entering half-open state')
|
||||
return false
|
||||
}
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
/**
|
||||
* Open the circuit breaker
|
||||
*/
|
||||
private openCircuit(): void {
|
||||
if (this.circuitState !== 'open') {
|
||||
this.circuitState = 'open'
|
||||
this.circuitOpenTime = Date.now()
|
||||
this.logger.warn('Circuit breaker opened due to high error rate')
|
||||
|
||||
// Reduce load immediately
|
||||
this.maxConcurrent = Math.max(10, Math.floor(this.maxConcurrent * 0.3))
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Close the circuit breaker
|
||||
*/
|
||||
private closeCircuit(): void {
|
||||
this.circuitState = 'closed'
|
||||
this.circuitFailures = 0
|
||||
this.logger.info('Circuit breaker closed - system recovered')
|
||||
|
||||
// Gradually increase capacity
|
||||
this.maxConcurrent = Math.min(500, Math.floor(this.maxConcurrent * 1.5))
|
||||
}
|
||||
|
||||
/**
|
||||
* Adapt configuration based on metrics
|
||||
*/
|
||||
private adaptIfNeeded(): void {
|
||||
const now = Date.now()
|
||||
if (now - this.lastAdaptation < 5000) { // Adapt every 5 seconds
|
||||
return
|
||||
}
|
||||
|
||||
this.lastAdaptation = now
|
||||
this.updateMetrics()
|
||||
|
||||
// Learn from current patterns
|
||||
this.learnPattern()
|
||||
|
||||
// Adapt based on metrics
|
||||
this.adaptConfiguration()
|
||||
}
|
||||
|
||||
/**
|
||||
* Update current metrics
|
||||
*/
|
||||
private updateMetrics(): void {
|
||||
// Calculate processing rate
|
||||
this.metrics.processingRate = this.completedOps.length > 0
|
||||
? 1000 / (this.completedOps.reduce((a, b) => a + b, 0) / this.completedOps.length)
|
||||
: 0
|
||||
|
||||
// Calculate error rate
|
||||
const totalOps = this.completedOps.length + this.errorOps
|
||||
this.metrics.errorRate = totalOps > 0 ? this.errorOps / totalOps : 0
|
||||
|
||||
// Calculate average latency
|
||||
this.metrics.latency = this.completedOps.length > 0
|
||||
? this.completedOps.reduce((a, b) => a + b, 0) / this.completedOps.length
|
||||
: 0
|
||||
|
||||
// Calculate throughput
|
||||
this.metrics.throughput = this.activeOperations.size + this.metrics.processingRate
|
||||
|
||||
// Reset error counter periodically
|
||||
if (this.completedOps.length > 100) {
|
||||
this.errorOps = Math.floor(this.errorOps * 0.9) // Decay error count
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Learn from current load patterns
|
||||
*/
|
||||
private learnPattern(): void {
|
||||
const currentLoad = this.activeOperations.size + this.queue.length
|
||||
const optimalConcurrency = this.calculateOptimalConcurrency()
|
||||
|
||||
this.patterns.push({
|
||||
timestamp: Date.now(),
|
||||
load: currentLoad,
|
||||
optimal: optimalConcurrency
|
||||
})
|
||||
|
||||
// Keep patterns bounded
|
||||
if (this.patterns.length > 1000) {
|
||||
this.patterns = this.patterns.slice(-500)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Calculate optimal concurrency based on Little's Law
|
||||
*/
|
||||
private calculateOptimalConcurrency(): number {
|
||||
// Little's Law: L = λ * W
|
||||
// L = number of requests in system
|
||||
// λ = arrival rate
|
||||
// W = average time in system
|
||||
|
||||
if (this.metrics.latency === 0 || this.metrics.processingRate === 0) {
|
||||
return this.maxConcurrent // Keep current if no data
|
||||
}
|
||||
|
||||
// Target: Keep latency under target while maximizing throughput
|
||||
const targetConcurrency = Math.ceil(
|
||||
this.metrics.processingRate * (this.config.targetLatency / 1000)
|
||||
)
|
||||
|
||||
// Adjust based on error rate
|
||||
const errorAdjustment = 1 - (this.metrics.errorRate * 2) // Reduce by up to 50% for errors
|
||||
|
||||
// Apply adjustment
|
||||
const adjusted = Math.floor(targetConcurrency * errorAdjustment)
|
||||
|
||||
// Apply bounds
|
||||
return Math.max(10, Math.min(500, adjusted))
|
||||
}
|
||||
|
||||
/**
|
||||
* Adapt configuration based on metrics and patterns
|
||||
*/
|
||||
private adaptConfiguration(): void {
|
||||
const optimal = this.calculateOptimalConcurrency()
|
||||
const current = this.maxConcurrent
|
||||
|
||||
// Smooth adaptation using exponential moving average
|
||||
const newConcurrency = Math.floor(
|
||||
current * (1 - this.config.adaptationRate) +
|
||||
optimal * this.config.adaptationRate
|
||||
)
|
||||
|
||||
// Check if adaptation is needed
|
||||
if (Math.abs(newConcurrency - current) > current * 0.1) { // 10% threshold
|
||||
const oldValue = this.maxConcurrent
|
||||
this.maxConcurrent = newConcurrency
|
||||
|
||||
this.logger.debug('Adapted concurrency', {
|
||||
from: oldValue,
|
||||
to: newConcurrency,
|
||||
metrics: this.metrics
|
||||
})
|
||||
}
|
||||
|
||||
// Adapt queue depth based on throughput
|
||||
if (this.metrics.throughput > 0) {
|
||||
// Allow queue depth to be 10 seconds worth of throughput
|
||||
this.config.maxQueueDepth = Math.max(
|
||||
100,
|
||||
Math.min(10000, Math.floor(this.metrics.throughput * 10))
|
||||
)
|
||||
}
|
||||
|
||||
// Adapt circuit breaker threshold based on error patterns
|
||||
if (this.metrics.errorRate < 0.01 && this.circuitThreshold > 5) {
|
||||
this.circuitThreshold = Math.max(5, this.circuitThreshold - 1)
|
||||
} else if (this.metrics.errorRate > 0.05 && this.circuitThreshold < 20) {
|
||||
this.circuitThreshold = Math.min(20, this.circuitThreshold + 1)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Predict future load based on patterns
|
||||
*/
|
||||
public predictLoad(futureSeconds: number = 60): number {
|
||||
if (this.patterns.length < 10) {
|
||||
return this.maxConcurrent // Not enough data
|
||||
}
|
||||
|
||||
// Simple linear regression on recent patterns
|
||||
const recentPatterns = this.patterns.slice(-50)
|
||||
const n = recentPatterns.length
|
||||
|
||||
// Calculate averages
|
||||
let sumX = 0, sumY = 0, sumXY = 0, sumX2 = 0
|
||||
const startTime = recentPatterns[0].timestamp
|
||||
|
||||
recentPatterns.forEach(p => {
|
||||
const x = (p.timestamp - startTime) / 1000 // Time in seconds
|
||||
const y = p.load
|
||||
sumX += x
|
||||
sumY += y
|
||||
sumXY += x * y
|
||||
sumX2 += x * x
|
||||
})
|
||||
|
||||
// Calculate slope and intercept
|
||||
const slope = (n * sumXY - sumX * sumY) / (n * sumX2 - sumX * sumX)
|
||||
const intercept = (sumY - slope * sumX) / n
|
||||
|
||||
// Predict future load
|
||||
const currentTime = (Date.now() - startTime) / 1000
|
||||
const predictedLoad = intercept + slope * (currentTime + futureSeconds)
|
||||
|
||||
return Math.max(0, Math.min(this.config.maxQueueDepth, Math.floor(predictedLoad)))
|
||||
}
|
||||
|
||||
/**
|
||||
* Get current configuration and metrics
|
||||
*/
|
||||
public getStatus(): {
|
||||
config: BackpressureConfig
|
||||
metrics: BackpressureMetrics
|
||||
circuit: string
|
||||
maxConcurrent: number
|
||||
activeOps: number
|
||||
queueLength: number
|
||||
} {
|
||||
return {
|
||||
config: { ...this.config },
|
||||
metrics: { ...this.metrics },
|
||||
circuit: this.circuitState,
|
||||
maxConcurrent: this.maxConcurrent,
|
||||
activeOps: this.activeOperations.size,
|
||||
queueLength: this.queue.length
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Reset to default state
|
||||
*/
|
||||
public reset(): void {
|
||||
this.queue = []
|
||||
this.activeOperations.clear()
|
||||
this.operationTimes.clear()
|
||||
this.completedOps = []
|
||||
this.errorOps = 0
|
||||
this.patterns = []
|
||||
this.circuitState = 'closed'
|
||||
this.circuitFailures = 0
|
||||
this.maxConcurrent = 100
|
||||
|
||||
this.logger.info('Backpressure system reset to defaults')
|
||||
}
|
||||
}
|
||||
|
||||
// Global singleton instance
|
||||
let globalBackpressure: AdaptiveBackpressure | null = null
|
||||
|
||||
/**
|
||||
* Get the global backpressure instance
|
||||
*/
|
||||
export function getGlobalBackpressure(): AdaptiveBackpressure {
|
||||
if (!globalBackpressure) {
|
||||
globalBackpressure = new AdaptiveBackpressure()
|
||||
}
|
||||
return globalBackpressure
|
||||
}
|
||||
474
src/utils/adaptiveSocketManager.ts
Normal file
474
src/utils/adaptiveSocketManager.ts
Normal file
|
|
@ -0,0 +1,474 @@
|
|||
/**
|
||||
* Adaptive Socket Manager
|
||||
* Automatically manages socket pools and connection settings based on load patterns
|
||||
* Zero-configuration approach that learns and adapts to workload characteristics
|
||||
*/
|
||||
|
||||
import { Agent as HttpsAgent } from 'https'
|
||||
import { NodeHttpHandler } from '@smithy/node-http-handler'
|
||||
import { createModuleLogger } from './logger.js'
|
||||
|
||||
interface LoadMetrics {
|
||||
requestsPerSecond: number
|
||||
pendingRequests: number
|
||||
socketUtilization: number
|
||||
errorRate: number
|
||||
latencyP50: number
|
||||
latencyP95: number
|
||||
memoryUsage: number
|
||||
}
|
||||
|
||||
interface AdaptiveConfig {
|
||||
maxSockets: number
|
||||
maxFreeSockets: number
|
||||
keepAliveTimeout: number
|
||||
connectionTimeout: number
|
||||
socketTimeout: number
|
||||
batchSize: number
|
||||
}
|
||||
|
||||
/**
|
||||
* Adaptive Socket Manager that automatically scales based on load patterns
|
||||
*/
|
||||
export class AdaptiveSocketManager {
|
||||
private logger = createModuleLogger('AdaptiveSocketManager')
|
||||
|
||||
// Current configuration
|
||||
private config: AdaptiveConfig = {
|
||||
maxSockets: 100, // Start conservative
|
||||
maxFreeSockets: 20,
|
||||
keepAliveTimeout: 60000,
|
||||
connectionTimeout: 10000,
|
||||
socketTimeout: 60000,
|
||||
batchSize: 10
|
||||
}
|
||||
|
||||
// Performance tracking
|
||||
private metrics: LoadMetrics = {
|
||||
requestsPerSecond: 0,
|
||||
pendingRequests: 0,
|
||||
socketUtilization: 0,
|
||||
errorRate: 0,
|
||||
latencyP50: 0,
|
||||
latencyP95: 0,
|
||||
memoryUsage: 0
|
||||
}
|
||||
|
||||
// Historical data for learning
|
||||
private history: LoadMetrics[] = []
|
||||
private maxHistorySize = 100
|
||||
|
||||
// Adaptation state
|
||||
private lastAdaptationTime = 0
|
||||
private adaptationInterval = 5000 // Check every 5 seconds
|
||||
private consecutiveHighLoad = 0
|
||||
private consecutiveLowLoad = 0
|
||||
|
||||
// Request tracking
|
||||
private requestStartTimes = new Map<string, number>()
|
||||
private requestLatencies: number[] = []
|
||||
private errorCount = 0
|
||||
private successCount = 0
|
||||
private lastMetricReset = Date.now()
|
||||
|
||||
// Socket pool instances
|
||||
private currentAgent: HttpsAgent | null = null
|
||||
private currentHandler: NodeHttpHandler | null = null
|
||||
|
||||
/**
|
||||
* Get or create an optimized HTTP handler
|
||||
*/
|
||||
public getHttpHandler(): NodeHttpHandler {
|
||||
// Adapt configuration if needed
|
||||
this.adaptIfNeeded()
|
||||
|
||||
// Create new handler if configuration changed
|
||||
if (!this.currentHandler || this.shouldRecreateHandler()) {
|
||||
this.currentAgent = new HttpsAgent({
|
||||
keepAlive: true,
|
||||
maxSockets: this.config.maxSockets,
|
||||
maxFreeSockets: this.config.maxFreeSockets,
|
||||
timeout: this.config.keepAliveTimeout,
|
||||
scheduling: 'fifo' // Fair scheduling for high-volume scenarios
|
||||
})
|
||||
|
||||
this.currentHandler = new NodeHttpHandler({
|
||||
httpsAgent: this.currentAgent,
|
||||
connectionTimeout: this.config.connectionTimeout,
|
||||
socketTimeout: this.config.socketTimeout
|
||||
})
|
||||
|
||||
this.logger.debug('Created new HTTP handler with config:', this.config)
|
||||
}
|
||||
|
||||
return this.currentHandler
|
||||
}
|
||||
|
||||
/**
|
||||
* Get current batch size recommendation
|
||||
*/
|
||||
public getBatchSize(): number {
|
||||
this.adaptIfNeeded()
|
||||
return this.config.batchSize
|
||||
}
|
||||
|
||||
/**
|
||||
* Track request start
|
||||
*/
|
||||
public trackRequestStart(requestId: string): void {
|
||||
this.requestStartTimes.set(requestId, Date.now())
|
||||
this.metrics.pendingRequests++
|
||||
}
|
||||
|
||||
/**
|
||||
* Track request completion
|
||||
*/
|
||||
public trackRequestComplete(requestId: string, success: boolean): void {
|
||||
const startTime = this.requestStartTimes.get(requestId)
|
||||
if (startTime) {
|
||||
const latency = Date.now() - startTime
|
||||
this.requestLatencies.push(latency)
|
||||
this.requestStartTimes.delete(requestId)
|
||||
|
||||
// Keep latency array bounded
|
||||
if (this.requestLatencies.length > 1000) {
|
||||
this.requestLatencies = this.requestLatencies.slice(-500)
|
||||
}
|
||||
}
|
||||
|
||||
if (success) {
|
||||
this.successCount++
|
||||
} else {
|
||||
this.errorCount++
|
||||
}
|
||||
|
||||
this.metrics.pendingRequests = Math.max(0, this.metrics.pendingRequests - 1)
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if we should adapt configuration
|
||||
*/
|
||||
private adaptIfNeeded(): void {
|
||||
const now = Date.now()
|
||||
if (now - this.lastAdaptationTime < this.adaptationInterval) {
|
||||
return
|
||||
}
|
||||
|
||||
this.lastAdaptationTime = now
|
||||
this.updateMetrics()
|
||||
this.analyzeAndAdapt()
|
||||
}
|
||||
|
||||
/**
|
||||
* Update current metrics
|
||||
*/
|
||||
private updateMetrics(): void {
|
||||
const now = Date.now()
|
||||
const timeSinceReset = (now - this.lastMetricReset) / 1000
|
||||
|
||||
// Calculate requests per second
|
||||
const totalRequests = this.successCount + this.errorCount
|
||||
this.metrics.requestsPerSecond = timeSinceReset > 0
|
||||
? totalRequests / timeSinceReset
|
||||
: 0
|
||||
|
||||
// Calculate error rate
|
||||
this.metrics.errorRate = totalRequests > 0
|
||||
? this.errorCount / totalRequests
|
||||
: 0
|
||||
|
||||
// Calculate latency percentiles
|
||||
if (this.requestLatencies.length > 0) {
|
||||
const sorted = [...this.requestLatencies].sort((a, b) => a - b)
|
||||
const p50Index = Math.floor(sorted.length * 0.5)
|
||||
const p95Index = Math.floor(sorted.length * 0.95)
|
||||
this.metrics.latencyP50 = sorted[p50Index] || 0
|
||||
this.metrics.latencyP95 = sorted[p95Index] || 0
|
||||
}
|
||||
|
||||
// Calculate socket utilization
|
||||
this.metrics.socketUtilization = this.metrics.pendingRequests / this.config.maxSockets
|
||||
|
||||
// Memory usage
|
||||
if (typeof process !== 'undefined' && process.memoryUsage) {
|
||||
const memUsage = process.memoryUsage()
|
||||
this.metrics.memoryUsage = memUsage.heapUsed / memUsage.heapTotal
|
||||
}
|
||||
|
||||
// Add to history
|
||||
this.history.push({ ...this.metrics })
|
||||
if (this.history.length > this.maxHistorySize) {
|
||||
this.history.shift()
|
||||
}
|
||||
|
||||
// Reset counters periodically
|
||||
if (timeSinceReset > 60) {
|
||||
this.lastMetricReset = now
|
||||
this.successCount = 0
|
||||
this.errorCount = 0
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Analyze metrics and adapt configuration
|
||||
*/
|
||||
private analyzeAndAdapt(): void {
|
||||
const wasConfig = { ...this.config }
|
||||
|
||||
// Detect high load conditions
|
||||
const isHighLoad = this.detectHighLoad()
|
||||
const isLowLoad = this.detectLowLoad()
|
||||
const hasErrors = this.metrics.errorRate > 0.01 // More than 1% errors
|
||||
|
||||
if (isHighLoad) {
|
||||
this.consecutiveHighLoad++
|
||||
this.consecutiveLowLoad = 0
|
||||
|
||||
if (this.consecutiveHighLoad >= 2) { // Wait for 2 consecutive high load readings
|
||||
this.scaleUp()
|
||||
}
|
||||
} else if (isLowLoad) {
|
||||
this.consecutiveLowLoad++
|
||||
this.consecutiveHighLoad = 0
|
||||
|
||||
if (this.consecutiveLowLoad >= 6) { // Wait longer before scaling down
|
||||
this.scaleDown()
|
||||
}
|
||||
} else {
|
||||
// Reset counters if load is normal
|
||||
this.consecutiveHighLoad = Math.max(0, this.consecutiveHighLoad - 1)
|
||||
this.consecutiveLowLoad = Math.max(0, this.consecutiveLowLoad - 1)
|
||||
}
|
||||
|
||||
// Handle error conditions
|
||||
if (hasErrors) {
|
||||
this.handleErrors()
|
||||
}
|
||||
|
||||
// Log significant changes
|
||||
if (JSON.stringify(wasConfig) !== JSON.stringify(this.config)) {
|
||||
this.logger.info('Adapted configuration', {
|
||||
from: wasConfig,
|
||||
to: this.config,
|
||||
metrics: this.metrics
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Detect high load conditions
|
||||
*/
|
||||
private detectHighLoad(): boolean {
|
||||
return (
|
||||
this.metrics.socketUtilization > 0.7 || // Sockets heavily used
|
||||
this.metrics.pendingRequests > this.config.maxSockets * 0.8 || // Many pending requests
|
||||
this.metrics.latencyP95 > 5000 || // High latency
|
||||
this.metrics.requestsPerSecond > 100 // High request rate
|
||||
)
|
||||
}
|
||||
|
||||
/**
|
||||
* Detect low load conditions
|
||||
*/
|
||||
private detectLowLoad(): boolean {
|
||||
return (
|
||||
this.metrics.socketUtilization < 0.2 && // Sockets barely used
|
||||
this.metrics.pendingRequests < 5 && // Few pending requests
|
||||
this.metrics.latencyP95 < 1000 && // Low latency
|
||||
this.metrics.requestsPerSecond < 10 && // Low request rate
|
||||
this.metrics.memoryUsage < 0.5 // Low memory usage
|
||||
)
|
||||
}
|
||||
|
||||
/**
|
||||
* Scale up resources for high load
|
||||
*/
|
||||
private scaleUp(): void {
|
||||
// Increase socket limits progressively
|
||||
const scaleFactor = this.metrics.errorRate > 0.05 ? 1.5 : 2.0 // Scale more aggressively if no errors
|
||||
|
||||
this.config.maxSockets = Math.min(
|
||||
2000, // Hard limit to prevent resource exhaustion
|
||||
Math.ceil(this.config.maxSockets * scaleFactor)
|
||||
)
|
||||
|
||||
this.config.maxFreeSockets = Math.min(
|
||||
200,
|
||||
Math.ceil(this.config.maxSockets * 0.1) // Keep 10% as free sockets
|
||||
)
|
||||
|
||||
// Increase batch size for better throughput
|
||||
this.config.batchSize = Math.min(
|
||||
100,
|
||||
Math.ceil(this.config.batchSize * 1.5)
|
||||
)
|
||||
|
||||
// Adjust timeouts for high load
|
||||
this.config.keepAliveTimeout = 120000 // Keep connections alive longer
|
||||
this.config.connectionTimeout = 15000 // Allow more time for connections
|
||||
this.config.socketTimeout = 90000 // Allow more time for responses
|
||||
|
||||
this.logger.debug('Scaled up for high load', {
|
||||
sockets: this.config.maxSockets,
|
||||
batchSize: this.config.batchSize
|
||||
})
|
||||
}
|
||||
|
||||
/**
|
||||
* Scale down resources for low load
|
||||
*/
|
||||
private scaleDown(): void {
|
||||
// Only scale down if memory pressure is low
|
||||
if (this.metrics.memoryUsage > 0.7) {
|
||||
return
|
||||
}
|
||||
|
||||
// Decrease socket limits conservatively
|
||||
this.config.maxSockets = Math.max(
|
||||
50, // Minimum sockets
|
||||
Math.floor(this.config.maxSockets * 0.7)
|
||||
)
|
||||
|
||||
this.config.maxFreeSockets = Math.max(
|
||||
10,
|
||||
Math.floor(this.config.maxSockets * 0.2) // Keep 20% as free sockets
|
||||
)
|
||||
|
||||
// Decrease batch size
|
||||
this.config.batchSize = Math.max(
|
||||
5,
|
||||
Math.floor(this.config.batchSize * 0.7)
|
||||
)
|
||||
|
||||
// Adjust timeouts for low load
|
||||
this.config.keepAliveTimeout = 60000
|
||||
this.config.connectionTimeout = 10000
|
||||
this.config.socketTimeout = 60000
|
||||
|
||||
this.logger.debug('Scaled down for low load', {
|
||||
sockets: this.config.maxSockets,
|
||||
batchSize: this.config.batchSize
|
||||
})
|
||||
}
|
||||
|
||||
/**
|
||||
* Handle error conditions by adjusting configuration
|
||||
*/
|
||||
private handleErrors(): void {
|
||||
const errorRate = this.metrics.errorRate
|
||||
|
||||
if (errorRate > 0.1) { // More than 10% errors
|
||||
// Severe errors - back off aggressively
|
||||
this.config.maxSockets = Math.max(50, Math.floor(this.config.maxSockets * 0.5))
|
||||
this.config.batchSize = Math.max(1, Math.floor(this.config.batchSize * 0.3))
|
||||
this.config.connectionTimeout = Math.min(30000, this.config.connectionTimeout * 2)
|
||||
this.config.socketTimeout = Math.min(120000, this.config.socketTimeout * 2)
|
||||
|
||||
this.logger.warn('High error rate detected, backing off', {
|
||||
errorRate,
|
||||
newConfig: this.config
|
||||
})
|
||||
} else if (errorRate > 0.05) { // More than 5% errors
|
||||
// Moderate errors - reduce load slightly
|
||||
this.config.batchSize = Math.max(1, Math.floor(this.config.batchSize * 0.7))
|
||||
this.config.connectionTimeout = Math.min(20000, this.config.connectionTimeout * 1.2)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if we should recreate the handler
|
||||
*/
|
||||
private shouldRecreateHandler(): boolean {
|
||||
if (!this.currentAgent) return true
|
||||
|
||||
// Recreate if socket configuration changed significantly
|
||||
const currentMaxSockets = (this.currentAgent as any).maxSockets
|
||||
const socketsDiff = Math.abs(currentMaxSockets - this.config.maxSockets)
|
||||
|
||||
return socketsDiff > currentMaxSockets * 0.5 // 50% change threshold
|
||||
}
|
||||
|
||||
/**
|
||||
* Get current configuration (for monitoring)
|
||||
*/
|
||||
public getConfig(): Readonly<AdaptiveConfig> {
|
||||
return { ...this.config }
|
||||
}
|
||||
|
||||
/**
|
||||
* Get current metrics (for monitoring)
|
||||
*/
|
||||
public getMetrics(): Readonly<LoadMetrics> {
|
||||
return { ...this.metrics }
|
||||
}
|
||||
|
||||
/**
|
||||
* Predict optimal configuration based on historical data
|
||||
*/
|
||||
public predictOptimalConfig(): AdaptiveConfig {
|
||||
if (this.history.length < 10) {
|
||||
return this.config // Not enough data to predict
|
||||
}
|
||||
|
||||
// Analyze recent history
|
||||
const recentHistory = this.history.slice(-20)
|
||||
const avgRPS = recentHistory.reduce((sum, m) => sum + m.requestsPerSecond, 0) / recentHistory.length
|
||||
const maxRPS = Math.max(...recentHistory.map(m => m.requestsPerSecond))
|
||||
const avgLatency = recentHistory.reduce((sum, m) => sum + m.latencyP95, 0) / recentHistory.length
|
||||
|
||||
// Predict optimal socket count based on request patterns
|
||||
const optimalSockets = Math.min(2000, Math.max(50, Math.ceil(maxRPS * 2)))
|
||||
|
||||
// Predict optimal batch size based on latency
|
||||
const optimalBatchSize = avgLatency < 1000 ? 50 : avgLatency < 3000 ? 20 : 10
|
||||
|
||||
return {
|
||||
maxSockets: optimalSockets,
|
||||
maxFreeSockets: Math.ceil(optimalSockets * 0.15),
|
||||
keepAliveTimeout: avgRPS > 50 ? 120000 : 60000,
|
||||
connectionTimeout: avgLatency > 3000 ? 20000 : 10000,
|
||||
socketTimeout: avgLatency > 3000 ? 90000 : 60000,
|
||||
batchSize: optimalBatchSize
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Reset to default configuration
|
||||
*/
|
||||
public reset(): void {
|
||||
this.config = {
|
||||
maxSockets: 100,
|
||||
maxFreeSockets: 20,
|
||||
keepAliveTimeout: 60000,
|
||||
connectionTimeout: 10000,
|
||||
socketTimeout: 60000,
|
||||
batchSize: 10
|
||||
}
|
||||
|
||||
this.consecutiveHighLoad = 0
|
||||
this.consecutiveLowLoad = 0
|
||||
this.history = []
|
||||
this.requestLatencies = []
|
||||
this.errorCount = 0
|
||||
this.successCount = 0
|
||||
|
||||
// Force recreation of handler
|
||||
this.currentAgent = null
|
||||
this.currentHandler = null
|
||||
|
||||
this.logger.info('Reset to default configuration')
|
||||
}
|
||||
}
|
||||
|
||||
// Global singleton instance
|
||||
let globalSocketManager: AdaptiveSocketManager | null = null
|
||||
|
||||
/**
|
||||
* Get the global socket manager instance
|
||||
*/
|
||||
export function getGlobalSocketManager(): AdaptiveSocketManager {
|
||||
if (!globalSocketManager) {
|
||||
globalSocketManager = new AdaptiveSocketManager()
|
||||
}
|
||||
return globalSocketManager
|
||||
}
|
||||
474
src/utils/autoConfiguration.ts
Normal file
474
src/utils/autoConfiguration.ts
Normal file
|
|
@ -0,0 +1,474 @@
|
|||
/**
|
||||
* Automatic Configuration System for Brainy Vector Database
|
||||
* Detects environment, resources, and data patterns to provide optimal settings
|
||||
*/
|
||||
|
||||
import { isBrowser, isNode, isThreadingAvailable } from './environment.js'
|
||||
|
||||
export interface AutoConfigResult {
|
||||
// Environment details
|
||||
environment: 'browser' | 'nodejs' | 'serverless' | 'unknown'
|
||||
|
||||
// Resource detection
|
||||
availableMemory: number // bytes
|
||||
cpuCores: number
|
||||
threadingAvailable: boolean
|
||||
|
||||
// Storage capabilities
|
||||
persistentStorageAvailable: boolean
|
||||
s3StorageDetected: boolean
|
||||
|
||||
// Recommended configuration
|
||||
recommendedConfig: {
|
||||
expectedDatasetSize: number
|
||||
maxMemoryUsage: number
|
||||
targetSearchLatency: number
|
||||
enablePartitioning: boolean
|
||||
enableCompression: boolean
|
||||
enableDistributedSearch: boolean
|
||||
enablePredictiveCaching: boolean
|
||||
partitionStrategy: 'semantic' | 'hash'
|
||||
maxNodesPerPartition: number
|
||||
semanticClusters: number
|
||||
}
|
||||
|
||||
// Performance optimization flags
|
||||
optimizationFlags: {
|
||||
useMemoryMapping: boolean
|
||||
aggressiveCaching: boolean
|
||||
backgroundOptimization: boolean
|
||||
compressionLevel: 'none' | 'light' | 'aggressive'
|
||||
}
|
||||
}
|
||||
|
||||
export interface DatasetAnalysis {
|
||||
estimatedSize: number
|
||||
vectorDimension?: number
|
||||
growthRate?: number // vectors per second
|
||||
accessPatterns?: 'read-heavy' | 'write-heavy' | 'balanced'
|
||||
}
|
||||
|
||||
/**
|
||||
* Automatic configuration system that detects environment and optimizes settings
|
||||
*/
|
||||
export class AutoConfiguration {
|
||||
private static instance: AutoConfiguration
|
||||
private cachedConfig: AutoConfigResult | null = null
|
||||
private datasetStats: DatasetAnalysis = { estimatedSize: 0 }
|
||||
|
||||
private constructor() {}
|
||||
|
||||
public static getInstance(): AutoConfiguration {
|
||||
if (!AutoConfiguration.instance) {
|
||||
AutoConfiguration.instance = new AutoConfiguration()
|
||||
}
|
||||
return AutoConfiguration.instance
|
||||
}
|
||||
|
||||
/**
|
||||
* Detect environment and generate optimal configuration
|
||||
*/
|
||||
public async detectAndConfigure(hints?: {
|
||||
expectedDataSize?: number
|
||||
s3Available?: boolean
|
||||
memoryBudget?: number
|
||||
}): Promise<AutoConfigResult> {
|
||||
if (this.cachedConfig && !hints) {
|
||||
return this.cachedConfig
|
||||
}
|
||||
|
||||
const environment = this.detectEnvironment()
|
||||
const resources = await this.detectResources()
|
||||
const storage = await this.detectStorageCapabilities(hints?.s3Available)
|
||||
|
||||
const config: AutoConfigResult = {
|
||||
environment,
|
||||
...resources,
|
||||
...storage,
|
||||
recommendedConfig: this.generateRecommendedConfig(environment, resources, hints),
|
||||
optimizationFlags: this.generateOptimizationFlags(environment, resources)
|
||||
}
|
||||
|
||||
this.cachedConfig = config
|
||||
return config
|
||||
}
|
||||
|
||||
/**
|
||||
* Update configuration based on runtime dataset analysis
|
||||
*/
|
||||
public async adaptToDataset(analysis: DatasetAnalysis): Promise<AutoConfigResult> {
|
||||
this.datasetStats = analysis
|
||||
|
||||
// Regenerate configuration with dataset insights
|
||||
const currentConfig = await this.detectAndConfigure()
|
||||
const adaptedConfig = this.adaptConfigurationToData(currentConfig, analysis)
|
||||
|
||||
this.cachedConfig = adaptedConfig
|
||||
return adaptedConfig
|
||||
}
|
||||
|
||||
/**
|
||||
* Learn from performance metrics and adjust configuration
|
||||
*/
|
||||
public async learnFromPerformance(metrics: {
|
||||
averageSearchTime: number
|
||||
memoryUsage: number
|
||||
cacheHitRate: number
|
||||
errorRate: number
|
||||
}): Promise<Partial<AutoConfigResult['recommendedConfig']>> {
|
||||
const adjustments: Partial<AutoConfigResult['recommendedConfig']> = {}
|
||||
|
||||
// Learn from search performance
|
||||
if (metrics.averageSearchTime > 200) {
|
||||
// Too slow - optimize for speed
|
||||
adjustments.enableDistributedSearch = true
|
||||
adjustments.maxNodesPerPartition = Math.max(10000, (this.cachedConfig?.recommendedConfig.maxNodesPerPartition || 50000) * 0.8)
|
||||
} else if (metrics.averageSearchTime < 50) {
|
||||
// Very fast - can optimize for quality
|
||||
adjustments.maxNodesPerPartition = Math.min(100000, (this.cachedConfig?.recommendedConfig.maxNodesPerPartition || 50000) * 1.2)
|
||||
}
|
||||
|
||||
// Learn from memory usage
|
||||
if (metrics.memoryUsage > (this.cachedConfig?.recommendedConfig.maxMemoryUsage || 0) * 0.9) {
|
||||
// High memory usage - enable compression
|
||||
adjustments.enableCompression = true
|
||||
}
|
||||
|
||||
// Learn from cache performance
|
||||
if (metrics.cacheHitRate < 0.7) {
|
||||
// Poor cache performance - enable predictive caching
|
||||
adjustments.enablePredictiveCaching = true
|
||||
}
|
||||
|
||||
// Update cached config with learned adjustments
|
||||
if (this.cachedConfig) {
|
||||
this.cachedConfig.recommendedConfig = {
|
||||
...this.cachedConfig.recommendedConfig,
|
||||
...adjustments
|
||||
}
|
||||
}
|
||||
|
||||
return adjustments
|
||||
}
|
||||
|
||||
/**
|
||||
* Get minimal configuration for quick setup
|
||||
*/
|
||||
public async getQuickSetupConfig(scenario: 'small' | 'medium' | 'large' | 'enterprise'): Promise<{
|
||||
expectedDatasetSize: number
|
||||
maxMemoryUsage: number
|
||||
targetSearchLatency: number
|
||||
s3Required: boolean
|
||||
}> {
|
||||
const environment = this.detectEnvironment()
|
||||
const resources = await this.detectResources()
|
||||
|
||||
switch (scenario) {
|
||||
case 'small':
|
||||
return {
|
||||
expectedDatasetSize: 10000,
|
||||
maxMemoryUsage: Math.min(resources.availableMemory * 0.3, 1024 * 1024 * 1024), // 1GB max
|
||||
targetSearchLatency: 100,
|
||||
s3Required: false
|
||||
}
|
||||
|
||||
case 'medium':
|
||||
return {
|
||||
expectedDatasetSize: 100000,
|
||||
maxMemoryUsage: Math.min(resources.availableMemory * 0.5, 4 * 1024 * 1024 * 1024), // 4GB max
|
||||
targetSearchLatency: 150,
|
||||
s3Required: environment === 'serverless'
|
||||
}
|
||||
|
||||
case 'large':
|
||||
return {
|
||||
expectedDatasetSize: 1000000,
|
||||
maxMemoryUsage: Math.min(resources.availableMemory * 0.7, 8 * 1024 * 1024 * 1024), // 8GB max
|
||||
targetSearchLatency: 200,
|
||||
s3Required: true
|
||||
}
|
||||
|
||||
case 'enterprise':
|
||||
return {
|
||||
expectedDatasetSize: 10000000,
|
||||
maxMemoryUsage: Math.min(resources.availableMemory * 0.8, 32 * 1024 * 1024 * 1024), // 32GB max
|
||||
targetSearchLatency: 300,
|
||||
s3Required: true
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Detect the current runtime environment
|
||||
*/
|
||||
private detectEnvironment(): 'browser' | 'nodejs' | 'serverless' | 'unknown' {
|
||||
if (isBrowser()) {
|
||||
return 'browser'
|
||||
}
|
||||
|
||||
if (isNode()) {
|
||||
// Check for serverless environment indicators
|
||||
if (process.env.AWS_LAMBDA_FUNCTION_NAME ||
|
||||
process.env.VERCEL ||
|
||||
process.env.NETLIFY ||
|
||||
process.env.CLOUDFLARE_WORKERS) {
|
||||
return 'serverless'
|
||||
}
|
||||
return 'nodejs'
|
||||
}
|
||||
|
||||
return 'unknown'
|
||||
}
|
||||
|
||||
/**
|
||||
* Detect available system resources
|
||||
*/
|
||||
private async detectResources(): Promise<{
|
||||
availableMemory: number
|
||||
cpuCores: number
|
||||
threadingAvailable: boolean
|
||||
}> {
|
||||
let availableMemory = 2 * 1024 * 1024 * 1024 // Default 2GB
|
||||
let cpuCores = 4 // Default 4 cores
|
||||
|
||||
// Browser memory detection
|
||||
if (isBrowser()) {
|
||||
// @ts-ignore - navigator.deviceMemory is experimental
|
||||
if (navigator.deviceMemory) {
|
||||
// @ts-ignore
|
||||
availableMemory = navigator.deviceMemory * 1024 * 1024 * 1024 * 0.3 // Use 30% of device memory
|
||||
} else {
|
||||
availableMemory = 512 * 1024 * 1024 // Conservative 512MB for browsers
|
||||
}
|
||||
|
||||
cpuCores = navigator.hardwareConcurrency || 4
|
||||
}
|
||||
|
||||
// Node.js memory detection
|
||||
if (isNode()) {
|
||||
try {
|
||||
const os = await import('os')
|
||||
availableMemory = os.totalmem() * 0.7 // Use 70% of total memory
|
||||
cpuCores = os.cpus().length
|
||||
} catch (error) {
|
||||
// Fallback to defaults
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
availableMemory,
|
||||
cpuCores,
|
||||
threadingAvailable: isThreadingAvailable()
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Detect available storage capabilities
|
||||
*/
|
||||
private async detectStorageCapabilities(s3Hint?: boolean): Promise<{
|
||||
persistentStorageAvailable: boolean
|
||||
s3StorageDetected: boolean
|
||||
}> {
|
||||
let persistentStorageAvailable = false
|
||||
let s3StorageDetected = s3Hint || false
|
||||
|
||||
if (isBrowser()) {
|
||||
// Check for OPFS support
|
||||
persistentStorageAvailable = 'navigator' in globalThis &&
|
||||
'storage' in navigator &&
|
||||
'getDirectory' in navigator.storage
|
||||
}
|
||||
|
||||
if (isNode()) {
|
||||
persistentStorageAvailable = true // Always available in Node.js
|
||||
|
||||
// Check for AWS SDK or S3 environment variables
|
||||
s3StorageDetected = s3Hint ||
|
||||
!!(process.env.AWS_ACCESS_KEY_ID && process.env.AWS_SECRET_ACCESS_KEY) ||
|
||||
!!(process.env.S3_BUCKET_NAME)
|
||||
}
|
||||
|
||||
return {
|
||||
persistentStorageAvailable,
|
||||
s3StorageDetected
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate recommended configuration based on detected environment and resources
|
||||
*/
|
||||
private generateRecommendedConfig(
|
||||
environment: string,
|
||||
resources: { availableMemory: number; cpuCores: number },
|
||||
hints?: { expectedDataSize?: number; memoryBudget?: number }
|
||||
): AutoConfigResult['recommendedConfig'] {
|
||||
const datasetSize = hints?.expectedDataSize || this.estimateDatasetSize()
|
||||
const memoryBudget = hints?.memoryBudget || Math.floor(resources.availableMemory * 0.6)
|
||||
|
||||
// Base configuration
|
||||
let config = {
|
||||
expectedDatasetSize: datasetSize,
|
||||
maxMemoryUsage: memoryBudget,
|
||||
targetSearchLatency: 150,
|
||||
enablePartitioning: datasetSize > 25000,
|
||||
enableCompression: environment === 'browser' || memoryBudget < 2 * 1024 * 1024 * 1024,
|
||||
enableDistributedSearch: resources.cpuCores > 2 && datasetSize > 50000,
|
||||
enablePredictiveCaching: true,
|
||||
partitionStrategy: 'semantic' as const,
|
||||
maxNodesPerPartition: 50000,
|
||||
semanticClusters: 8
|
||||
}
|
||||
|
||||
// Environment-specific adjustments
|
||||
switch (environment) {
|
||||
case 'browser':
|
||||
config = {
|
||||
...config,
|
||||
maxMemoryUsage: Math.min(memoryBudget, 1024 * 1024 * 1024), // Cap at 1GB
|
||||
targetSearchLatency: 200, // More lenient for browsers
|
||||
enableCompression: true, // Always enable for browsers
|
||||
maxNodesPerPartition: 25000, // Smaller partitions
|
||||
semanticClusters: 4 // Fewer clusters to save memory
|
||||
}
|
||||
break
|
||||
|
||||
case 'serverless':
|
||||
config = {
|
||||
...config,
|
||||
targetSearchLatency: 500, // Account for cold starts
|
||||
enablePredictiveCaching: false, // Avoid background processes
|
||||
maxNodesPerPartition: 30000 // Moderate partition size
|
||||
}
|
||||
break
|
||||
|
||||
case 'nodejs':
|
||||
config = {
|
||||
...config,
|
||||
targetSearchLatency: 100, // Aggressive for Node.js
|
||||
maxNodesPerPartition: Math.min(100000, Math.floor(datasetSize / 10)), // Larger partitions
|
||||
semanticClusters: Math.min(16, Math.max(4, Math.floor(datasetSize / 50000))) // Scale clusters with data
|
||||
}
|
||||
break
|
||||
}
|
||||
|
||||
// Dataset size adjustments
|
||||
if (datasetSize > 1000000) {
|
||||
config.semanticClusters = Math.min(32, Math.floor(datasetSize / 100000))
|
||||
config.maxNodesPerPartition = 100000
|
||||
} else if (datasetSize < 10000) {
|
||||
config.enablePartitioning = false
|
||||
config.enableDistributedSearch = false
|
||||
config.partitionStrategy = 'semantic' // Keep semantic but disable partitioning
|
||||
}
|
||||
|
||||
return config
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate optimization flags based on environment and resources
|
||||
*/
|
||||
private generateOptimizationFlags(
|
||||
environment: string,
|
||||
resources: { availableMemory: number; cpuCores: number }
|
||||
): AutoConfigResult['optimizationFlags'] {
|
||||
return {
|
||||
useMemoryMapping: environment === 'nodejs' && resources.availableMemory > 4 * 1024 * 1024 * 1024,
|
||||
aggressiveCaching: resources.availableMemory > 2 * 1024 * 1024 * 1024,
|
||||
backgroundOptimization: environment !== 'serverless' && resources.cpuCores > 2,
|
||||
compressionLevel: resources.availableMemory < 1024 * 1024 * 1024 ? 'aggressive' :
|
||||
resources.availableMemory < 4 * 1024 * 1024 * 1024 ? 'light' : 'none'
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Adapt configuration based on actual dataset analysis
|
||||
*/
|
||||
private adaptConfigurationToData(
|
||||
baseConfig: AutoConfigResult,
|
||||
analysis: DatasetAnalysis
|
||||
): AutoConfigResult {
|
||||
const updatedConfig = { ...baseConfig }
|
||||
|
||||
// Adjust based on actual dataset size
|
||||
if (analysis.estimatedSize !== baseConfig.recommendedConfig.expectedDatasetSize) {
|
||||
const sizeRatio = analysis.estimatedSize / baseConfig.recommendedConfig.expectedDatasetSize
|
||||
|
||||
updatedConfig.recommendedConfig.expectedDatasetSize = analysis.estimatedSize
|
||||
|
||||
// Scale partition size with dataset
|
||||
if (sizeRatio > 2) {
|
||||
updatedConfig.recommendedConfig.maxNodesPerPartition = Math.min(
|
||||
100000,
|
||||
Math.floor(updatedConfig.recommendedConfig.maxNodesPerPartition * 1.5)
|
||||
)
|
||||
updatedConfig.recommendedConfig.semanticClusters = Math.min(
|
||||
32,
|
||||
Math.floor(updatedConfig.recommendedConfig.semanticClusters * 1.5)
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
// Adjust based on vector dimension
|
||||
if (analysis.vectorDimension) {
|
||||
if (analysis.vectorDimension > 1024) {
|
||||
// High-dimensional vectors - optimize for compression
|
||||
updatedConfig.recommendedConfig.enableCompression = true
|
||||
updatedConfig.optimizationFlags.compressionLevel = 'aggressive'
|
||||
}
|
||||
}
|
||||
|
||||
// Adjust based on access patterns
|
||||
if (analysis.accessPatterns === 'read-heavy') {
|
||||
updatedConfig.recommendedConfig.enablePredictiveCaching = true
|
||||
updatedConfig.optimizationFlags.aggressiveCaching = true
|
||||
} else if (analysis.accessPatterns === 'write-heavy') {
|
||||
updatedConfig.recommendedConfig.enablePredictiveCaching = false
|
||||
updatedConfig.optimizationFlags.backgroundOptimization = false
|
||||
}
|
||||
|
||||
return updatedConfig
|
||||
}
|
||||
|
||||
/**
|
||||
* Estimate dataset size if not provided
|
||||
*/
|
||||
private estimateDatasetSize(): number {
|
||||
// Start with conservative estimate
|
||||
const environment = this.detectEnvironment()
|
||||
|
||||
switch (environment) {
|
||||
case 'browser': return 10000
|
||||
case 'serverless': return 50000
|
||||
case 'nodejs': return 100000
|
||||
default: return 25000
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Reset cached configuration (for testing or manual refresh)
|
||||
*/
|
||||
public resetCache(): void {
|
||||
this.cachedConfig = null
|
||||
this.datasetStats = { estimatedSize: 0 }
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Convenience function for quick auto-configuration
|
||||
*/
|
||||
export async function autoConfigureBrainy(hints?: {
|
||||
expectedDataSize?: number
|
||||
s3Available?: boolean
|
||||
memoryBudget?: number
|
||||
}): Promise<AutoConfigResult> {
|
||||
const autoConfig = AutoConfiguration.getInstance()
|
||||
return autoConfig.detectAndConfigure(hints)
|
||||
}
|
||||
|
||||
/**
|
||||
* Get quick setup configuration for common scenarios
|
||||
*/
|
||||
export async function getQuickSetup(scenario: 'small' | 'medium' | 'large' | 'enterprise') {
|
||||
const autoConfig = AutoConfiguration.getInstance()
|
||||
return autoConfig.getQuickSetupConfig(scenario)
|
||||
}
|
||||
330
src/utils/cacheAutoConfig.ts
Normal file
330
src/utils/cacheAutoConfig.ts
Normal file
|
|
@ -0,0 +1,330 @@
|
|||
/**
|
||||
* Intelligent cache auto-configuration system
|
||||
* Adapts cache settings based on environment, usage patterns, and storage type
|
||||
*/
|
||||
|
||||
import { SearchCacheConfig } from './searchCache.js'
|
||||
import { BrainyDataConfig } from '../brainyData.js'
|
||||
|
||||
export interface CacheUsageStats {
|
||||
totalQueries: number
|
||||
repeatQueries: number
|
||||
avgQueryTime: number
|
||||
memoryPressure: number
|
||||
storageType: 'memory' | 'opfs' | 's3' | 'filesystem'
|
||||
isDistributed: boolean
|
||||
changeFrequency: number // changes per minute
|
||||
readWriteRatio: number // reads / writes
|
||||
}
|
||||
|
||||
export interface AutoConfigResult {
|
||||
cacheConfig: SearchCacheConfig
|
||||
realtimeConfig: NonNullable<BrainyDataConfig['realtimeUpdates']>
|
||||
reasoning: string[]
|
||||
}
|
||||
|
||||
export class CacheAutoConfigurator {
|
||||
private stats: CacheUsageStats = {
|
||||
totalQueries: 0,
|
||||
repeatQueries: 0,
|
||||
avgQueryTime: 50,
|
||||
memoryPressure: 0,
|
||||
storageType: 'memory',
|
||||
isDistributed: false,
|
||||
changeFrequency: 0,
|
||||
readWriteRatio: 10,
|
||||
}
|
||||
|
||||
private configHistory: AutoConfigResult[] = []
|
||||
private lastOptimization = 0
|
||||
|
||||
/**
|
||||
* Auto-detect optimal cache configuration based on current conditions
|
||||
*/
|
||||
public autoDetectOptimalConfig(
|
||||
storageConfig?: BrainyDataConfig['storage'],
|
||||
currentStats?: Partial<CacheUsageStats>
|
||||
): AutoConfigResult {
|
||||
// Update stats with current information
|
||||
if (currentStats) {
|
||||
this.stats = { ...this.stats, ...currentStats }
|
||||
}
|
||||
|
||||
// Detect environment characteristics
|
||||
this.detectEnvironment(storageConfig)
|
||||
|
||||
// Generate optimal configuration
|
||||
const result = this.generateOptimalConfig()
|
||||
|
||||
// Store for learning
|
||||
this.configHistory.push(result)
|
||||
this.lastOptimization = Date.now()
|
||||
|
||||
return result
|
||||
}
|
||||
|
||||
/**
|
||||
* Dynamically adjust configuration based on runtime performance
|
||||
*/
|
||||
public adaptConfiguration(
|
||||
currentConfig: SearchCacheConfig,
|
||||
performanceMetrics: {
|
||||
hitRate: number
|
||||
avgResponseTime: number
|
||||
memoryUsage: number
|
||||
externalChangesDetected: number
|
||||
timeSinceLastChange: number
|
||||
}
|
||||
): AutoConfigResult | null {
|
||||
const reasoning: string[] = []
|
||||
let needsUpdate = false
|
||||
|
||||
// Check if we should update (don't over-optimize)
|
||||
if (Date.now() - this.lastOptimization < 60000) {
|
||||
return null // Wait at least 1 minute between optimizations
|
||||
}
|
||||
|
||||
// Analyze performance patterns
|
||||
const adaptations: Partial<SearchCacheConfig> = {}
|
||||
|
||||
// Low hit rate → adjust cache size or TTL
|
||||
if (performanceMetrics.hitRate < 0.3) {
|
||||
if (performanceMetrics.externalChangesDetected > 5) {
|
||||
// Too many external changes → shorter TTL
|
||||
adaptations.maxAge = Math.max(60000, currentConfig.maxAge! * 0.7)
|
||||
reasoning.push('Reduced cache TTL due to frequent external changes')
|
||||
needsUpdate = true
|
||||
} else {
|
||||
// Expand cache size for better hit rate
|
||||
adaptations.maxSize = Math.min(500, (currentConfig.maxSize || 100) * 1.5)
|
||||
reasoning.push('Increased cache size due to low hit rate')
|
||||
needsUpdate = true
|
||||
}
|
||||
}
|
||||
|
||||
// High hit rate but slow responses → might need cache warming
|
||||
if (performanceMetrics.hitRate > 0.8 && performanceMetrics.avgResponseTime > 100) {
|
||||
reasoning.push('High hit rate but slow responses - consider cache warming')
|
||||
}
|
||||
|
||||
// Memory pressure → reduce cache size
|
||||
if (performanceMetrics.memoryUsage > 100 * 1024 * 1024) { // 100MB
|
||||
adaptations.maxSize = Math.max(20, (currentConfig.maxSize || 100) * 0.7)
|
||||
reasoning.push('Reduced cache size due to memory pressure')
|
||||
needsUpdate = true
|
||||
}
|
||||
|
||||
// Recent external changes → adaptive TTL
|
||||
if (performanceMetrics.timeSinceLastChange < 30000) { // 30 seconds
|
||||
adaptations.maxAge = Math.max(30000, currentConfig.maxAge! * 0.8)
|
||||
reasoning.push('Shortened TTL due to recent external changes')
|
||||
needsUpdate = true
|
||||
}
|
||||
|
||||
if (!needsUpdate) {
|
||||
return null
|
||||
}
|
||||
|
||||
const newCacheConfig: SearchCacheConfig = {
|
||||
...currentConfig,
|
||||
...adaptations
|
||||
}
|
||||
|
||||
const newRealtimeConfig = this.calculateRealtimeConfig()
|
||||
|
||||
return {
|
||||
cacheConfig: newCacheConfig,
|
||||
realtimeConfig: newRealtimeConfig,
|
||||
reasoning
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get recommended configuration for specific use case
|
||||
*/
|
||||
public getRecommendedConfig(useCase: 'high-consistency' | 'balanced' | 'performance-first'): AutoConfigResult {
|
||||
const configs = {
|
||||
'high-consistency': {
|
||||
cache: { maxAge: 120000, maxSize: 50 },
|
||||
realtime: { interval: 15000, enabled: true },
|
||||
reasoning: ['Optimized for data consistency and real-time updates']
|
||||
},
|
||||
'balanced': {
|
||||
cache: { maxAge: 300000, maxSize: 100 },
|
||||
realtime: { interval: 30000, enabled: true },
|
||||
reasoning: ['Balanced performance and consistency']
|
||||
},
|
||||
'performance-first': {
|
||||
cache: { maxAge: 600000, maxSize: 200 },
|
||||
realtime: { interval: 60000, enabled: true },
|
||||
reasoning: ['Optimized for maximum cache performance']
|
||||
}
|
||||
}
|
||||
|
||||
const config = configs[useCase]
|
||||
return {
|
||||
cacheConfig: {
|
||||
enabled: true,
|
||||
...config.cache
|
||||
},
|
||||
realtimeConfig: {
|
||||
updateIndex: true,
|
||||
updateStatistics: true,
|
||||
...config.realtime
|
||||
},
|
||||
reasoning: config.reasoning
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Learn from usage patterns and improve recommendations
|
||||
*/
|
||||
public learnFromUsage(usageData: {
|
||||
queryPatterns: string[]
|
||||
responseTime: number
|
||||
cacheHits: number
|
||||
totalQueries: number
|
||||
dataChanges: number
|
||||
timeWindow: number
|
||||
}): void {
|
||||
// Update internal stats for better future recommendations
|
||||
this.stats.totalQueries += usageData.totalQueries
|
||||
this.stats.repeatQueries += usageData.cacheHits
|
||||
this.stats.avgQueryTime = (this.stats.avgQueryTime + usageData.responseTime) / 2
|
||||
this.stats.changeFrequency = usageData.dataChanges / (usageData.timeWindow / 60000)
|
||||
|
||||
// Calculate read/write ratio
|
||||
const writes = usageData.dataChanges
|
||||
const reads = usageData.totalQueries
|
||||
this.stats.readWriteRatio = reads > 0 ? reads / Math.max(writes, 1) : 10
|
||||
}
|
||||
|
||||
private detectEnvironment(storageConfig?: BrainyDataConfig['storage']): void {
|
||||
// Detect storage type
|
||||
if (storageConfig?.s3Storage || storageConfig?.customS3Storage) {
|
||||
this.stats.storageType = 's3'
|
||||
this.stats.isDistributed = true
|
||||
} else if (storageConfig?.forceFileSystemStorage) {
|
||||
this.stats.storageType = 'filesystem'
|
||||
} else if (storageConfig?.forceMemoryStorage) {
|
||||
this.stats.storageType = 'memory'
|
||||
} else {
|
||||
// Auto-detect browser vs Node.js
|
||||
this.stats.storageType = typeof window !== 'undefined' ? 'opfs' : 'filesystem'
|
||||
}
|
||||
|
||||
// Detect distributed mode indicators
|
||||
this.stats.isDistributed = this.stats.isDistributed ||
|
||||
Boolean(storageConfig?.s3Storage || storageConfig?.customS3Storage)
|
||||
}
|
||||
|
||||
private generateOptimalConfig(): AutoConfigResult {
|
||||
const reasoning: string[] = []
|
||||
|
||||
// Base configuration
|
||||
let cacheConfig: SearchCacheConfig = {
|
||||
enabled: true,
|
||||
maxSize: 100,
|
||||
maxAge: 300000, // 5 minutes
|
||||
hitCountWeight: 0.3
|
||||
}
|
||||
|
||||
let realtimeConfig = {
|
||||
enabled: false,
|
||||
interval: 60000,
|
||||
updateIndex: true,
|
||||
updateStatistics: true
|
||||
}
|
||||
|
||||
// Adjust for storage type
|
||||
if (this.stats.storageType === 's3' || this.stats.isDistributed) {
|
||||
cacheConfig.maxAge = 180000 // 3 minutes for distributed
|
||||
realtimeConfig.enabled = true
|
||||
realtimeConfig.interval = 30000 // 30 seconds
|
||||
reasoning.push('Distributed storage detected - enabled real-time updates')
|
||||
reasoning.push('Reduced cache TTL for distributed consistency')
|
||||
}
|
||||
|
||||
// Adjust for read/write patterns
|
||||
if (this.stats.readWriteRatio > 20) {
|
||||
// Read-heavy workload
|
||||
cacheConfig.maxSize = Math.min(300, (cacheConfig.maxSize || 100) * 2)
|
||||
cacheConfig.maxAge = Math.min(900000, (cacheConfig.maxAge || 300000) * 1.5) // Up to 15 minutes
|
||||
reasoning.push('Read-heavy workload detected - increased cache size and TTL')
|
||||
} else if (this.stats.readWriteRatio < 5) {
|
||||
// Write-heavy workload
|
||||
cacheConfig.maxSize = Math.max(50, (cacheConfig.maxSize || 100) * 0.7)
|
||||
cacheConfig.maxAge = Math.max(60000, (cacheConfig.maxAge || 300000) * 0.6)
|
||||
reasoning.push('Write-heavy workload detected - reduced cache size and TTL')
|
||||
}
|
||||
|
||||
// Adjust for change frequency
|
||||
if (this.stats.changeFrequency > 10) { // More than 10 changes per minute
|
||||
realtimeConfig.interval = Math.max(10000, realtimeConfig.interval * 0.5)
|
||||
cacheConfig.maxAge = Math.max(30000, (cacheConfig.maxAge || 300000) * 0.5)
|
||||
reasoning.push('High change frequency detected - increased update frequency')
|
||||
}
|
||||
|
||||
// Memory constraints
|
||||
if (this.detectMemoryConstraints()) {
|
||||
cacheConfig.maxSize = Math.max(20, (cacheConfig.maxSize || 100) * 0.6)
|
||||
reasoning.push('Memory constraints detected - reduced cache size')
|
||||
}
|
||||
|
||||
// Performance optimization
|
||||
if (this.stats.avgQueryTime > 200) {
|
||||
cacheConfig.maxSize = Math.min(500, (cacheConfig.maxSize || 100) * 1.5)
|
||||
reasoning.push('Slow queries detected - increased cache size')
|
||||
}
|
||||
|
||||
return {
|
||||
cacheConfig,
|
||||
realtimeConfig,
|
||||
reasoning
|
||||
}
|
||||
}
|
||||
|
||||
private calculateRealtimeConfig() {
|
||||
return {
|
||||
enabled: this.stats.isDistributed || this.stats.changeFrequency > 1,
|
||||
interval: this.stats.isDistributed ? 30000 : 60000,
|
||||
updateIndex: true,
|
||||
updateStatistics: true
|
||||
}
|
||||
}
|
||||
|
||||
private detectMemoryConstraints(): boolean {
|
||||
// Simple heuristic for memory constraints
|
||||
try {
|
||||
if (typeof performance !== 'undefined' && 'memory' in performance) {
|
||||
const memInfo = (performance as any).memory
|
||||
return memInfo.usedJSHeapSize > memInfo.jsHeapSizeLimit * 0.8
|
||||
}
|
||||
} catch (e) {
|
||||
// Ignore errors
|
||||
}
|
||||
|
||||
// Default assumption for constrained environments
|
||||
return false
|
||||
}
|
||||
|
||||
/**
|
||||
* Get human-readable explanation of current configuration
|
||||
*/
|
||||
public getConfigExplanation(config: AutoConfigResult): string {
|
||||
const lines = [
|
||||
'🤖 Brainy Auto-Configuration:',
|
||||
'',
|
||||
`📊 Cache: ${config.cacheConfig.maxSize} queries, ${config.cacheConfig.maxAge! / 1000}s TTL`,
|
||||
`🔄 Updates: ${config.realtimeConfig.enabled ? `Every ${(config.realtimeConfig.interval || 30000) / 1000}s` : 'Disabled'}`,
|
||||
'',
|
||||
'🎯 Optimizations applied:'
|
||||
]
|
||||
|
||||
config.reasoning.forEach(reason => {
|
||||
lines.push(` • ${reason}`)
|
||||
})
|
||||
|
||||
return lines.join('\n')
|
||||
}
|
||||
}
|
||||
47
src/utils/crypto.ts
Normal file
47
src/utils/crypto.ts
Normal file
|
|
@ -0,0 +1,47 @@
|
|||
/**
|
||||
* Cross-platform crypto utilities
|
||||
* Provides hashing functions that work in both Node.js and browser environments
|
||||
*/
|
||||
|
||||
/**
|
||||
* Simple string hash function that works in all environments
|
||||
* Uses djb2 algorithm - fast and good distribution
|
||||
* @param str - String to hash
|
||||
* @returns Positive integer hash
|
||||
*/
|
||||
export function hashString(str: string): number {
|
||||
let hash = 5381
|
||||
for (let i = 0; i < str.length; i++) {
|
||||
const char = str.charCodeAt(i)
|
||||
hash = ((hash << 5) + hash) + char // hash * 33 + char
|
||||
}
|
||||
// Ensure positive number
|
||||
return Math.abs(hash)
|
||||
}
|
||||
|
||||
/**
|
||||
* Alternative: FNV-1a hash algorithm
|
||||
* Good distribution and fast
|
||||
* @param str - String to hash
|
||||
* @returns Positive integer hash
|
||||
*/
|
||||
export function fnv1aHash(str: string): number {
|
||||
let hash = 2166136261
|
||||
for (let i = 0; i < str.length; i++) {
|
||||
hash ^= str.charCodeAt(i)
|
||||
hash = (hash * 16777619) >>> 0
|
||||
}
|
||||
return hash
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate a deterministic hash for partitioning
|
||||
* Uses the most appropriate algorithm for the environment
|
||||
* @param input - Input string to hash
|
||||
* @returns Positive integer hash suitable for modulo operations
|
||||
*/
|
||||
export function getPartitionHash(input: string): number {
|
||||
// Use djb2 by default as it's fast and has good distribution
|
||||
// This ensures consistent partitioning across all environments
|
||||
return hashString(input)
|
||||
}
|
||||
215
src/utils/distance.ts
Normal file
215
src/utils/distance.ts
Normal file
|
|
@ -0,0 +1,215 @@
|
|||
/**
|
||||
* Distance functions for vector similarity calculations
|
||||
* Optimized pure JavaScript implementations using enhanced array methods
|
||||
* Faster than GPU for small vectors (384 dims) due to no transfer overhead
|
||||
*/
|
||||
|
||||
import { DistanceFunction, Vector } from '../coreTypes.js'
|
||||
import { executeInThread } from './workerUtils.js'
|
||||
import { isThreadingAvailable } from './environment.js'
|
||||
|
||||
/**
|
||||
* Calculates the Euclidean distance between two vectors
|
||||
* Lower values indicate higher similarity
|
||||
* Optimized using array methods for Node.js 23.11+
|
||||
*/
|
||||
export const euclideanDistance: DistanceFunction = (
|
||||
a: Vector,
|
||||
b: Vector
|
||||
): number => {
|
||||
if (a.length !== b.length) {
|
||||
throw new Error('Vectors must have the same dimensions')
|
||||
}
|
||||
|
||||
// Use array.reduce for better performance in Node.js 23.11+
|
||||
const sum = a.reduce((acc, val, i) => {
|
||||
const diff = val - b[i]
|
||||
return acc + diff * diff
|
||||
}, 0)
|
||||
|
||||
return Math.sqrt(sum)
|
||||
}
|
||||
|
||||
/**
|
||||
* Calculates the cosine distance between two vectors
|
||||
* Lower values indicate higher similarity
|
||||
* Range: 0 (identical) to 2 (opposite)
|
||||
* Optimized using array methods for Node.js 23.11+
|
||||
*/
|
||||
export const cosineDistance: DistanceFunction = (
|
||||
a: Vector,
|
||||
b: Vector
|
||||
): number => {
|
||||
if (a.length !== b.length) {
|
||||
throw new Error('Vectors must have the same dimensions')
|
||||
}
|
||||
|
||||
// Use array.reduce to calculate all values in a single pass
|
||||
const { dotProduct, normA, normB } = a.reduce(
|
||||
(acc, val, i) => {
|
||||
return {
|
||||
dotProduct: acc.dotProduct + val * b[i],
|
||||
normA: acc.normA + val * val,
|
||||
normB: acc.normB + b[i] * b[i]
|
||||
}
|
||||
},
|
||||
{ dotProduct: 0, normA: 0, normB: 0 }
|
||||
)
|
||||
|
||||
if (normA === 0 || normB === 0) {
|
||||
return 2 // Maximum distance for zero vectors
|
||||
}
|
||||
|
||||
const similarity = dotProduct / (Math.sqrt(normA) * Math.sqrt(normB))
|
||||
// Convert cosine similarity (-1 to 1) to distance (0 to 2)
|
||||
return 1 - similarity
|
||||
}
|
||||
|
||||
/**
|
||||
* Calculates the Manhattan (L1) distance between two vectors
|
||||
* Lower values indicate higher similarity
|
||||
* Optimized using array methods for Node.js 23.11+
|
||||
*/
|
||||
export const manhattanDistance: DistanceFunction = (
|
||||
a: Vector,
|
||||
b: Vector
|
||||
): number => {
|
||||
if (a.length !== b.length) {
|
||||
throw new Error('Vectors must have the same dimensions')
|
||||
}
|
||||
|
||||
// Use array.reduce for better performance in Node.js 23.11+
|
||||
return a.reduce((sum, val, i) => sum + Math.abs(val - b[i]), 0)
|
||||
}
|
||||
|
||||
/**
|
||||
* Calculates the dot product similarity between two vectors
|
||||
* Higher values indicate higher similarity
|
||||
* Converted to a distance metric (lower is better)
|
||||
* Optimized using array methods for Node.js 23.11+
|
||||
*/
|
||||
export const dotProductDistance: DistanceFunction = (
|
||||
a: Vector,
|
||||
b: Vector
|
||||
): number => {
|
||||
if (a.length !== b.length) {
|
||||
throw new Error('Vectors must have the same dimensions')
|
||||
}
|
||||
|
||||
// Use array.reduce for better performance in Node.js 23.11+
|
||||
const dotProduct = a.reduce((sum, val, i) => sum + val * b[i], 0)
|
||||
|
||||
// Convert to a distance metric (lower is better)
|
||||
return -dotProduct
|
||||
}
|
||||
|
||||
/**
|
||||
* Batch distance calculation using optimized JavaScript
|
||||
* More efficient than GPU for small vectors due to no memory transfer overhead
|
||||
*
|
||||
* @param queryVector The query vector to compare against all vectors
|
||||
* @param vectors Array of vectors to compare against
|
||||
* @param distanceFunction The distance function to use
|
||||
* @returns Promise resolving to array of distances
|
||||
*/
|
||||
export async function calculateDistancesBatch(
|
||||
queryVector: Vector,
|
||||
vectors: Vector[],
|
||||
distanceFunction: DistanceFunction = euclideanDistance
|
||||
): Promise<number[]> {
|
||||
// For small batches, use the standard distance function
|
||||
if (vectors.length < 10) {
|
||||
return vectors.map((vector) => distanceFunction(queryVector, vector))
|
||||
}
|
||||
|
||||
try {
|
||||
// Function for optimized batch distance calculation
|
||||
const distanceCalculator = (args: {
|
||||
queryVector: Vector
|
||||
vectors: Vector[]
|
||||
distanceFnString: string
|
||||
}) => {
|
||||
const { queryVector, vectors, distanceFnString } = args
|
||||
|
||||
// Optimized JavaScript implementations for different distance functions
|
||||
let distances: number[]
|
||||
|
||||
if (distanceFnString.includes('euclideanDistance')) {
|
||||
// Euclidean distance: sqrt(sum((a - b)^2))
|
||||
distances = vectors.map((vector) => {
|
||||
let sum = 0
|
||||
for (let i = 0; i < queryVector.length; i++) {
|
||||
const diff = queryVector[i] - vector[i]
|
||||
sum += diff * diff
|
||||
}
|
||||
return Math.sqrt(sum)
|
||||
})
|
||||
} else if (distanceFnString.includes('cosineDistance')) {
|
||||
// Cosine distance: 1 - (a·b / (||a|| * ||b||))
|
||||
distances = vectors.map((vector) => {
|
||||
let dotProduct = 0
|
||||
let queryNorm = 0
|
||||
let vectorNorm = 0
|
||||
|
||||
for (let i = 0; i < queryVector.length; i++) {
|
||||
dotProduct += queryVector[i] * vector[i]
|
||||
queryNorm += queryVector[i] * queryVector[i]
|
||||
vectorNorm += vector[i] * vector[i]
|
||||
}
|
||||
|
||||
queryNorm = Math.sqrt(queryNorm)
|
||||
vectorNorm = Math.sqrt(vectorNorm)
|
||||
|
||||
if (queryNorm === 0 || vectorNorm === 0) {
|
||||
return 1 // Maximum distance for zero vectors
|
||||
}
|
||||
|
||||
const cosineSimilarity = dotProduct / (queryNorm * vectorNorm)
|
||||
return 1 - cosineSimilarity
|
||||
})
|
||||
} else if (distanceFnString.includes('manhattanDistance')) {
|
||||
// Manhattan distance: sum(|a - b|)
|
||||
distances = vectors.map((vector) => {
|
||||
let sum = 0
|
||||
for (let i = 0; i < queryVector.length; i++) {
|
||||
sum += Math.abs(queryVector[i] - vector[i])
|
||||
}
|
||||
return sum
|
||||
})
|
||||
} else if (distanceFnString.includes('dotProductDistance')) {
|
||||
// Dot product distance: -sum(a * b)
|
||||
distances = vectors.map((vector) => {
|
||||
let dotProduct = 0
|
||||
for (let i = 0; i < queryVector.length; i++) {
|
||||
dotProduct += queryVector[i] * vector[i]
|
||||
}
|
||||
return -dotProduct
|
||||
})
|
||||
} else {
|
||||
// For unknown distance functions, use the provided function
|
||||
const distanceFunction = new Function(
|
||||
'return ' + distanceFnString
|
||||
)() as DistanceFunction
|
||||
|
||||
distances = vectors.map((vector) =>
|
||||
distanceFunction(queryVector, vector)
|
||||
)
|
||||
}
|
||||
|
||||
return { distances }
|
||||
}
|
||||
|
||||
// Use the optimized distance calculator
|
||||
const result = distanceCalculator({
|
||||
queryVector,
|
||||
vectors,
|
||||
distanceFnString: distanceFunction.toString()
|
||||
})
|
||||
|
||||
return result.distances
|
||||
} catch (error) {
|
||||
// If anything fails, fall back to the standard distance function
|
||||
console.error('Batch distance calculation failed:', error)
|
||||
return vectors.map((vector) => distanceFunction(queryVector, vector))
|
||||
}
|
||||
}
|
||||
502
src/utils/embedding.ts
Normal file
502
src/utils/embedding.ts
Normal file
|
|
@ -0,0 +1,502 @@
|
|||
/**
|
||||
* Embedding functions for converting data to vectors using Transformers.js
|
||||
* Complete rewrite to eliminate TensorFlow.js and use ONNX-based models
|
||||
*/
|
||||
|
||||
import { EmbeddingFunction, EmbeddingModel, Vector } from '../coreTypes.js'
|
||||
import { executeInThread } from './workerUtils.js'
|
||||
import { isBrowser } from './environment.js'
|
||||
import { ModelManager } from '../embeddings/model-manager.js'
|
||||
// @ts-ignore - Transformers.js is now the primary embedding library
|
||||
import { pipeline, env } from '@huggingface/transformers'
|
||||
|
||||
// CRITICAL: Disable ONNX memory arena to prevent 4-8GB allocation
|
||||
// This is needed for BOTH production and testing - reduces memory by 50-75%
|
||||
if (typeof process !== 'undefined' && process.env) {
|
||||
process.env.ORT_DISABLE_MEMORY_ARENA = '1'
|
||||
process.env.ORT_DISABLE_MEMORY_PATTERN = '1'
|
||||
// Also limit ONNX thread count for more predictable memory usage
|
||||
process.env.ORT_INTRA_OP_NUM_THREADS = '2'
|
||||
process.env.ORT_INTER_OP_NUM_THREADS = '2'
|
||||
}
|
||||
|
||||
/**
|
||||
* Detect the best available GPU device for the current environment
|
||||
*/
|
||||
export async function detectBestDevice(): Promise<'cpu' | 'webgpu' | 'cuda'> {
|
||||
// Browser environment - check for WebGPU support
|
||||
if (isBrowser()) {
|
||||
if (typeof navigator !== 'undefined' && 'gpu' in navigator) {
|
||||
try {
|
||||
const adapter = await (navigator as any).gpu?.requestAdapter()
|
||||
if (adapter) {
|
||||
return 'webgpu'
|
||||
}
|
||||
} catch (error) {
|
||||
// WebGPU not available or failed to initialize
|
||||
}
|
||||
}
|
||||
return 'cpu'
|
||||
}
|
||||
|
||||
// Node.js environment - check for CUDA support
|
||||
try {
|
||||
// Check if ONNX Runtime GPU packages are available
|
||||
// This is a simple heuristic - in production you might want more sophisticated detection
|
||||
const hasGpu = process.env.CUDA_VISIBLE_DEVICES !== undefined ||
|
||||
process.env.ONNXRUNTIME_GPU_ENABLED === 'true'
|
||||
return hasGpu ? 'cuda' : 'cpu'
|
||||
} catch (error) {
|
||||
return 'cpu'
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolve device string to actual device configuration
|
||||
*/
|
||||
export async function resolveDevice(device: string = 'auto'): Promise<string> {
|
||||
if (device === 'auto') {
|
||||
return await detectBestDevice()
|
||||
}
|
||||
|
||||
// Map 'gpu' to appropriate GPU type for current environment
|
||||
if (device === 'gpu') {
|
||||
const detected = await detectBestDevice()
|
||||
return detected === 'cpu' ? 'cpu' : detected
|
||||
}
|
||||
|
||||
return device
|
||||
}
|
||||
|
||||
/**
|
||||
* Transformers.js Sentence Encoder embedding model
|
||||
* Uses ONNX Runtime for fast, offline embeddings with smaller models
|
||||
* Default model: all-MiniLM-L6-v2 (384 dimensions, ~90MB)
|
||||
*/
|
||||
export interface TransformerEmbeddingOptions {
|
||||
/** Model name/path to use - defaults to all-MiniLM-L6-v2 */
|
||||
model?: string
|
||||
/** Whether to enable verbose logging */
|
||||
verbose?: boolean
|
||||
/** Custom cache directory for models */
|
||||
cacheDir?: string
|
||||
/** Force local files only (no downloads) */
|
||||
localFilesOnly?: boolean
|
||||
/** Quantization setting (fp32, fp16, q8, q4) */
|
||||
dtype?: 'fp32' | 'fp16' | 'q8' | 'q4'
|
||||
/** Device to run inference on - 'auto' detects best available */
|
||||
device?: 'auto' | 'cpu' | 'webgpu' | 'cuda' | 'gpu'
|
||||
}
|
||||
|
||||
export class TransformerEmbedding implements EmbeddingModel {
|
||||
private extractor: any = null
|
||||
private initialized = false
|
||||
private verbose: boolean = true
|
||||
private options: Required<TransformerEmbeddingOptions>
|
||||
|
||||
/**
|
||||
* Create a new TransformerEmbedding instance
|
||||
*/
|
||||
constructor(options: TransformerEmbeddingOptions = {}) {
|
||||
this.verbose = options.verbose !== undefined ? options.verbose : true
|
||||
|
||||
// PRODUCTION-READY MODEL CONFIGURATION
|
||||
// Priority order: explicit option > environment variable > smart default
|
||||
|
||||
let localFilesOnly: boolean
|
||||
|
||||
if (options.localFilesOnly !== undefined) {
|
||||
// 1. Explicit option takes highest priority
|
||||
localFilesOnly = options.localFilesOnly
|
||||
} else if (process.env.BRAINY_ALLOW_REMOTE_MODELS !== undefined) {
|
||||
// 2. Environment variable override
|
||||
localFilesOnly = process.env.BRAINY_ALLOW_REMOTE_MODELS !== 'true'
|
||||
} else if (process.env.NODE_ENV === 'development') {
|
||||
// 3. Development mode allows remote models
|
||||
localFilesOnly = false
|
||||
} else if (isBrowser()) {
|
||||
// 4. Browser defaults to allowing remote models
|
||||
localFilesOnly = false
|
||||
} else {
|
||||
// 5. Node.js production: try local first, but allow remote as fallback
|
||||
// This is the NEW production-friendly default
|
||||
localFilesOnly = false
|
||||
}
|
||||
|
||||
this.options = {
|
||||
model: options.model || 'Xenova/all-MiniLM-L6-v2',
|
||||
verbose: this.verbose,
|
||||
cacheDir: options.cacheDir || './models',
|
||||
localFilesOnly: localFilesOnly,
|
||||
dtype: options.dtype || 'q8', // Changed from fp32 to q8 for 75% memory reduction
|
||||
device: options.device || 'auto'
|
||||
}
|
||||
|
||||
if (this.verbose) {
|
||||
this.logger('log', `Embedding config: localFilesOnly=${localFilesOnly}, model=${this.options.model}, cacheDir=${this.options.cacheDir}`)
|
||||
}
|
||||
|
||||
// Configure transformers.js environment
|
||||
if (!isBrowser()) {
|
||||
// Set cache directory for Node.js
|
||||
env.cacheDir = this.options.cacheDir
|
||||
// Prioritize local models for offline operation
|
||||
env.allowRemoteModels = !this.options.localFilesOnly
|
||||
env.allowLocalModels = true
|
||||
} else {
|
||||
// Browser configuration
|
||||
// Allow both local and remote models, but prefer local if available
|
||||
env.allowLocalModels = true
|
||||
env.allowRemoteModels = true
|
||||
// Force the configuration to ensure it's applied
|
||||
if (this.verbose) {
|
||||
this.logger('log', `Browser env config - allowLocalModels: ${env.allowLocalModels}, allowRemoteModels: ${env.allowRemoteModels}, localFilesOnly: ${this.options.localFilesOnly}`)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the default cache directory for models
|
||||
*/
|
||||
private async getDefaultCacheDir(): Promise<string> {
|
||||
if (isBrowser()) {
|
||||
return './models' // Browser default
|
||||
}
|
||||
|
||||
// Check for bundled models in the package
|
||||
const possiblePaths = [
|
||||
// In the installed package
|
||||
'./node_modules/@soulcraft/brainy/models',
|
||||
// In development/source
|
||||
'./models',
|
||||
'./dist/../models',
|
||||
// Alternative locations
|
||||
'../models',
|
||||
'../../models'
|
||||
]
|
||||
|
||||
// Check if we're in Node.js and try to find the bundled models
|
||||
if (typeof process !== 'undefined' && process.versions?.node) {
|
||||
try {
|
||||
// Use dynamic import instead of require for ES modules compatibility
|
||||
const { createRequire } = await import('module')
|
||||
const require = createRequire(import.meta.url)
|
||||
|
||||
const path = require('path')
|
||||
const fs = require('fs')
|
||||
|
||||
// Try to resolve the package location
|
||||
try {
|
||||
const brainyPackagePath = require.resolve('@soulcraft/brainy/package.json')
|
||||
const brainyPackageDir = path.dirname(brainyPackagePath)
|
||||
const bundledModelsPath = path.join(brainyPackageDir, 'models')
|
||||
|
||||
if (fs.existsSync(bundledModelsPath)) {
|
||||
this.logger('log', `Using bundled models from package: ${bundledModelsPath}`)
|
||||
return bundledModelsPath
|
||||
}
|
||||
} catch (e) {
|
||||
// Not installed as package, continue
|
||||
}
|
||||
|
||||
// Try relative paths from current location
|
||||
for (const relativePath of possiblePaths) {
|
||||
const fullPath = path.resolve(relativePath)
|
||||
if (fs.existsSync(fullPath)) {
|
||||
this.logger('log', `Using bundled models from: ${fullPath}`)
|
||||
return fullPath
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
// Silently fall back to default path if module detection fails
|
||||
}
|
||||
}
|
||||
|
||||
// Fallback to default cache directory
|
||||
return './models'
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if we're running in a test environment
|
||||
*/
|
||||
private isTestEnvironment(): boolean {
|
||||
// Always use real implementation - no more mocking
|
||||
return false
|
||||
}
|
||||
|
||||
/**
|
||||
* Log message only if verbose mode is enabled
|
||||
*/
|
||||
private logger(level: 'log' | 'warn' | 'error', message: string, ...args: any[]): void {
|
||||
if (level === 'error' || this.verbose) {
|
||||
console[level](`[TransformerEmbedding] ${message}`, ...args)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Initialize the embedding model
|
||||
*/
|
||||
public async init(): Promise<void> {
|
||||
if (this.initialized) {
|
||||
return
|
||||
}
|
||||
|
||||
// Always use real implementation - no mocking
|
||||
|
||||
try {
|
||||
// Ensure models are available (downloads if needed)
|
||||
const modelManager = ModelManager.getInstance()
|
||||
await modelManager.ensureModels(this.options.model)
|
||||
|
||||
// Resolve device configuration and cache directory
|
||||
const device = await resolveDevice(this.options.device)
|
||||
const cacheDir = this.options.cacheDir === './models'
|
||||
? await this.getDefaultCacheDir()
|
||||
: this.options.cacheDir
|
||||
|
||||
this.logger('log', `Loading Transformer model: ${this.options.model} on device: ${device}`)
|
||||
|
||||
const startTime = Date.now()
|
||||
|
||||
// Load the feature extraction pipeline with memory optimizations
|
||||
const pipelineOptions: any = {
|
||||
cache_dir: cacheDir,
|
||||
local_files_only: isBrowser() ? false : this.options.localFilesOnly,
|
||||
dtype: this.options.dtype || 'q8', // Use quantized model for lower memory
|
||||
// CRITICAL: ONNX memory optimizations
|
||||
session_options: {
|
||||
enableCpuMemArena: false, // Disable pre-allocated memory arena
|
||||
enableMemPattern: false, // Disable memory pattern optimization
|
||||
interOpNumThreads: 2, // Limit thread count
|
||||
intraOpNumThreads: 2, // Limit parallelism
|
||||
graphOptimizationLevel: 'all'
|
||||
}
|
||||
}
|
||||
|
||||
// Add device configuration for GPU acceleration
|
||||
if (device !== 'cpu') {
|
||||
pipelineOptions.device = device
|
||||
this.logger('log', `🚀 GPU acceleration enabled: ${device}`)
|
||||
}
|
||||
|
||||
if (this.verbose) {
|
||||
this.logger('log', `Pipeline options: ${JSON.stringify(pipelineOptions)}`)
|
||||
}
|
||||
|
||||
try {
|
||||
this.extractor = await pipeline('feature-extraction', this.options.model, pipelineOptions)
|
||||
} catch (gpuError: any) {
|
||||
// Fallback to CPU if GPU initialization fails
|
||||
if (device !== 'cpu') {
|
||||
this.logger('warn', `GPU initialization failed, falling back to CPU: ${gpuError?.message || gpuError}`)
|
||||
const cpuOptions = { ...pipelineOptions }
|
||||
delete cpuOptions.device
|
||||
this.extractor = await pipeline('feature-extraction', this.options.model, cpuOptions)
|
||||
} else {
|
||||
// PRODUCTION-READY ERROR HANDLING
|
||||
// If local_files_only is true and models are missing, try enabling remote downloads
|
||||
if (pipelineOptions.local_files_only && gpuError?.message?.includes('local_files_only')) {
|
||||
this.logger('warn', 'Local models not found, attempting remote download as fallback...')
|
||||
|
||||
try {
|
||||
const remoteOptions = { ...pipelineOptions, local_files_only: false }
|
||||
this.extractor = await pipeline('feature-extraction', this.options.model, remoteOptions)
|
||||
this.logger('log', '✅ Successfully downloaded and loaded model from remote')
|
||||
|
||||
// Update the configuration to reflect what actually worked
|
||||
this.options.localFilesOnly = false
|
||||
} catch (remoteError: any) {
|
||||
// Both local and remote failed - throw comprehensive error
|
||||
const errorMsg = `Failed to load embedding model "${this.options.model}". ` +
|
||||
`Local models not found and remote download failed. ` +
|
||||
`To fix: 1) Set BRAINY_ALLOW_REMOTE_MODELS=true, ` +
|
||||
`2) Run "npm run download-models", or ` +
|
||||
`3) Use a custom embedding function.`
|
||||
throw new Error(errorMsg)
|
||||
}
|
||||
} else {
|
||||
throw gpuError
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const loadTime = Date.now() - startTime
|
||||
this.logger('log', `✅ Model loaded successfully in ${loadTime}ms`)
|
||||
|
||||
this.initialized = true
|
||||
} catch (error) {
|
||||
this.logger('error', 'Failed to initialize Transformer embedding model:', error)
|
||||
throw new Error(`Transformer embedding initialization failed: ${error}`)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate embeddings for text data
|
||||
*/
|
||||
public async embed(data: string | string[]): Promise<Vector> {
|
||||
if (!this.initialized) {
|
||||
await this.init()
|
||||
}
|
||||
|
||||
try {
|
||||
// Handle different input types
|
||||
let textToEmbed: string[]
|
||||
|
||||
if (typeof data === 'string') {
|
||||
// Handle empty string case
|
||||
if (data.trim() === '') {
|
||||
// Return a zero vector of 384 dimensions (all-MiniLM-L6-v2 standard)
|
||||
return new Array(384).fill(0)
|
||||
}
|
||||
textToEmbed = [data]
|
||||
} else if (Array.isArray(data) && data.every((item) => typeof item === 'string')) {
|
||||
// Handle empty array or array with empty strings
|
||||
if (data.length === 0 || data.every((item) => item.trim() === '')) {
|
||||
return new Array(384).fill(0)
|
||||
}
|
||||
// Filter out empty strings
|
||||
textToEmbed = data.filter((item) => item.trim() !== '')
|
||||
if (textToEmbed.length === 0) {
|
||||
return new Array(384).fill(0)
|
||||
}
|
||||
} else {
|
||||
throw new Error('TransformerEmbedding only supports string or string[] data')
|
||||
}
|
||||
|
||||
// Ensure the extractor is available
|
||||
if (!this.extractor) {
|
||||
throw new Error('Transformer embedding model is not available')
|
||||
}
|
||||
|
||||
// Generate embeddings with mean pooling and normalization
|
||||
const result = await this.extractor(textToEmbed, {
|
||||
pooling: 'mean',
|
||||
normalize: true
|
||||
})
|
||||
|
||||
// Extract the embedding data
|
||||
let embedding: number[]
|
||||
|
||||
if (textToEmbed.length === 1) {
|
||||
// Single text input - return first embedding
|
||||
embedding = Array.from(result.data.slice(0, 384))
|
||||
} else {
|
||||
// Multiple texts - return first embedding (maintain compatibility)
|
||||
embedding = Array.from(result.data.slice(0, 384))
|
||||
}
|
||||
|
||||
// Validate embedding dimensions
|
||||
if (embedding.length !== 384) {
|
||||
this.logger('warn', `Unexpected embedding dimension: ${embedding.length}, expected 384`)
|
||||
// Pad or truncate to 384 dimensions
|
||||
if (embedding.length < 384) {
|
||||
embedding = [...embedding, ...new Array(384 - embedding.length).fill(0)]
|
||||
} else {
|
||||
embedding = embedding.slice(0, 384)
|
||||
}
|
||||
}
|
||||
|
||||
return embedding
|
||||
} catch (error) {
|
||||
this.logger('error', 'Error generating embeddings:', error)
|
||||
throw new Error(`Failed to generate embeddings: ${error}`)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Dispose of the model and free resources
|
||||
*/
|
||||
public async dispose(): Promise<void> {
|
||||
if (this.extractor && typeof this.extractor.dispose === 'function') {
|
||||
await this.extractor.dispose()
|
||||
}
|
||||
this.extractor = null
|
||||
this.initialized = false
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the dimension of embeddings produced by this model
|
||||
*/
|
||||
public getDimension(): number {
|
||||
return 384
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if the model is initialized
|
||||
*/
|
||||
public isInitialized(): boolean {
|
||||
return this.initialized
|
||||
}
|
||||
}
|
||||
|
||||
// Legacy alias for backward compatibility
|
||||
export const UniversalSentenceEncoder = TransformerEmbedding
|
||||
|
||||
/**
|
||||
* Create a new embedding model instance
|
||||
*/
|
||||
export function createEmbeddingModel(options?: TransformerEmbeddingOptions): EmbeddingModel {
|
||||
return new TransformerEmbedding(options)
|
||||
}
|
||||
|
||||
/**
|
||||
* Default embedding function using the hybrid model manager (BEST OF BOTH WORLDS)
|
||||
* Prevents multiple model loads while supporting multi-source downloading
|
||||
*/
|
||||
export const defaultEmbeddingFunction: EmbeddingFunction = async (data: string | string[]): Promise<Vector> => {
|
||||
const { getHybridEmbeddingFunction } = await import('./hybridModelManager.js')
|
||||
const embeddingFn = await getHybridEmbeddingFunction()
|
||||
return await embeddingFn(data)
|
||||
}
|
||||
|
||||
/**
|
||||
* Create an embedding function with custom options
|
||||
*/
|
||||
export function createEmbeddingFunction(options: TransformerEmbeddingOptions = {}): EmbeddingFunction {
|
||||
const embedder = new TransformerEmbedding(options)
|
||||
|
||||
return async (data: string | string[]): Promise<Vector> => {
|
||||
return await embedder.embed(data)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Batch embedding function for processing multiple texts efficiently
|
||||
*/
|
||||
export async function batchEmbed(
|
||||
texts: string[],
|
||||
options: TransformerEmbeddingOptions = {}
|
||||
): Promise<Vector[]> {
|
||||
const embedder = new TransformerEmbedding(options)
|
||||
await embedder.init()
|
||||
|
||||
const embeddings: Vector[] = []
|
||||
|
||||
// Process in batches for memory efficiency
|
||||
const batchSize = 32
|
||||
for (let i = 0; i < texts.length; i += batchSize) {
|
||||
const batch = texts.slice(i, i + batchSize)
|
||||
|
||||
for (const text of batch) {
|
||||
const embedding = await embedder.embed(text)
|
||||
embeddings.push(embedding)
|
||||
}
|
||||
}
|
||||
|
||||
await embedder.dispose()
|
||||
return embeddings
|
||||
}
|
||||
|
||||
/**
|
||||
* Embedding functions for specific model types
|
||||
*/
|
||||
export const embeddingFunctions = {
|
||||
/** Default lightweight model (all-MiniLM-L6-v2, 384 dimensions) */
|
||||
default: defaultEmbeddingFunction,
|
||||
|
||||
/** Create custom embedding function */
|
||||
create: createEmbeddingFunction,
|
||||
|
||||
/** Batch processing */
|
||||
batch: batchEmbed
|
||||
}
|
||||
186
src/utils/environment.ts
Normal file
186
src/utils/environment.ts
Normal file
|
|
@ -0,0 +1,186 @@
|
|||
/**
|
||||
* Utility functions for environment detection
|
||||
*/
|
||||
|
||||
/**
|
||||
* Check if code is running in a browser environment
|
||||
*/
|
||||
export function isBrowser(): boolean {
|
||||
return typeof window !== 'undefined' && typeof document !== 'undefined'
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if code is running in a Node.js environment
|
||||
*/
|
||||
export function isNode(): boolean {
|
||||
// If browser environment is detected, prioritize it over Node.js
|
||||
// This handles cases like jsdom where both window and process exist
|
||||
if (isBrowser()) {
|
||||
return false
|
||||
}
|
||||
|
||||
return (
|
||||
typeof process !== 'undefined' &&
|
||||
process.versions != null &&
|
||||
process.versions.node != null
|
||||
)
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if code is running in a Web Worker environment
|
||||
*/
|
||||
export function isWebWorker(): boolean {
|
||||
return (
|
||||
typeof self === 'object' &&
|
||||
self.constructor &&
|
||||
self.constructor.name === 'DedicatedWorkerGlobalScope'
|
||||
)
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if Web Workers are available in the current environment
|
||||
*/
|
||||
export function areWebWorkersAvailable(): boolean {
|
||||
return isBrowser() && typeof Worker !== 'undefined'
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if Worker Threads are available in the current environment (Node.js)
|
||||
*/
|
||||
export async function areWorkerThreadsAvailable(): Promise<boolean> {
|
||||
if (!isNode()) return false
|
||||
|
||||
try {
|
||||
// Use dynamic import to avoid errors in browser environments
|
||||
await import('worker_threads')
|
||||
return true
|
||||
} catch (e) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Synchronous version that doesn't actually try to load the module
|
||||
* This is safer in ES module environments
|
||||
*/
|
||||
export function areWorkerThreadsAvailableSync(): boolean {
|
||||
if (!isNode()) return false
|
||||
|
||||
// In Node.js 24.4.0+, worker_threads is always available
|
||||
return parseInt(process.versions.node.split('.')[0]) >= 24
|
||||
}
|
||||
|
||||
/**
|
||||
* Determine if threading is available in the current environment
|
||||
* Returns true if either Web Workers (browser) or Worker Threads (Node.js) are available
|
||||
*/
|
||||
export function isThreadingAvailable(): boolean {
|
||||
return areWebWorkersAvailable() || areWorkerThreadsAvailableSync()
|
||||
}
|
||||
|
||||
/**
|
||||
* Async version of isThreadingAvailable
|
||||
*/
|
||||
export async function isThreadingAvailableAsync(): Promise<boolean> {
|
||||
return areWebWorkersAvailable() || (await areWorkerThreadsAvailable())
|
||||
}
|
||||
|
||||
/**
|
||||
* Auto-detect production environment to minimize logging costs
|
||||
*/
|
||||
export function isProductionEnvironment(): boolean {
|
||||
// Node.js environment detection
|
||||
if (isNode()) {
|
||||
// Check common production environment indicators
|
||||
const nodeEnv = process.env.NODE_ENV?.toLowerCase()
|
||||
if (nodeEnv === 'production' || nodeEnv === 'prod') return true
|
||||
|
||||
// Google Cloud Run detection
|
||||
if (process.env.K_SERVICE || process.env.GOOGLE_CLOUD_PROJECT) return true
|
||||
|
||||
// AWS Lambda detection
|
||||
if (process.env.AWS_LAMBDA_FUNCTION_NAME || process.env.AWS_EXECUTION_ENV) return true
|
||||
|
||||
// Azure Functions detection
|
||||
if (process.env.AZURE_FUNCTIONS_ENVIRONMENT || process.env.WEBSITE_SITE_NAME) return true
|
||||
|
||||
// Vercel detection
|
||||
if (process.env.VERCEL || process.env.VERCEL_ENV === 'production') return true
|
||||
|
||||
// Netlify detection
|
||||
if (process.env.NETLIFY && process.env.CONTEXT === 'production') return true
|
||||
|
||||
// Heroku detection
|
||||
if (process.env.DYNO && process.env.NODE_ENV !== 'development') return true
|
||||
|
||||
// Railway detection
|
||||
if (process.env.RAILWAY_ENVIRONMENT === 'production') return true
|
||||
|
||||
// Fly.io detection
|
||||
if (process.env.FLY_APP_NAME && process.env.FLY_REGION) return true
|
||||
|
||||
// Docker in production (common patterns)
|
||||
if (process.env.DOCKER_ENV === 'production' || process.env.ENVIRONMENT === 'production') return true
|
||||
|
||||
// Generic production indicators
|
||||
if (process.env.PROD === 'true' || process.env.PRODUCTION === 'true') return true
|
||||
}
|
||||
|
||||
// Browser environment - assume development unless explicitly production
|
||||
if (isBrowser()) {
|
||||
// Check for production domain patterns
|
||||
const hostname = window?.location?.hostname
|
||||
if (hostname) {
|
||||
// Avoid logging on production domains
|
||||
if (hostname.includes('.com') || hostname.includes('.org') || hostname.includes('.net')) {
|
||||
return !hostname.includes('localhost') && !hostname.includes('127.0.0.1') && !hostname.includes('dev')
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return false
|
||||
}
|
||||
|
||||
/**
|
||||
* Get appropriate log level based on environment
|
||||
*/
|
||||
export function getLogLevel(): 'silent' | 'error' | 'warn' | 'info' | 'verbose' {
|
||||
// Explicit log level override
|
||||
const explicitLevel = process.env.BRAINY_LOG_LEVEL?.toLowerCase()
|
||||
if (explicitLevel && ['silent', 'error', 'warn', 'info', 'verbose'].includes(explicitLevel)) {
|
||||
return explicitLevel as 'silent' | 'error' | 'warn' | 'info' | 'verbose'
|
||||
}
|
||||
|
||||
// Auto-detect based on environment
|
||||
if (isProductionEnvironment()) {
|
||||
return 'error' // Only log errors in production to minimize costs
|
||||
}
|
||||
|
||||
// Development environments get more verbose logging
|
||||
if (process.env.NODE_ENV === 'development' || process.env.NODE_ENV === 'dev') {
|
||||
return 'verbose'
|
||||
}
|
||||
|
||||
// Test environments should be quieter
|
||||
if (process.env.NODE_ENV === 'test') {
|
||||
return 'warn'
|
||||
}
|
||||
|
||||
// Default to info level
|
||||
return 'info'
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if logging should be enabled for a given level
|
||||
*/
|
||||
export function shouldLog(level: 'error' | 'warn' | 'info' | 'verbose'): boolean {
|
||||
const currentLevel = getLogLevel()
|
||||
|
||||
if (currentLevel === 'silent') return false
|
||||
|
||||
const levels = ['error', 'warn', 'info', 'verbose']
|
||||
const currentIndex = levels.indexOf(currentLevel)
|
||||
const messageIndex = levels.indexOf(level)
|
||||
|
||||
return messageIndex <= currentIndex
|
||||
}
|
||||
114
src/utils/fieldNameTracking.ts
Normal file
114
src/utils/fieldNameTracking.ts
Normal file
|
|
@ -0,0 +1,114 @@
|
|||
/**
|
||||
* Utility functions for tracking and managing field names in JSON documents
|
||||
*/
|
||||
|
||||
/**
|
||||
* Extracts field names from a JSON document
|
||||
* @param jsonObject The JSON object to extract field names from
|
||||
* @param options Configuration options
|
||||
* @returns An array of field paths (e.g., "user.name", "addresses[0].city")
|
||||
*/
|
||||
export function extractFieldNamesFromJson(
|
||||
jsonObject: any,
|
||||
options: {
|
||||
maxDepth?: number
|
||||
currentDepth?: number
|
||||
currentPath?: string
|
||||
fieldNames?: Set<string>
|
||||
} = {}
|
||||
): string[] {
|
||||
const {
|
||||
maxDepth = 5,
|
||||
currentDepth = 0,
|
||||
currentPath = '',
|
||||
fieldNames = new Set<string>()
|
||||
} = options
|
||||
|
||||
if (
|
||||
jsonObject === null ||
|
||||
jsonObject === undefined ||
|
||||
typeof jsonObject !== 'object' ||
|
||||
currentDepth >= maxDepth
|
||||
) {
|
||||
return Array.from(fieldNames)
|
||||
}
|
||||
|
||||
if (Array.isArray(jsonObject)) {
|
||||
// For arrays, we'll just check the first item to avoid explosion of paths
|
||||
if (jsonObject.length > 0) {
|
||||
const arrayPath = currentPath ? `${currentPath}[0]` : '[0]'
|
||||
extractFieldNamesFromJson(jsonObject[0], {
|
||||
maxDepth,
|
||||
currentDepth: currentDepth + 1,
|
||||
currentPath: arrayPath,
|
||||
fieldNames
|
||||
})
|
||||
}
|
||||
} else {
|
||||
// For objects, process each property
|
||||
for (const key of Object.keys(jsonObject)) {
|
||||
const value = jsonObject[key]
|
||||
const fieldPath = currentPath ? `${currentPath}.${key}` : key
|
||||
|
||||
// Add this field path
|
||||
fieldNames.add(fieldPath)
|
||||
|
||||
// Recursively process nested objects
|
||||
if (typeof value === 'object' && value !== null) {
|
||||
extractFieldNamesFromJson(value, {
|
||||
maxDepth,
|
||||
currentDepth: currentDepth + 1,
|
||||
currentPath: fieldPath,
|
||||
fieldNames
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return Array.from(fieldNames)
|
||||
}
|
||||
|
||||
/**
|
||||
* Maps field names to standard field names based on common patterns
|
||||
* @param fieldName The field name to map
|
||||
* @returns The standard field name if a match is found, or null if no match
|
||||
*/
|
||||
export function mapToStandardField(fieldName: string): string | null {
|
||||
// Standard field mappings
|
||||
const standardMappings: Record<string, string[]> = {
|
||||
'title': ['title', 'name', 'headline', 'subject'],
|
||||
'description': ['description', 'summary', 'content', 'text', 'body'],
|
||||
'author': ['author', 'creator', 'user', 'owner', 'by'],
|
||||
'date': ['date', 'created', 'createdAt', 'timestamp', 'published'],
|
||||
'url': ['url', 'link', 'href', 'source'],
|
||||
'image': ['image', 'thumbnail', 'photo', 'picture'],
|
||||
'tags': ['tags', 'categories', 'keywords', 'topics']
|
||||
}
|
||||
|
||||
// Check for matches
|
||||
for (const [standardField, possibleMatches] of Object.entries(standardMappings)) {
|
||||
// Exact match
|
||||
if (possibleMatches.includes(fieldName)) {
|
||||
return standardField
|
||||
}
|
||||
|
||||
// Path match (e.g., "user.name" matches "name")
|
||||
const parts = fieldName.split('.')
|
||||
const lastPart = parts[parts.length - 1]
|
||||
if (possibleMatches.includes(lastPart)) {
|
||||
return standardField
|
||||
}
|
||||
|
||||
// Array match (e.g., "items[0].name" matches "name")
|
||||
if (fieldName.includes('[')) {
|
||||
for (const part of parts) {
|
||||
const cleanPart = part.split('[')[0]
|
||||
if (possibleMatches.includes(cleanPart)) {
|
||||
return standardField
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return null
|
||||
}
|
||||
309
src/utils/hybridModelManager.ts
Normal file
309
src/utils/hybridModelManager.ts
Normal file
|
|
@ -0,0 +1,309 @@
|
|||
/**
|
||||
* Hybrid Model Manager - BEST OF BOTH WORLDS
|
||||
*
|
||||
* Combines:
|
||||
* 1. Multi-source downloading strategy (GitHub → CDN → Hugging Face)
|
||||
* 2. Singleton pattern preventing multiple ONNX model loads
|
||||
* 3. Environment-specific optimizations
|
||||
* 4. Graceful fallbacks and error handling
|
||||
*/
|
||||
|
||||
import { TransformerEmbedding, TransformerEmbeddingOptions } from './embedding.js'
|
||||
import { EmbeddingFunction, Vector } from '../coreTypes.js'
|
||||
import { existsSync } from 'fs'
|
||||
import { mkdir, writeFile, readFile } from 'fs/promises'
|
||||
import { join, dirname } from 'path'
|
||||
|
||||
/**
|
||||
* Global singleton model manager - PREVENTS MULTIPLE MODEL LOADS
|
||||
*/
|
||||
class HybridModelManager {
|
||||
private static instance: HybridModelManager | null = null
|
||||
private primaryModel: TransformerEmbedding | null = null
|
||||
private modelPromise: Promise<TransformerEmbedding> | null = null
|
||||
private isInitialized = false
|
||||
private modelsPath: string
|
||||
|
||||
private constructor() {
|
||||
// Smart model path detection
|
||||
this.modelsPath = this.getModelsPath()
|
||||
}
|
||||
|
||||
public static getInstance(): HybridModelManager {
|
||||
if (!HybridModelManager.instance) {
|
||||
HybridModelManager.instance = new HybridModelManager()
|
||||
}
|
||||
return HybridModelManager.instance
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the primary embedding model - LOADS ONCE, REUSES FOREVER
|
||||
*/
|
||||
public async getPrimaryModel(): Promise<TransformerEmbedding> {
|
||||
// If already initialized, return immediately
|
||||
if (this.primaryModel && this.isInitialized) {
|
||||
return this.primaryModel
|
||||
}
|
||||
|
||||
// If initialization is in progress, wait for it
|
||||
if (this.modelPromise) {
|
||||
return await this.modelPromise
|
||||
}
|
||||
|
||||
// Start initialization with multi-source strategy
|
||||
this.modelPromise = this.initializePrimaryModel()
|
||||
return await this.modelPromise
|
||||
}
|
||||
|
||||
/**
|
||||
* Smart model path detection
|
||||
*/
|
||||
private getModelsPath(): string {
|
||||
const paths = [
|
||||
process.env.BRAINY_MODELS_PATH,
|
||||
'./models',
|
||||
'./node_modules/@soulcraft/brainy/models',
|
||||
join(process.cwd(), 'models')
|
||||
]
|
||||
|
||||
// Find first existing path or use default
|
||||
for (const path of paths) {
|
||||
if (path && existsSync(path)) {
|
||||
return path
|
||||
}
|
||||
}
|
||||
|
||||
return join(process.cwd(), 'models')
|
||||
}
|
||||
|
||||
/**
|
||||
* Initialize with BEST OF BOTH: Multi-source + Singleton
|
||||
*/
|
||||
private async initializePrimaryModel(): Promise<TransformerEmbedding> {
|
||||
try {
|
||||
// Environment detection for optimal configuration
|
||||
const isTest = (globalThis as any).__BRAINY_TEST_ENV__ || process.env.NODE_ENV === 'test'
|
||||
const isBrowser = typeof window !== 'undefined' && typeof document !== 'undefined'
|
||||
const isServerless = typeof process !== 'undefined' && (
|
||||
process.env.VERCEL ||
|
||||
process.env.NETLIFY ||
|
||||
process.env.AWS_LAMBDA_FUNCTION_NAME ||
|
||||
process.env.FUNCTIONS_WORKER_RUNTIME
|
||||
)
|
||||
const isDocker = typeof process !== 'undefined' && (
|
||||
process.env.DOCKER_CONTAINER ||
|
||||
process.env.KUBERNETES_SERVICE_HOST
|
||||
)
|
||||
|
||||
// Respect BRAINY_ALLOW_REMOTE_MODELS environment variable first
|
||||
let forceLocalOnly = false
|
||||
if (process.env.BRAINY_ALLOW_REMOTE_MODELS !== undefined) {
|
||||
forceLocalOnly = process.env.BRAINY_ALLOW_REMOTE_MODELS !== 'true'
|
||||
}
|
||||
|
||||
// Smart configuration based on environment
|
||||
let options: TransformerEmbeddingOptions = {
|
||||
verbose: !isTest && !isServerless,
|
||||
dtype: 'q8',
|
||||
device: 'cpu'
|
||||
}
|
||||
|
||||
// Environment-specific optimizations
|
||||
if (isBrowser) {
|
||||
options = {
|
||||
...options,
|
||||
localFilesOnly: forceLocalOnly || false, // Respect environment variable
|
||||
dtype: 'q8',
|
||||
device: 'cpu',
|
||||
verbose: false
|
||||
}
|
||||
} else if (isServerless) {
|
||||
options = {
|
||||
...options,
|
||||
localFilesOnly: forceLocalOnly || true, // Default true for serverless, but respect env
|
||||
dtype: 'q8',
|
||||
device: 'cpu',
|
||||
verbose: false
|
||||
}
|
||||
} else if (isDocker) {
|
||||
options = {
|
||||
...options,
|
||||
localFilesOnly: forceLocalOnly || true, // Default true for docker, but respect env
|
||||
dtype: 'fp32',
|
||||
device: 'auto',
|
||||
verbose: false
|
||||
}
|
||||
} else if (isTest) {
|
||||
// CRITICAL FOR TESTS: Allow remote downloads but be smart about it
|
||||
options = {
|
||||
...options,
|
||||
localFilesOnly: forceLocalOnly || false, // Respect environment variable for tests
|
||||
dtype: 'q8',
|
||||
device: 'cpu',
|
||||
verbose: false
|
||||
}
|
||||
} else {
|
||||
options = {
|
||||
...options,
|
||||
localFilesOnly: forceLocalOnly || false, // Respect environment variable for default node
|
||||
dtype: 'q8',
|
||||
device: 'auto',
|
||||
verbose: true
|
||||
}
|
||||
}
|
||||
|
||||
const environmentName = isBrowser ? 'browser' :
|
||||
isServerless ? 'serverless' :
|
||||
isDocker ? 'container' :
|
||||
isTest ? 'test' : 'node'
|
||||
|
||||
if (options.verbose) {
|
||||
console.log(`🧠 Initializing hybrid model manager (${environmentName} mode)...`)
|
||||
}
|
||||
|
||||
// MULTI-SOURCE STRATEGY: Try local first, then remote fallbacks
|
||||
this.primaryModel = await this.createModelWithFallbacks(options, environmentName)
|
||||
|
||||
this.isInitialized = true
|
||||
this.modelPromise = null // Clear the promise
|
||||
|
||||
if (options.verbose) {
|
||||
console.log(`✅ Hybrid model manager initialized successfully`)
|
||||
}
|
||||
|
||||
return this.primaryModel
|
||||
} catch (error) {
|
||||
this.modelPromise = null // Clear failed promise
|
||||
|
||||
const errorMessage = error instanceof Error ? error.message : String(error)
|
||||
const environmentInfo = typeof window !== 'undefined' ? 'browser' :
|
||||
typeof process !== 'undefined' ? `node (${process.version})` : 'unknown'
|
||||
|
||||
throw new Error(
|
||||
`Failed to initialize hybrid model manager in ${environmentInfo} environment: ${errorMessage}. ` +
|
||||
`This is critical for all Brainy operations.`
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Create model with multi-source fallback strategy
|
||||
*/
|
||||
private async createModelWithFallbacks(
|
||||
options: TransformerEmbeddingOptions,
|
||||
environmentName: string
|
||||
): Promise<TransformerEmbedding> {
|
||||
const attempts = [
|
||||
// 1. Try with current configuration (may use local cache)
|
||||
{ ...options, localFilesOnly: false, source: 'primary' },
|
||||
|
||||
// 2. If that fails, explicitly allow remote with verbose logging
|
||||
{ ...options, localFilesOnly: false, verbose: true, source: 'fallback-verbose' },
|
||||
|
||||
// 3. Last resort: basic configuration
|
||||
{ verbose: false, dtype: 'q8' as const, device: 'cpu' as const, localFilesOnly: false, source: 'last-resort' }
|
||||
]
|
||||
|
||||
let lastError: Error | null = null
|
||||
|
||||
for (const attemptOptions of attempts) {
|
||||
try {
|
||||
const { source, ...modelOptions } = attemptOptions
|
||||
|
||||
if (attemptOptions.verbose) {
|
||||
console.log(`🔄 Attempting model load (${source})...`)
|
||||
}
|
||||
|
||||
const model = new TransformerEmbedding(modelOptions)
|
||||
await model.init()
|
||||
|
||||
if (attemptOptions.verbose) {
|
||||
console.log(`✅ Model loaded successfully with ${source} strategy`)
|
||||
}
|
||||
|
||||
return model
|
||||
} catch (error) {
|
||||
lastError = error instanceof Error ? error : new Error(String(error))
|
||||
|
||||
if (attemptOptions.verbose) {
|
||||
console.log(`❌ Failed ${attemptOptions.source} strategy:`, lastError.message)
|
||||
}
|
||||
|
||||
// Continue to next attempt
|
||||
}
|
||||
}
|
||||
|
||||
// All attempts failed
|
||||
throw new Error(
|
||||
`All model loading strategies failed in ${environmentName} environment. ` +
|
||||
`Last error: ${lastError?.message}. ` +
|
||||
`Check network connectivity or ensure models are available locally.`
|
||||
)
|
||||
}
|
||||
|
||||
/**
|
||||
* Get embedding function that reuses the singleton model
|
||||
*/
|
||||
public async getEmbeddingFunction(): Promise<EmbeddingFunction> {
|
||||
const model = await this.getPrimaryModel()
|
||||
|
||||
return async (data: string | string[]): Promise<Vector> => {
|
||||
return await model.embed(data)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if model is ready (loaded and initialized)
|
||||
*/
|
||||
public isModelReady(): boolean {
|
||||
return this.isInitialized && this.primaryModel !== null
|
||||
}
|
||||
|
||||
/**
|
||||
* Force model reload (for testing or recovery)
|
||||
*/
|
||||
public async reloadModel(): Promise<void> {
|
||||
this.primaryModel = null
|
||||
this.isInitialized = false
|
||||
this.modelPromise = null
|
||||
await this.getPrimaryModel()
|
||||
}
|
||||
|
||||
/**
|
||||
* Get model status for debugging
|
||||
*/
|
||||
public getModelStatus(): { loaded: boolean, ready: boolean, modelType: string } {
|
||||
return {
|
||||
loaded: this.primaryModel !== null,
|
||||
ready: this.isInitialized,
|
||||
modelType: 'HybridModelManager (Multi-source + Singleton)'
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Export singleton instance
|
||||
export const hybridModelManager = HybridModelManager.getInstance()
|
||||
|
||||
/**
|
||||
* Get the hybrid singleton embedding function - USE THIS EVERYWHERE!
|
||||
*/
|
||||
export async function getHybridEmbeddingFunction(): Promise<EmbeddingFunction> {
|
||||
return await hybridModelManager.getEmbeddingFunction()
|
||||
}
|
||||
|
||||
/**
|
||||
* Optimized hybrid embedding function that uses multi-source + singleton
|
||||
*/
|
||||
export const hybridEmbeddingFunction: EmbeddingFunction = async (data: string | string[]): Promise<Vector> => {
|
||||
const embeddingFn = await getHybridEmbeddingFunction()
|
||||
return await embeddingFn(data)
|
||||
}
|
||||
|
||||
/**
|
||||
* Preload model for tests or production - CALL THIS ONCE AT START
|
||||
*/
|
||||
export async function preloadHybridModel(): Promise<void> {
|
||||
console.log('🚀 Preloading hybrid model...')
|
||||
await hybridModelManager.getPrimaryModel()
|
||||
console.log('✅ Hybrid model preloaded and ready!')
|
||||
}
|
||||
7
src/utils/index.ts
Normal file
7
src/utils/index.ts
Normal file
|
|
@ -0,0 +1,7 @@
|
|||
export * from './distance.js'
|
||||
export * from './embedding.js'
|
||||
export * from './workerUtils.js'
|
||||
export * from './statistics.js'
|
||||
export * from './jsonProcessing.js'
|
||||
export * from './fieldNameTracking.js'
|
||||
export * from './version.js'
|
||||
226
src/utils/jsonProcessing.ts
Normal file
226
src/utils/jsonProcessing.ts
Normal file
|
|
@ -0,0 +1,226 @@
|
|||
/**
|
||||
* Utility functions for processing JSON documents for vectorization and search
|
||||
*/
|
||||
|
||||
/**
|
||||
* Extracts text from a JSON object for vectorization
|
||||
* This function recursively processes the JSON object and extracts text from all fields
|
||||
* It can also prioritize specific fields if provided
|
||||
*
|
||||
* @param jsonObject The JSON object to extract text from
|
||||
* @param options Configuration options for text extraction
|
||||
* @returns A string containing the extracted text
|
||||
*/
|
||||
export function extractTextFromJson(
|
||||
jsonObject: any,
|
||||
options: {
|
||||
priorityFields?: string[] // Fields to prioritize (will be repeated for emphasis)
|
||||
excludeFields?: string[] // Fields to exclude from extraction
|
||||
includeFieldNames?: boolean // Whether to include field names in the extracted text
|
||||
maxDepth?: number // Maximum depth to recurse into nested objects
|
||||
currentDepth?: number // Current recursion depth (internal use)
|
||||
fieldPath?: string[] // Current field path (internal use)
|
||||
} = {}
|
||||
): string {
|
||||
// Set default options
|
||||
const {
|
||||
priorityFields = [],
|
||||
excludeFields = [],
|
||||
includeFieldNames = true,
|
||||
maxDepth = 5,
|
||||
currentDepth = 0,
|
||||
fieldPath = []
|
||||
} = options
|
||||
|
||||
// If input is not an object or array, or we've reached max depth, return as string
|
||||
if (
|
||||
jsonObject === null ||
|
||||
jsonObject === undefined ||
|
||||
typeof jsonObject !== 'object' ||
|
||||
currentDepth >= maxDepth
|
||||
) {
|
||||
return String(jsonObject || '')
|
||||
}
|
||||
|
||||
const extractedText: string[] = []
|
||||
const priorityText: string[] = []
|
||||
|
||||
// Process arrays
|
||||
if (Array.isArray(jsonObject)) {
|
||||
for (let i = 0; i < jsonObject.length; i++) {
|
||||
const value = jsonObject[i]
|
||||
const newPath = [...fieldPath, i.toString()]
|
||||
|
||||
// Recursively extract text from array items
|
||||
const itemText = extractTextFromJson(value, {
|
||||
priorityFields,
|
||||
excludeFields,
|
||||
includeFieldNames,
|
||||
maxDepth,
|
||||
currentDepth: currentDepth + 1,
|
||||
fieldPath: newPath
|
||||
})
|
||||
|
||||
if (itemText) {
|
||||
extractedText.push(itemText)
|
||||
}
|
||||
}
|
||||
}
|
||||
// Process objects
|
||||
else {
|
||||
for (const [key, value] of Object.entries(jsonObject)) {
|
||||
// Skip excluded fields
|
||||
if (excludeFields.includes(key)) {
|
||||
continue
|
||||
}
|
||||
|
||||
const newPath = [...fieldPath, key]
|
||||
const fullPath = newPath.join('.')
|
||||
|
||||
// Check if this is a priority field
|
||||
const isPriority = priorityFields.some(field => {
|
||||
// Exact match
|
||||
if (field === key) return true
|
||||
// Path match
|
||||
if (field === fullPath) return true
|
||||
// Wildcard match (e.g., "user.*" matches "user.name", "user.email", etc.)
|
||||
if (field.endsWith('.*') && fullPath.startsWith(field.slice(0, -2))) return true
|
||||
return false
|
||||
})
|
||||
|
||||
// Get the field value as text
|
||||
let fieldText: string
|
||||
|
||||
if (typeof value === 'object' && value !== null) {
|
||||
// Recursively extract text from nested objects
|
||||
fieldText = extractTextFromJson(value, {
|
||||
priorityFields,
|
||||
excludeFields,
|
||||
includeFieldNames,
|
||||
maxDepth,
|
||||
currentDepth: currentDepth + 1,
|
||||
fieldPath: newPath
|
||||
})
|
||||
} else {
|
||||
fieldText = String(value || '')
|
||||
}
|
||||
|
||||
// Add field name if requested
|
||||
if (includeFieldNames && fieldText) {
|
||||
fieldText = `${key}: ${fieldText}`
|
||||
}
|
||||
|
||||
// Add to appropriate collection
|
||||
if (fieldText) {
|
||||
if (isPriority) {
|
||||
priorityText.push(fieldText)
|
||||
} else {
|
||||
extractedText.push(fieldText)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Combine priority text (repeated for emphasis) and regular text
|
||||
return [...priorityText, ...priorityText, ...extractedText].join(' ')
|
||||
}
|
||||
|
||||
/**
|
||||
* Prepares a JSON document for vectorization
|
||||
* This function extracts text from the JSON document and formats it for optimal vectorization
|
||||
*
|
||||
* @param jsonDocument The JSON document to prepare
|
||||
* @param options Configuration options for preparation
|
||||
* @returns A string ready for vectorization
|
||||
*/
|
||||
export function prepareJsonForVectorization(
|
||||
jsonDocument: any,
|
||||
options: {
|
||||
priorityFields?: string[]
|
||||
excludeFields?: string[]
|
||||
includeFieldNames?: boolean
|
||||
maxDepth?: number
|
||||
} = {}
|
||||
): string {
|
||||
// If input is a string, try to parse it as JSON
|
||||
let document = jsonDocument
|
||||
if (typeof jsonDocument === 'string') {
|
||||
try {
|
||||
document = JSON.parse(jsonDocument)
|
||||
} catch (e) {
|
||||
// If parsing fails, treat it as a plain string
|
||||
return jsonDocument
|
||||
}
|
||||
}
|
||||
|
||||
// If not an object after parsing, return as is
|
||||
if (typeof document !== 'object' || document === null) {
|
||||
return String(document || '')
|
||||
}
|
||||
|
||||
// Extract text from the document
|
||||
return extractTextFromJson(document, options)
|
||||
}
|
||||
|
||||
/**
|
||||
* Extracts text from a specific field in a JSON document
|
||||
* This is useful for searching within specific fields
|
||||
*
|
||||
* @param jsonDocument The JSON document to extract from
|
||||
* @param fieldPath The path to the field (e.g., "user.name" or "addresses[0].city")
|
||||
* @returns The extracted text or empty string if field not found
|
||||
*/
|
||||
export function extractFieldFromJson(
|
||||
jsonDocument: any,
|
||||
fieldPath: string
|
||||
): string {
|
||||
// If input is a string, try to parse it as JSON
|
||||
let document = jsonDocument
|
||||
if (typeof jsonDocument === 'string') {
|
||||
try {
|
||||
document = JSON.parse(jsonDocument)
|
||||
} catch (e) {
|
||||
// If parsing fails, return empty string
|
||||
return ''
|
||||
}
|
||||
}
|
||||
|
||||
// If not an object after parsing, return empty string
|
||||
if (typeof document !== 'object' || document === null) {
|
||||
return ''
|
||||
}
|
||||
|
||||
// Parse the field path
|
||||
const parts = fieldPath.split('.')
|
||||
let current = document
|
||||
|
||||
// Navigate through the path
|
||||
for (const part of parts) {
|
||||
// Handle array indexing (e.g., "addresses[0]")
|
||||
const match = part.match(/^([^[]+)(?:\[(\d+)\])?$/)
|
||||
if (!match) {
|
||||
return ''
|
||||
}
|
||||
|
||||
const [, key, indexStr] = match
|
||||
|
||||
// Move to the next level
|
||||
current = current[key]
|
||||
|
||||
// If we have an array index, access that element
|
||||
if (indexStr !== undefined && Array.isArray(current)) {
|
||||
const index = parseInt(indexStr, 10)
|
||||
current = current[index]
|
||||
}
|
||||
|
||||
// If we've reached a null or undefined value, return empty string
|
||||
if (current === null || current === undefined) {
|
||||
return ''
|
||||
}
|
||||
}
|
||||
|
||||
// Convert the final value to string
|
||||
return typeof current === 'object'
|
||||
? JSON.stringify(current)
|
||||
: String(current)
|
||||
}
|
||||
268
src/utils/logger.ts
Normal file
268
src/utils/logger.ts
Normal file
|
|
@ -0,0 +1,268 @@
|
|||
/**
|
||||
* Centralized logging utility for Brainy
|
||||
* Provides configurable log levels and consistent logging across the codebase
|
||||
* Automatically reduces logging in production environments to minimize costs
|
||||
*/
|
||||
|
||||
import { isProductionEnvironment, getLogLevel } from './environment.js'
|
||||
|
||||
export enum LogLevel {
|
||||
ERROR = 0,
|
||||
WARN = 1,
|
||||
INFO = 2,
|
||||
DEBUG = 3,
|
||||
TRACE = 4
|
||||
}
|
||||
|
||||
export interface LoggerConfig {
|
||||
level: LogLevel
|
||||
// Specific module log levels
|
||||
modules?: {
|
||||
[moduleName: string]: LogLevel
|
||||
}
|
||||
// Whether to include timestamps
|
||||
timestamps?: boolean
|
||||
// Whether to include module names
|
||||
includeModule?: boolean
|
||||
// Custom log handler
|
||||
handler?: (level: LogLevel, module: string, message: string, ...args: any[]) => void
|
||||
}
|
||||
|
||||
class Logger {
|
||||
private static instance: Logger
|
||||
private config: LoggerConfig = {
|
||||
level: LogLevel.ERROR, // Default to ERROR in production for cost optimization
|
||||
timestamps: false, // Disable timestamps in production to reduce log size
|
||||
includeModule: true
|
||||
}
|
||||
|
||||
private constructor() {
|
||||
// Auto-detect production environment and set appropriate defaults
|
||||
this.applyEnvironmentDefaults()
|
||||
|
||||
// Set log level from environment variable if available (overrides auto-detection)
|
||||
const envLogLevel = process.env.BRAINY_LOG_LEVEL
|
||||
if (envLogLevel) {
|
||||
const level = LogLevel[envLogLevel.toUpperCase() as keyof typeof LogLevel]
|
||||
if (level !== undefined) {
|
||||
this.config.level = level
|
||||
}
|
||||
}
|
||||
|
||||
// Parse module-specific log levels
|
||||
const moduleLogLevels = process.env.BRAINY_MODULE_LOG_LEVELS
|
||||
if (moduleLogLevels) {
|
||||
try {
|
||||
this.config.modules = JSON.parse(moduleLogLevels)
|
||||
} catch (e) {
|
||||
// Ignore parsing errors
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private applyEnvironmentDefaults(): void {
|
||||
const envLogLevel = getLogLevel()
|
||||
|
||||
// Convert environment log level to Logger LogLevel
|
||||
switch (envLogLevel) {
|
||||
case 'silent':
|
||||
this.config.level = -1 as LogLevel // Below ERROR to silence all logs
|
||||
break
|
||||
case 'error':
|
||||
this.config.level = LogLevel.ERROR
|
||||
this.config.timestamps = false // Minimize log size in production
|
||||
break
|
||||
case 'warn':
|
||||
this.config.level = LogLevel.WARN
|
||||
this.config.timestamps = false
|
||||
break
|
||||
case 'info':
|
||||
this.config.level = LogLevel.INFO
|
||||
this.config.timestamps = true
|
||||
break
|
||||
case 'verbose':
|
||||
this.config.level = LogLevel.DEBUG
|
||||
this.config.timestamps = true
|
||||
break
|
||||
}
|
||||
|
||||
// In production environments, be extra conservative to minimize costs
|
||||
if (isProductionEnvironment()) {
|
||||
this.config.level = Math.min(this.config.level, LogLevel.ERROR)
|
||||
this.config.timestamps = false
|
||||
this.config.includeModule = false // Reduce log size
|
||||
}
|
||||
}
|
||||
|
||||
static getInstance(): Logger {
|
||||
if (!Logger.instance) {
|
||||
Logger.instance = new Logger()
|
||||
}
|
||||
return Logger.instance
|
||||
}
|
||||
|
||||
configure(config: Partial<LoggerConfig>): void {
|
||||
this.config = { ...this.config, ...config }
|
||||
}
|
||||
|
||||
private shouldLog(level: LogLevel, module: string): boolean {
|
||||
// Check module-specific level first
|
||||
if (this.config.modules && this.config.modules[module] !== undefined) {
|
||||
return level <= this.config.modules[module]
|
||||
}
|
||||
// Otherwise use global level
|
||||
return level <= this.config.level
|
||||
}
|
||||
|
||||
private formatMessage(level: LogLevel, module: string, message: string): string {
|
||||
const parts: string[] = []
|
||||
|
||||
if (this.config.timestamps) {
|
||||
parts.push(`[${new Date().toISOString()}]`)
|
||||
}
|
||||
|
||||
parts.push(`[${LogLevel[level]}]`)
|
||||
|
||||
if (this.config.includeModule) {
|
||||
parts.push(`[${module}]`)
|
||||
}
|
||||
|
||||
parts.push(message)
|
||||
|
||||
return parts.join(' ')
|
||||
}
|
||||
|
||||
private log(level: LogLevel, module: string, message: string, ...args: any[]): void {
|
||||
if (!this.shouldLog(level, module)) {
|
||||
return
|
||||
}
|
||||
|
||||
if (this.config.handler) {
|
||||
this.config.handler(level, module, message, ...args)
|
||||
return
|
||||
}
|
||||
|
||||
const formattedMessage = this.formatMessage(level, module, message)
|
||||
|
||||
switch (level) {
|
||||
case LogLevel.ERROR:
|
||||
console.error(formattedMessage, ...args)
|
||||
break
|
||||
case LogLevel.WARN:
|
||||
console.warn(formattedMessage, ...args)
|
||||
break
|
||||
case LogLevel.INFO:
|
||||
console.info(formattedMessage, ...args)
|
||||
break
|
||||
case LogLevel.DEBUG:
|
||||
case LogLevel.TRACE:
|
||||
console.log(formattedMessage, ...args)
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
error(module: string, message: string, ...args: any[]): void {
|
||||
this.log(LogLevel.ERROR, module, message, ...args)
|
||||
}
|
||||
|
||||
warn(module: string, message: string, ...args: any[]): void {
|
||||
this.log(LogLevel.WARN, module, message, ...args)
|
||||
}
|
||||
|
||||
info(module: string, message: string, ...args: any[]): void {
|
||||
this.log(LogLevel.INFO, module, message, ...args)
|
||||
}
|
||||
|
||||
debug(module: string, message: string, ...args: any[]): void {
|
||||
this.log(LogLevel.DEBUG, module, message, ...args)
|
||||
}
|
||||
|
||||
trace(module: string, message: string, ...args: any[]): void {
|
||||
this.log(LogLevel.TRACE, module, message, ...args)
|
||||
}
|
||||
|
||||
// Create a module-specific logger
|
||||
createModuleLogger(module: string) {
|
||||
return {
|
||||
error: (message: string, ...args: any[]) => this.error(module, message, ...args),
|
||||
warn: (message: string, ...args: any[]) => this.warn(module, message, ...args),
|
||||
info: (message: string, ...args: any[]) => this.info(module, message, ...args),
|
||||
debug: (message: string, ...args: any[]) => this.debug(module, message, ...args),
|
||||
trace: (message: string, ...args: any[]) => this.trace(module, message, ...args)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Export singleton instance
|
||||
export const logger = Logger.getInstance()
|
||||
|
||||
// Export convenience function for creating module loggers
|
||||
export function createModuleLogger(module: string) {
|
||||
return logger.createModuleLogger(module)
|
||||
}
|
||||
|
||||
// Export function to configure logger
|
||||
export function configureLogger(config: Partial<LoggerConfig>) {
|
||||
logger.configure(config)
|
||||
}
|
||||
|
||||
/**
|
||||
* Smart console replacement that automatically reduces logging in production
|
||||
* Dramatically reduces Google Cloud Run logging costs
|
||||
*
|
||||
* Usage: Replace console.log with smartConsole.log, etc.
|
||||
*/
|
||||
export const smartConsole = {
|
||||
log: (message?: any, ...args: any[]) => {
|
||||
if (logger['shouldLog'](LogLevel.INFO, 'console')) {
|
||||
console.log(message, ...args)
|
||||
}
|
||||
},
|
||||
|
||||
info: (message?: any, ...args: any[]) => {
|
||||
if (logger['shouldLog'](LogLevel.INFO, 'console')) {
|
||||
console.info(message, ...args)
|
||||
}
|
||||
},
|
||||
|
||||
warn: (message?: any, ...args: any[]) => {
|
||||
if (logger['shouldLog'](LogLevel.WARN, 'console')) {
|
||||
console.warn(message, ...args)
|
||||
}
|
||||
},
|
||||
|
||||
error: (message?: any, ...args: any[]) => {
|
||||
if (logger['shouldLog'](LogLevel.ERROR, 'console')) {
|
||||
console.error(message, ...args)
|
||||
}
|
||||
},
|
||||
|
||||
debug: (message?: any, ...args: any[]) => {
|
||||
if (logger['shouldLog'](LogLevel.DEBUG, 'console')) {
|
||||
console.debug(message, ...args)
|
||||
}
|
||||
},
|
||||
|
||||
trace: (message?: any, ...args: any[]) => {
|
||||
if (logger['shouldLog'](LogLevel.TRACE, 'console')) {
|
||||
console.trace(message, ...args)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Production-optimized logging functions
|
||||
* These only log in non-production environments or when explicitly enabled
|
||||
*/
|
||||
export const prodLog = {
|
||||
// Only log errors in production (always visible)
|
||||
error: (message?: any, ...args: any[]) => {
|
||||
console.error(message, ...args)
|
||||
},
|
||||
|
||||
// These are suppressed in production unless BRAINY_LOG_LEVEL is set
|
||||
warn: (message?: any, ...args: any[]) => smartConsole.warn(message, ...args),
|
||||
info: (message?: any, ...args: any[]) => smartConsole.info(message, ...args),
|
||||
debug: (message?: any, ...args: any[]) => smartConsole.debug(message, ...args),
|
||||
log: (message?: any, ...args: any[]) => smartConsole.log(message, ...args)
|
||||
}
|
||||
382
src/utils/metadataFilter.ts
Normal file
382
src/utils/metadataFilter.ts
Normal file
|
|
@ -0,0 +1,382 @@
|
|||
/**
|
||||
* Smart metadata filtering for vector search
|
||||
* Filters DURING search to ensure relevant results
|
||||
* Simple API that just works without configuration
|
||||
*/
|
||||
|
||||
import { SearchResult, HNSWNoun } from '../coreTypes.js'
|
||||
|
||||
/**
|
||||
* Brainy Field Operators (BFO) - Our own field query system
|
||||
* Designed for performance, clarity, and patent independence
|
||||
*/
|
||||
export interface BrainyFieldOperators {
|
||||
// Equality operators
|
||||
equals?: any
|
||||
notEquals?: any
|
||||
is?: any
|
||||
isNot?: any
|
||||
|
||||
// Comparison operators
|
||||
greaterThan?: any
|
||||
greaterEqual?: any
|
||||
lessThan?: any
|
||||
lessEqual?: any
|
||||
between?: [any, any]
|
||||
|
||||
// Array/Set operators
|
||||
oneOf?: any[]
|
||||
noneOf?: any[]
|
||||
contains?: any
|
||||
excludes?: any
|
||||
hasAll?: any[]
|
||||
length?: number
|
||||
|
||||
// Existence operators
|
||||
exists?: boolean
|
||||
missing?: boolean
|
||||
|
||||
// Pattern operators
|
||||
matches?: string | RegExp
|
||||
startsWith?: string
|
||||
endsWith?: string
|
||||
|
||||
// Logical operators
|
||||
allOf?: MetadataFilter[]
|
||||
anyOf?: MetadataFilter[]
|
||||
not?: MetadataFilter
|
||||
|
||||
// Short aliases for common operations
|
||||
eq?: any
|
||||
ne?: any
|
||||
gt?: any
|
||||
gte?: any
|
||||
lt?: any
|
||||
lte?: any
|
||||
}
|
||||
|
||||
/**
|
||||
* Metadata filter definition
|
||||
*/
|
||||
export interface MetadataFilter {
|
||||
[key: string]: any | BrainyFieldOperators
|
||||
}
|
||||
|
||||
/**
|
||||
* Options for metadata filtering
|
||||
*/
|
||||
export interface MetadataFilterOptions {
|
||||
metadata?: MetadataFilter
|
||||
scoring?: {
|
||||
vectorWeight?: number
|
||||
metadataWeight?: number
|
||||
metadataBoosts?: Record<string, number | ((value: any, query: any) => number)>
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if a value matches a query with operators
|
||||
*/
|
||||
function matchesQuery(value: any, query: any): boolean {
|
||||
// Direct equality check
|
||||
if (typeof query !== 'object' || query === null || Array.isArray(query)) {
|
||||
return value === query
|
||||
}
|
||||
|
||||
// Check for Brainy Field Operators (BFO)
|
||||
for (const [op, operand] of Object.entries(query)) {
|
||||
switch (op) {
|
||||
// Equality operators
|
||||
case 'equals':
|
||||
case 'is':
|
||||
case 'eq':
|
||||
if (value !== operand) return false
|
||||
break
|
||||
case 'notEquals':
|
||||
case 'isNot':
|
||||
case 'ne':
|
||||
if (value === operand) return false
|
||||
break
|
||||
|
||||
// Comparison operators
|
||||
case 'greaterThan':
|
||||
case 'gt':
|
||||
if (typeof value !== 'number' || typeof operand !== 'number' || !(value > operand)) return false
|
||||
break
|
||||
case 'greaterEqual':
|
||||
case 'gte':
|
||||
if (typeof value !== 'number' || typeof operand !== 'number' || !(value >= operand)) return false
|
||||
break
|
||||
case 'lessThan':
|
||||
case 'lt':
|
||||
if (typeof value !== 'number' || typeof operand !== 'number' || !(value < operand)) return false
|
||||
break
|
||||
case 'lessEqual':
|
||||
case 'lte':
|
||||
if (typeof value !== 'number' || typeof operand !== 'number' || !(value <= operand)) return false
|
||||
break
|
||||
case 'between':
|
||||
if (!Array.isArray(operand) || operand.length !== 2) return false
|
||||
if (typeof value !== 'number' || !(value >= operand[0] && value <= operand[1])) return false
|
||||
break
|
||||
|
||||
// Array/Set operators
|
||||
case 'oneOf':
|
||||
if (!Array.isArray(operand) || !operand.includes(value)) return false
|
||||
break
|
||||
case 'noneOf':
|
||||
if (!Array.isArray(operand) || operand.includes(value)) return false
|
||||
break
|
||||
case 'contains':
|
||||
if (!Array.isArray(value) || !value.includes(operand)) return false
|
||||
break
|
||||
case 'excludes':
|
||||
if (!Array.isArray(value) || value.includes(operand)) return false
|
||||
break
|
||||
case 'hasAll':
|
||||
if (!Array.isArray(value) || !Array.isArray(operand)) return false
|
||||
for (const item of operand) {
|
||||
if (!value.includes(item)) return false
|
||||
}
|
||||
break
|
||||
case 'length':
|
||||
if (!Array.isArray(value) || value.length !== operand) return false
|
||||
break
|
||||
|
||||
// Existence operators
|
||||
case 'exists':
|
||||
if ((value !== undefined) !== operand) return false
|
||||
break
|
||||
case 'missing':
|
||||
if ((value === undefined) !== operand) return false
|
||||
break
|
||||
|
||||
// Pattern operators
|
||||
case 'matches':
|
||||
const regex = typeof operand === 'string' ? new RegExp(operand) : operand as RegExp
|
||||
if (!(regex instanceof RegExp) || !regex.test(String(value))) return false
|
||||
break
|
||||
case 'startsWith':
|
||||
if (typeof value !== 'string' || !value.startsWith(String(operand))) return false
|
||||
break
|
||||
case 'endsWith':
|
||||
if (typeof value !== 'string' || !value.endsWith(String(operand))) return false
|
||||
break
|
||||
|
||||
default:
|
||||
// Unknown operator, treat as field name
|
||||
if (!matchesFieldQuery(value, op, operand)) return false
|
||||
}
|
||||
}
|
||||
|
||||
return true
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if a field matches a query
|
||||
*/
|
||||
function matchesFieldQuery(obj: any, field: string, query: any): boolean {
|
||||
const value = getNestedValue(obj, field)
|
||||
return matchesQuery(value, query)
|
||||
}
|
||||
|
||||
/**
|
||||
* Get nested value from object using dot notation
|
||||
*/
|
||||
function getNestedValue(obj: any, path: string): any {
|
||||
const parts = path.split('.')
|
||||
let current = obj
|
||||
|
||||
for (const part of parts) {
|
||||
if (current === null || current === undefined) {
|
||||
return undefined
|
||||
}
|
||||
current = current[part]
|
||||
}
|
||||
|
||||
return current
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if metadata matches the filter
|
||||
*/
|
||||
export function matchesMetadataFilter(metadata: any, filter: MetadataFilter): boolean {
|
||||
if (!filter || Object.keys(filter).length === 0) {
|
||||
return true
|
||||
}
|
||||
|
||||
for (const [key, query] of Object.entries(filter)) {
|
||||
// Handle logical operators
|
||||
if (key === 'allOf') {
|
||||
if (!Array.isArray(query)) return false
|
||||
for (const subFilter of query) {
|
||||
if (!matchesMetadataFilter(metadata, subFilter)) return false
|
||||
}
|
||||
continue
|
||||
}
|
||||
|
||||
if (key === 'anyOf') {
|
||||
if (!Array.isArray(query)) return false
|
||||
let matched = false
|
||||
for (const subFilter of query) {
|
||||
if (matchesMetadataFilter(metadata, subFilter)) {
|
||||
matched = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if (!matched) return false
|
||||
continue
|
||||
}
|
||||
|
||||
if (key === 'not') {
|
||||
if (matchesMetadataFilter(metadata, query)) return false
|
||||
continue
|
||||
}
|
||||
|
||||
// Handle field queries
|
||||
const value = getNestedValue(metadata, key)
|
||||
if (!matchesQuery(value, query)) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
return true
|
||||
}
|
||||
|
||||
/**
|
||||
* Calculate metadata boost score
|
||||
*/
|
||||
export function calculateMetadataScore(
|
||||
metadata: any,
|
||||
filter: MetadataFilter,
|
||||
scoring?: MetadataFilterOptions['scoring']
|
||||
): number {
|
||||
if (!scoring || !scoring.metadataBoosts) {
|
||||
return 0
|
||||
}
|
||||
|
||||
let score = 0
|
||||
|
||||
for (const [field, boost] of Object.entries(scoring.metadataBoosts)) {
|
||||
const value = getNestedValue(metadata, field)
|
||||
|
||||
if (typeof boost === 'function') {
|
||||
score += boost(value, filter)
|
||||
} else if (value !== undefined) {
|
||||
// Check if the field matches the filter
|
||||
const fieldFilter = filter[field]
|
||||
if (fieldFilter && matchesQuery(value, fieldFilter)) {
|
||||
score += boost
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return score
|
||||
}
|
||||
|
||||
/**
|
||||
* Apply compound scoring to search results
|
||||
*/
|
||||
export function applyCompoundScoring<T>(
|
||||
results: SearchResult<T>[],
|
||||
filter: MetadataFilter,
|
||||
scoring?: MetadataFilterOptions['scoring']
|
||||
): SearchResult<T>[] {
|
||||
if (!scoring || (!scoring.vectorWeight && !scoring.metadataWeight)) {
|
||||
return results
|
||||
}
|
||||
|
||||
const vectorWeight = scoring.vectorWeight ?? 1.0
|
||||
const metadataWeight = scoring.metadataWeight ?? 0.0
|
||||
|
||||
return results.map(result => {
|
||||
const metadataScore = calculateMetadataScore(result.metadata, filter, scoring)
|
||||
const combinedScore = (result.score * vectorWeight) + (metadataScore * metadataWeight)
|
||||
|
||||
return {
|
||||
...result,
|
||||
score: combinedScore
|
||||
}
|
||||
}).sort((a, b) => b.score - a.score) // Re-sort by combined score
|
||||
}
|
||||
|
||||
/**
|
||||
* Filter search results by metadata
|
||||
*/
|
||||
export function filterSearchResultsByMetadata<T>(
|
||||
results: SearchResult<T>[],
|
||||
filter: MetadataFilter
|
||||
): SearchResult<T>[] {
|
||||
if (!filter || Object.keys(filter).length === 0) {
|
||||
return results
|
||||
}
|
||||
|
||||
return results.filter(result =>
|
||||
matchesMetadataFilter(result.metadata, filter)
|
||||
)
|
||||
}
|
||||
|
||||
/**
|
||||
* Filter nouns by metadata before search
|
||||
*/
|
||||
export function filterNounsByMetadata(
|
||||
nouns: HNSWNoun[],
|
||||
filter: MetadataFilter
|
||||
): HNSWNoun[] {
|
||||
if (!filter || Object.keys(filter).length === 0) {
|
||||
return nouns
|
||||
}
|
||||
|
||||
return nouns.filter(noun =>
|
||||
matchesMetadataFilter(noun.metadata, filter)
|
||||
)
|
||||
}
|
||||
|
||||
/**
|
||||
* Aggregate search results for faceted search
|
||||
*/
|
||||
export interface FacetConfig {
|
||||
field: string
|
||||
limit?: number
|
||||
}
|
||||
|
||||
export interface FacetResult {
|
||||
[value: string]: number
|
||||
}
|
||||
|
||||
export interface AggregationResult<T> {
|
||||
results: SearchResult<T>[]
|
||||
facets: Record<string, FacetResult>
|
||||
}
|
||||
|
||||
export function aggregateSearchResults<T>(
|
||||
results: SearchResult<T>[],
|
||||
facets: Record<string, FacetConfig>
|
||||
): AggregationResult<T> {
|
||||
const facetResults: Record<string, FacetResult> = {}
|
||||
|
||||
for (const [facetName, config] of Object.entries(facets)) {
|
||||
const counts: Record<string, number> = {}
|
||||
|
||||
for (const result of results) {
|
||||
const value = getNestedValue(result.metadata, config.field)
|
||||
|
||||
if (value !== undefined) {
|
||||
const key = String(value)
|
||||
counts[key] = (counts[key] || 0) + 1
|
||||
}
|
||||
}
|
||||
|
||||
// Sort by count and apply limit
|
||||
const sorted = Object.entries(counts)
|
||||
.sort((a, b) => b[1] - a[1])
|
||||
.slice(0, config.limit || 10)
|
||||
|
||||
facetResults[facetName] = Object.fromEntries(sorted)
|
||||
}
|
||||
|
||||
return {
|
||||
results,
|
||||
facets: facetResults
|
||||
}
|
||||
}
|
||||
1352
src/utils/metadataIndex.ts
Normal file
1352
src/utils/metadataIndex.ts
Normal file
File diff suppressed because it is too large
Load diff
151
src/utils/metadataIndexCache.ts
Normal file
151
src/utils/metadataIndexCache.ts
Normal file
|
|
@ -0,0 +1,151 @@
|
|||
/**
|
||||
* MetadataIndexCache - Caches metadata index data for improved performance
|
||||
* Reuses the same pattern as SearchCache for consistency
|
||||
*/
|
||||
|
||||
export interface MetadataCacheEntry {
|
||||
data: any // Field index or value chunk data
|
||||
timestamp: number
|
||||
hits: number
|
||||
}
|
||||
|
||||
export interface MetadataIndexCacheConfig {
|
||||
maxAge?: number // Maximum age in milliseconds (default: 5 minutes)
|
||||
maxSize?: number // Maximum number of cached entries (default: 500)
|
||||
enabled?: boolean // Whether caching is enabled (default: true)
|
||||
hitCountWeight?: number // Weight for hit count in eviction policy (default: 0.3)
|
||||
}
|
||||
|
||||
export class MetadataIndexCache {
|
||||
private cache = new Map<string, MetadataCacheEntry>()
|
||||
private maxAge: number
|
||||
private maxSize: number
|
||||
private enabled: boolean
|
||||
private hitCountWeight: number
|
||||
|
||||
// Cache statistics
|
||||
private hits = 0
|
||||
private misses = 0
|
||||
private evictions = 0
|
||||
|
||||
constructor(config: MetadataIndexCacheConfig = {}) {
|
||||
this.maxAge = config.maxAge ?? 5 * 60 * 1000 // 5 minutes
|
||||
this.maxSize = config.maxSize ?? 500 // More entries than SearchCache since indexes are smaller
|
||||
this.enabled = config.enabled ?? true
|
||||
this.hitCountWeight = config.hitCountWeight ?? 0.3
|
||||
}
|
||||
|
||||
/**
|
||||
* Get cached entry
|
||||
*/
|
||||
get(key: string): any | undefined {
|
||||
if (!this.enabled) return undefined
|
||||
|
||||
const entry = this.cache.get(key)
|
||||
if (!entry) {
|
||||
this.misses++
|
||||
return undefined
|
||||
}
|
||||
|
||||
// Check if entry is expired
|
||||
if (Date.now() - entry.timestamp > this.maxAge) {
|
||||
this.cache.delete(key)
|
||||
this.misses++
|
||||
return undefined
|
||||
}
|
||||
|
||||
// Update hit count
|
||||
entry.hits++
|
||||
this.hits++
|
||||
return entry.data
|
||||
}
|
||||
|
||||
/**
|
||||
* Set cache entry
|
||||
*/
|
||||
set(key: string, data: any): void {
|
||||
if (!this.enabled) return
|
||||
|
||||
// Evict entries if at max size
|
||||
if (this.cache.size >= this.maxSize) {
|
||||
this.evictLeastValuable()
|
||||
}
|
||||
|
||||
this.cache.set(key, {
|
||||
data,
|
||||
timestamp: Date.now(),
|
||||
hits: 0
|
||||
})
|
||||
}
|
||||
|
||||
/**
|
||||
* Evict least valuable entry based on age and hit count
|
||||
*/
|
||||
private evictLeastValuable(): void {
|
||||
let leastValuableKey: string | null = null
|
||||
let lowestScore = Infinity
|
||||
|
||||
for (const [key, entry] of this.cache.entries()) {
|
||||
const age = Date.now() - entry.timestamp
|
||||
const ageScore = age / this.maxAge
|
||||
const hitScore = entry.hits * this.hitCountWeight
|
||||
const score = hitScore - ageScore
|
||||
|
||||
if (score < lowestScore) {
|
||||
lowestScore = score
|
||||
leastValuableKey = key
|
||||
}
|
||||
}
|
||||
|
||||
if (leastValuableKey) {
|
||||
this.cache.delete(leastValuableKey)
|
||||
this.evictions++
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Invalidate cache entries matching a pattern
|
||||
*/
|
||||
invalidatePattern(pattern: string): void {
|
||||
const keysToDelete: string[] = []
|
||||
for (const key of this.cache.keys()) {
|
||||
if (key.includes(pattern)) {
|
||||
keysToDelete.push(key)
|
||||
}
|
||||
}
|
||||
keysToDelete.forEach(key => this.cache.delete(key))
|
||||
}
|
||||
|
||||
/**
|
||||
* Clear all cache entries
|
||||
*/
|
||||
clear(): void {
|
||||
this.cache.clear()
|
||||
}
|
||||
|
||||
/**
|
||||
* Get cache statistics
|
||||
*/
|
||||
getStats() {
|
||||
return {
|
||||
size: this.cache.size,
|
||||
hits: this.hits,
|
||||
misses: this.misses,
|
||||
hitRate: this.hits / (this.hits + this.misses) || 0,
|
||||
evictions: this.evictions
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get estimated memory usage
|
||||
*/
|
||||
getMemoryUsage(): number {
|
||||
// Rough estimate: 100 bytes per entry + data size
|
||||
let totalSize = 0
|
||||
for (const entry of this.cache.values()) {
|
||||
totalSize += 100 // Base overhead
|
||||
totalSize += JSON.stringify(entry.data).length * 2 // Unicode chars
|
||||
}
|
||||
return totalSize
|
||||
}
|
||||
}
|
||||
204
src/utils/operationUtils.ts
Normal file
204
src/utils/operationUtils.ts
Normal file
|
|
@ -0,0 +1,204 @@
|
|||
/**
|
||||
* Utility functions for timeout and retry logic
|
||||
* Used by storage adapters to handle network operations reliably
|
||||
*/
|
||||
|
||||
import { BrainyError } from '../errors/brainyError.js'
|
||||
|
||||
export interface TimeoutConfig {
|
||||
get?: number
|
||||
add?: number
|
||||
delete?: number
|
||||
}
|
||||
|
||||
export interface RetryConfig {
|
||||
maxRetries?: number
|
||||
initialDelay?: number
|
||||
maxDelay?: number
|
||||
backoffMultiplier?: number
|
||||
}
|
||||
|
||||
export interface OperationConfig {
|
||||
timeouts?: TimeoutConfig
|
||||
retryPolicy?: RetryConfig
|
||||
}
|
||||
|
||||
// Default configuration values
|
||||
export const DEFAULT_TIMEOUTS: Required<TimeoutConfig> = {
|
||||
get: 30000, // 30 seconds
|
||||
add: 60000, // 1 minute
|
||||
delete: 30000 // 30 seconds
|
||||
}
|
||||
|
||||
export const DEFAULT_RETRY_POLICY: Required<RetryConfig> = {
|
||||
maxRetries: 3,
|
||||
initialDelay: 1000,
|
||||
maxDelay: 10000,
|
||||
backoffMultiplier: 2
|
||||
}
|
||||
|
||||
/**
|
||||
* Wraps a promise with a timeout
|
||||
*/
|
||||
export function withTimeout<T>(
|
||||
promise: Promise<T>,
|
||||
timeoutMs: number,
|
||||
operation: string
|
||||
): Promise<T> {
|
||||
return new Promise((resolve, reject) => {
|
||||
const timeoutId = setTimeout(() => {
|
||||
reject(BrainyError.timeout(operation, timeoutMs))
|
||||
}, timeoutMs)
|
||||
|
||||
promise
|
||||
.then((result) => {
|
||||
clearTimeout(timeoutId)
|
||||
resolve(result)
|
||||
})
|
||||
.catch((error) => {
|
||||
clearTimeout(timeoutId)
|
||||
reject(error)
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
/**
|
||||
* Calculates the delay for exponential backoff
|
||||
*/
|
||||
function calculateBackoffDelay(
|
||||
attemptNumber: number,
|
||||
initialDelay: number,
|
||||
maxDelay: number,
|
||||
backoffMultiplier: number
|
||||
): number {
|
||||
const delay = initialDelay * Math.pow(backoffMultiplier, attemptNumber - 1)
|
||||
return Math.min(delay, maxDelay)
|
||||
}
|
||||
|
||||
/**
|
||||
* Sleeps for the specified number of milliseconds
|
||||
*/
|
||||
function sleep(ms: number): Promise<void> {
|
||||
return new Promise(resolve => setTimeout(resolve, ms))
|
||||
}
|
||||
|
||||
/**
|
||||
* Executes an operation with retry logic and exponential backoff
|
||||
*/
|
||||
export async function withRetry<T>(
|
||||
operation: () => Promise<T>,
|
||||
operationName: string,
|
||||
config: RetryConfig = {}
|
||||
): Promise<T> {
|
||||
const {
|
||||
maxRetries = DEFAULT_RETRY_POLICY.maxRetries,
|
||||
initialDelay = DEFAULT_RETRY_POLICY.initialDelay,
|
||||
maxDelay = DEFAULT_RETRY_POLICY.maxDelay,
|
||||
backoffMultiplier = DEFAULT_RETRY_POLICY.backoffMultiplier
|
||||
} = config
|
||||
|
||||
let lastError: Error | undefined
|
||||
|
||||
for (let attempt = 1; attempt <= maxRetries + 1; attempt++) {
|
||||
try {
|
||||
return await operation()
|
||||
} catch (error) {
|
||||
lastError = error instanceof Error ? error : new Error(String(error))
|
||||
|
||||
// If this is the last attempt, don't retry
|
||||
if (attempt > maxRetries) {
|
||||
break
|
||||
}
|
||||
|
||||
// Check if the error is retryable
|
||||
if (!BrainyError.isRetryable(lastError)) {
|
||||
throw BrainyError.fromError(lastError, operationName)
|
||||
}
|
||||
|
||||
// Calculate delay for exponential backoff
|
||||
const delay = calculateBackoffDelay(attempt, initialDelay, maxDelay, backoffMultiplier)
|
||||
|
||||
console.warn(
|
||||
`Operation '${operationName}' failed on attempt ${attempt}/${maxRetries + 1}. ` +
|
||||
`Retrying in ${delay}ms. Error: ${lastError.message}`
|
||||
)
|
||||
|
||||
// Wait before retrying
|
||||
await sleep(delay)
|
||||
}
|
||||
}
|
||||
|
||||
// All retries exhausted
|
||||
throw BrainyError.retryExhausted(operationName, maxRetries, lastError)
|
||||
}
|
||||
|
||||
/**
|
||||
* Executes an operation with both timeout and retry logic
|
||||
*/
|
||||
export async function withTimeoutAndRetry<T>(
|
||||
operation: () => Promise<T>,
|
||||
operationName: string,
|
||||
timeoutMs: number,
|
||||
retryConfig: RetryConfig = {}
|
||||
): Promise<T> {
|
||||
return withRetry(
|
||||
() => withTimeout(operation(), timeoutMs, operationName),
|
||||
operationName,
|
||||
retryConfig
|
||||
)
|
||||
}
|
||||
|
||||
/**
|
||||
* Creates a configured operation executor for a specific operation type
|
||||
*/
|
||||
export function createOperationExecutor(
|
||||
operationType: keyof TimeoutConfig,
|
||||
config: OperationConfig = {}
|
||||
) {
|
||||
const timeouts = { ...DEFAULT_TIMEOUTS, ...config.timeouts }
|
||||
const retryPolicy = { ...DEFAULT_RETRY_POLICY, ...config.retryPolicy }
|
||||
const timeoutMs = timeouts[operationType]
|
||||
|
||||
return async function executeOperation<T>(
|
||||
operation: () => Promise<T>,
|
||||
operationName: string
|
||||
): Promise<T> {
|
||||
return withTimeoutAndRetry(operation, operationName, timeoutMs, retryPolicy)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Storage operation executors for different operation types
|
||||
*/
|
||||
export class StorageOperationExecutors {
|
||||
private getExecutor: ReturnType<typeof createOperationExecutor>
|
||||
private addExecutor: ReturnType<typeof createOperationExecutor>
|
||||
private deleteExecutor: ReturnType<typeof createOperationExecutor>
|
||||
|
||||
constructor(config: OperationConfig = {}) {
|
||||
this.getExecutor = createOperationExecutor('get', config)
|
||||
this.addExecutor = createOperationExecutor('add', config)
|
||||
this.deleteExecutor = createOperationExecutor('delete', config)
|
||||
}
|
||||
|
||||
/**
|
||||
* Execute a get operation with timeout and retry
|
||||
*/
|
||||
async executeGet<T>(operation: () => Promise<T>, operationName: string): Promise<T> {
|
||||
return this.getExecutor(operation, operationName)
|
||||
}
|
||||
|
||||
/**
|
||||
* Execute an add operation with timeout and retry
|
||||
*/
|
||||
async executeAdd<T>(operation: () => Promise<T>, operationName: string): Promise<T> {
|
||||
return this.addExecutor(operation, operationName)
|
||||
}
|
||||
|
||||
/**
|
||||
* Execute a delete operation with timeout and retry
|
||||
*/
|
||||
async executeDelete<T>(operation: () => Promise<T>, operationName: string): Promise<T> {
|
||||
return this.deleteExecutor(operation, operationName)
|
||||
}
|
||||
}
|
||||
496
src/utils/performanceMonitor.ts
Normal file
496
src/utils/performanceMonitor.ts
Normal file
|
|
@ -0,0 +1,496 @@
|
|||
/**
|
||||
* Performance Monitor
|
||||
* Automatically tracks and optimizes system performance
|
||||
* Provides real-time insights and auto-tuning recommendations
|
||||
*/
|
||||
|
||||
import { createModuleLogger } from './logger.js'
|
||||
import { getGlobalSocketManager } from './adaptiveSocketManager.js'
|
||||
import { getGlobalBackpressure } from './adaptiveBackpressure.js'
|
||||
|
||||
interface PerformanceMetrics {
|
||||
// Operation metrics
|
||||
totalOperations: number
|
||||
successfulOperations: number
|
||||
failedOperations: number
|
||||
averageLatency: number
|
||||
p95Latency: number
|
||||
p99Latency: number
|
||||
|
||||
// Throughput metrics
|
||||
operationsPerSecond: number
|
||||
bytesPerSecond: number
|
||||
|
||||
// Resource metrics
|
||||
memoryUsage: number
|
||||
cpuUsage: number
|
||||
socketUtilization: number
|
||||
queueDepth: number
|
||||
|
||||
// Health indicators
|
||||
errorRate: number
|
||||
healthScore: number // 0-100
|
||||
}
|
||||
|
||||
interface PerformanceTrend {
|
||||
metric: string
|
||||
direction: 'improving' | 'degrading' | 'stable'
|
||||
changeRate: number
|
||||
prediction: number
|
||||
}
|
||||
|
||||
/**
|
||||
* Comprehensive performance monitoring and optimization
|
||||
*/
|
||||
export class PerformanceMonitor {
|
||||
private logger = createModuleLogger('PerformanceMonitor')
|
||||
|
||||
// Current metrics
|
||||
private metrics: PerformanceMetrics = {
|
||||
totalOperations: 0,
|
||||
successfulOperations: 0,
|
||||
failedOperations: 0,
|
||||
averageLatency: 0,
|
||||
p95Latency: 0,
|
||||
p99Latency: 0,
|
||||
operationsPerSecond: 0,
|
||||
bytesPerSecond: 0,
|
||||
memoryUsage: 0,
|
||||
cpuUsage: 0,
|
||||
socketUtilization: 0,
|
||||
queueDepth: 0,
|
||||
errorRate: 0,
|
||||
healthScore: 100
|
||||
}
|
||||
|
||||
// Historical data for trend analysis
|
||||
private history: PerformanceMetrics[] = []
|
||||
private maxHistorySize = 1000
|
||||
|
||||
// Operation tracking
|
||||
private operationLatencies: number[] = []
|
||||
private operationSizes: number[] = []
|
||||
private lastReset = Date.now()
|
||||
private resetInterval = 60000 // Reset counters every minute
|
||||
|
||||
// CPU tracking
|
||||
private lastCpuUsage = process.cpuUsage ? process.cpuUsage() : null
|
||||
private lastCpuCheck = Date.now()
|
||||
|
||||
// Alert thresholds
|
||||
private thresholds = {
|
||||
errorRate: 0.05, // 5% error rate
|
||||
latencyP95: 5000, // 5 second P95
|
||||
memoryUsage: 0.8, // 80% memory
|
||||
cpuUsage: 0.9, // 90% CPU
|
||||
healthScore: 70 // Health score below 70
|
||||
}
|
||||
|
||||
// Optimization recommendations
|
||||
private recommendations: string[] = []
|
||||
|
||||
// Auto-optimization state
|
||||
private autoOptimizeEnabled = true
|
||||
private lastOptimization = Date.now()
|
||||
private optimizationInterval = 30000 // Optimize every 30 seconds
|
||||
|
||||
/**
|
||||
* Track an operation completion
|
||||
*/
|
||||
public trackOperation(
|
||||
success: boolean,
|
||||
latency: number,
|
||||
bytes: number = 0
|
||||
): void {
|
||||
// Update counters
|
||||
this.metrics.totalOperations++
|
||||
if (success) {
|
||||
this.metrics.successfulOperations++
|
||||
} else {
|
||||
this.metrics.failedOperations++
|
||||
}
|
||||
|
||||
// Track latency
|
||||
this.operationLatencies.push(latency)
|
||||
if (this.operationLatencies.length > 10000) {
|
||||
this.operationLatencies = this.operationLatencies.slice(-5000)
|
||||
}
|
||||
|
||||
// Track size
|
||||
if (bytes > 0) {
|
||||
this.operationSizes.push(bytes)
|
||||
if (this.operationSizes.length > 10000) {
|
||||
this.operationSizes = this.operationSizes.slice(-5000)
|
||||
}
|
||||
}
|
||||
|
||||
// Update metrics periodically
|
||||
this.updateMetrics()
|
||||
}
|
||||
|
||||
/**
|
||||
* Update all metrics
|
||||
*/
|
||||
private updateMetrics(): void {
|
||||
const now = Date.now()
|
||||
const timeSinceReset = (now - this.lastReset) / 1000
|
||||
|
||||
// Calculate latency percentiles
|
||||
if (this.operationLatencies.length > 0) {
|
||||
const sorted = [...this.operationLatencies].sort((a, b) => a - b)
|
||||
const p95Index = Math.floor(sorted.length * 0.95)
|
||||
const p99Index = Math.floor(sorted.length * 0.99)
|
||||
|
||||
this.metrics.averageLatency = sorted.reduce((a, b) => a + b, 0) / sorted.length
|
||||
this.metrics.p95Latency = sorted[p95Index] || 0
|
||||
this.metrics.p99Latency = sorted[p99Index] || 0
|
||||
}
|
||||
|
||||
// Calculate throughput
|
||||
if (timeSinceReset > 0) {
|
||||
this.metrics.operationsPerSecond = this.metrics.totalOperations / timeSinceReset
|
||||
|
||||
const totalBytes = this.operationSizes.reduce((a, b) => a + b, 0)
|
||||
this.metrics.bytesPerSecond = totalBytes / timeSinceReset
|
||||
}
|
||||
|
||||
// Calculate error rate
|
||||
this.metrics.errorRate = this.metrics.totalOperations > 0
|
||||
? this.metrics.failedOperations / this.metrics.totalOperations
|
||||
: 0
|
||||
|
||||
// Update resource metrics
|
||||
this.updateResourceMetrics()
|
||||
|
||||
// Calculate health score
|
||||
this.calculateHealthScore()
|
||||
|
||||
// Store in history
|
||||
this.history.push({ ...this.metrics })
|
||||
if (this.history.length > this.maxHistorySize) {
|
||||
this.history.shift()
|
||||
}
|
||||
|
||||
// Check for alerts
|
||||
this.checkAlerts()
|
||||
|
||||
// Auto-optimize if enabled
|
||||
if (this.autoOptimizeEnabled && now - this.lastOptimization > this.optimizationInterval) {
|
||||
this.autoOptimize()
|
||||
this.lastOptimization = now
|
||||
}
|
||||
|
||||
// Reset counters periodically
|
||||
if (now - this.lastReset > this.resetInterval) {
|
||||
this.resetCounters()
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Update resource metrics
|
||||
*/
|
||||
private updateResourceMetrics(): void {
|
||||
// Memory usage
|
||||
if (typeof process !== 'undefined' && process.memoryUsage) {
|
||||
const memUsage = process.memoryUsage()
|
||||
this.metrics.memoryUsage = memUsage.heapUsed / memUsage.heapTotal
|
||||
}
|
||||
|
||||
// CPU usage (Node.js only)
|
||||
if (this.lastCpuUsage && process.cpuUsage) {
|
||||
const currentCpuUsage = process.cpuUsage()
|
||||
const now = Date.now()
|
||||
const timeDiff = now - this.lastCpuCheck
|
||||
|
||||
if (timeDiff > 1000) { // Update CPU every second
|
||||
const userDiff = currentCpuUsage.user - this.lastCpuUsage.user
|
||||
const systemDiff = currentCpuUsage.system - this.lastCpuUsage.system
|
||||
const totalDiff = userDiff + systemDiff
|
||||
|
||||
// CPU percentage (approximate)
|
||||
this.metrics.cpuUsage = totalDiff / (timeDiff * 1000)
|
||||
|
||||
this.lastCpuUsage = currentCpuUsage
|
||||
this.lastCpuCheck = now
|
||||
}
|
||||
}
|
||||
|
||||
// Get metrics from socket manager
|
||||
const socketMetrics = getGlobalSocketManager().getMetrics()
|
||||
this.metrics.socketUtilization = socketMetrics.socketUtilization
|
||||
|
||||
// Get metrics from backpressure system
|
||||
const backpressureStatus = getGlobalBackpressure().getStatus()
|
||||
this.metrics.queueDepth = backpressureStatus.queueLength
|
||||
}
|
||||
|
||||
/**
|
||||
* Calculate overall health score
|
||||
*/
|
||||
private calculateHealthScore(): void {
|
||||
let score = 100
|
||||
|
||||
// Deduct points for high error rate
|
||||
if (this.metrics.errorRate > 0.01) {
|
||||
score -= Math.min(30, this.metrics.errorRate * 300)
|
||||
}
|
||||
|
||||
// Deduct points for high latency
|
||||
if (this.metrics.p95Latency > 3000) {
|
||||
score -= Math.min(20, (this.metrics.p95Latency - 3000) / 100)
|
||||
}
|
||||
|
||||
// Deduct points for high memory usage
|
||||
if (this.metrics.memoryUsage > 0.7) {
|
||||
score -= Math.min(20, (this.metrics.memoryUsage - 0.7) * 66)
|
||||
}
|
||||
|
||||
// Deduct points for high CPU usage
|
||||
if (this.metrics.cpuUsage > 0.8) {
|
||||
score -= Math.min(15, (this.metrics.cpuUsage - 0.8) * 75)
|
||||
}
|
||||
|
||||
// Deduct points for low throughput
|
||||
if (this.metrics.operationsPerSecond < 1 && this.metrics.totalOperations > 10) {
|
||||
score -= 10
|
||||
}
|
||||
|
||||
// Deduct points for queue depth
|
||||
if (this.metrics.queueDepth > 100) {
|
||||
score -= Math.min(15, this.metrics.queueDepth / 20)
|
||||
}
|
||||
|
||||
this.metrics.healthScore = Math.max(0, Math.min(100, score))
|
||||
}
|
||||
|
||||
/**
|
||||
* Check for alert conditions
|
||||
*/
|
||||
private checkAlerts(): void {
|
||||
const alerts: string[] = []
|
||||
|
||||
if (this.metrics.errorRate > this.thresholds.errorRate) {
|
||||
alerts.push(`High error rate: ${(this.metrics.errorRate * 100).toFixed(1)}%`)
|
||||
}
|
||||
|
||||
if (this.metrics.p95Latency > this.thresholds.latencyP95) {
|
||||
alerts.push(`High P95 latency: ${this.metrics.p95Latency}ms`)
|
||||
}
|
||||
|
||||
if (this.metrics.memoryUsage > this.thresholds.memoryUsage) {
|
||||
alerts.push(`High memory usage: ${(this.metrics.memoryUsage * 100).toFixed(1)}%`)
|
||||
}
|
||||
|
||||
if (this.metrics.cpuUsage > this.thresholds.cpuUsage) {
|
||||
alerts.push(`High CPU usage: ${(this.metrics.cpuUsage * 100).toFixed(1)}%`)
|
||||
}
|
||||
|
||||
if (this.metrics.healthScore < this.thresholds.healthScore) {
|
||||
alerts.push(`Low health score: ${this.metrics.healthScore.toFixed(0)}`)
|
||||
}
|
||||
|
||||
if (alerts.length > 0) {
|
||||
this.logger.warn('Performance alerts', { alerts, metrics: this.metrics })
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Auto-optimize system based on metrics
|
||||
*/
|
||||
private autoOptimize(): void {
|
||||
this.recommendations = []
|
||||
|
||||
// Analyze trends
|
||||
const trends = this.analyzeTrends()
|
||||
|
||||
// Generate recommendations based on metrics and trends
|
||||
if (this.metrics.errorRate > 0.02) {
|
||||
this.recommendations.push('Reduce load or increase timeouts due to high error rate')
|
||||
}
|
||||
|
||||
if (this.metrics.p95Latency > 3000) {
|
||||
this.recommendations.push('Increase batch size or socket limits to improve latency')
|
||||
}
|
||||
|
||||
if (this.metrics.memoryUsage > 0.7) {
|
||||
this.recommendations.push('Reduce cache sizes or batch sizes to free memory')
|
||||
}
|
||||
|
||||
if (this.metrics.queueDepth > 50) {
|
||||
this.recommendations.push('Increase concurrency limits to reduce queue depth')
|
||||
}
|
||||
|
||||
// Check for degrading trends
|
||||
trends.forEach(trend => {
|
||||
if (trend.direction === 'degrading' && Math.abs(trend.changeRate) > 0.1) {
|
||||
this.recommendations.push(`${trend.metric} is degrading at ${(trend.changeRate * 100).toFixed(1)}% per minute`)
|
||||
}
|
||||
})
|
||||
|
||||
// Log recommendations if any
|
||||
if (this.recommendations.length > 0) {
|
||||
this.logger.info('Performance optimization recommendations', {
|
||||
recommendations: this.recommendations,
|
||||
metrics: this.metrics
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Analyze performance trends
|
||||
*/
|
||||
private analyzeTrends(): PerformanceTrend[] {
|
||||
const trends: PerformanceTrend[] = []
|
||||
|
||||
if (this.history.length < 10) {
|
||||
return trends // Not enough data
|
||||
}
|
||||
|
||||
// Get recent history
|
||||
const recent = this.history.slice(-20)
|
||||
const older = this.history.slice(-40, -20)
|
||||
|
||||
// Compare key metrics
|
||||
const metricsToAnalyze = [
|
||||
'errorRate',
|
||||
'averageLatency',
|
||||
'operationsPerSecond',
|
||||
'memoryUsage',
|
||||
'healthScore'
|
||||
] as const
|
||||
|
||||
metricsToAnalyze.forEach(metric => {
|
||||
const recentAvg = recent.reduce((sum, m) => sum + m[metric], 0) / recent.length
|
||||
const olderAvg = older.length > 0
|
||||
? older.reduce((sum, m) => sum + m[metric], 0) / older.length
|
||||
: recentAvg
|
||||
|
||||
const changeRate = olderAvg !== 0 ? (recentAvg - olderAvg) / olderAvg : 0
|
||||
|
||||
let direction: 'improving' | 'degrading' | 'stable' = 'stable'
|
||||
if (Math.abs(changeRate) > 0.05) { // 5% threshold
|
||||
// For error rate and latency, increase is bad
|
||||
if (metric === 'errorRate' || metric === 'averageLatency' || metric === 'memoryUsage') {
|
||||
direction = changeRate > 0 ? 'degrading' : 'improving'
|
||||
} else {
|
||||
// For throughput and health score, increase is good
|
||||
direction = changeRate > 0 ? 'improving' : 'degrading'
|
||||
}
|
||||
}
|
||||
|
||||
// Simple linear prediction
|
||||
const prediction = recentAvg + (recentAvg * changeRate)
|
||||
|
||||
trends.push({
|
||||
metric,
|
||||
direction,
|
||||
changeRate,
|
||||
prediction
|
||||
})
|
||||
})
|
||||
|
||||
return trends
|
||||
}
|
||||
|
||||
/**
|
||||
* Reset counters
|
||||
*/
|
||||
private resetCounters(): void {
|
||||
this.metrics.totalOperations = 0
|
||||
this.metrics.successfulOperations = 0
|
||||
this.metrics.failedOperations = 0
|
||||
this.operationSizes = []
|
||||
this.lastReset = Date.now()
|
||||
}
|
||||
|
||||
/**
|
||||
* Get current metrics
|
||||
*/
|
||||
public getMetrics(): Readonly<PerformanceMetrics> {
|
||||
return { ...this.metrics }
|
||||
}
|
||||
|
||||
/**
|
||||
* Get performance trends
|
||||
*/
|
||||
public getTrends(): PerformanceTrend[] {
|
||||
return this.analyzeTrends()
|
||||
}
|
||||
|
||||
/**
|
||||
* Get recommendations
|
||||
*/
|
||||
public getRecommendations(): string[] {
|
||||
return [...this.recommendations]
|
||||
}
|
||||
|
||||
/**
|
||||
* Get performance report
|
||||
*/
|
||||
public getReport(): {
|
||||
metrics: PerformanceMetrics
|
||||
trends: PerformanceTrend[]
|
||||
recommendations: string[]
|
||||
socketConfig: any
|
||||
backpressureStatus: any
|
||||
} {
|
||||
return {
|
||||
metrics: this.getMetrics(),
|
||||
trends: this.getTrends(),
|
||||
recommendations: this.getRecommendations(),
|
||||
socketConfig: getGlobalSocketManager().getConfig(),
|
||||
backpressureStatus: getGlobalBackpressure().getStatus()
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Enable/disable auto-optimization
|
||||
*/
|
||||
public setAutoOptimize(enabled: boolean): void {
|
||||
this.autoOptimizeEnabled = enabled
|
||||
this.logger.info(`Auto-optimization ${enabled ? 'enabled' : 'disabled'}`)
|
||||
}
|
||||
|
||||
/**
|
||||
* Reset all metrics and history
|
||||
*/
|
||||
public reset(): void {
|
||||
this.metrics = {
|
||||
totalOperations: 0,
|
||||
successfulOperations: 0,
|
||||
failedOperations: 0,
|
||||
averageLatency: 0,
|
||||
p95Latency: 0,
|
||||
p99Latency: 0,
|
||||
operationsPerSecond: 0,
|
||||
bytesPerSecond: 0,
|
||||
memoryUsage: 0,
|
||||
cpuUsage: 0,
|
||||
socketUtilization: 0,
|
||||
queueDepth: 0,
|
||||
errorRate: 0,
|
||||
healthScore: 100
|
||||
}
|
||||
|
||||
this.history = []
|
||||
this.operationLatencies = []
|
||||
this.operationSizes = []
|
||||
this.recommendations = []
|
||||
this.lastReset = Date.now()
|
||||
|
||||
this.logger.info('Performance monitor reset')
|
||||
}
|
||||
}
|
||||
|
||||
// Global singleton instance
|
||||
let globalMonitor: PerformanceMonitor | null = null
|
||||
|
||||
/**
|
||||
* Get the global performance monitor instance
|
||||
*/
|
||||
export function getGlobalPerformanceMonitor(): PerformanceMonitor {
|
||||
if (!globalMonitor) {
|
||||
globalMonitor = new PerformanceMonitor()
|
||||
}
|
||||
return globalMonitor
|
||||
}
|
||||
398
src/utils/requestCoalescer.ts
Normal file
398
src/utils/requestCoalescer.ts
Normal file
|
|
@ -0,0 +1,398 @@
|
|||
/**
|
||||
* Request Coalescer
|
||||
* Batches and deduplicates operations to reduce S3 API calls
|
||||
* Automatically flushes based on size, time, or pressure
|
||||
*/
|
||||
|
||||
import { createModuleLogger } from './logger.js'
|
||||
|
||||
interface CoalescedOperation {
|
||||
type: 'write' | 'read' | 'delete'
|
||||
key: string
|
||||
data?: any
|
||||
resolve: (value: any) => void
|
||||
reject: (error: any) => void
|
||||
timestamp: number
|
||||
}
|
||||
|
||||
interface BatchStats {
|
||||
totalOperations: number
|
||||
coalescedOperations: number
|
||||
deduplicated: number
|
||||
batchesProcessed: number
|
||||
averageBatchSize: number
|
||||
}
|
||||
|
||||
/**
|
||||
* Coalesces multiple operations into efficient batches
|
||||
*/
|
||||
export class RequestCoalescer {
|
||||
private logger = createModuleLogger('RequestCoalescer')
|
||||
|
||||
// Operation queues by type
|
||||
private writeQueue = new Map<string, CoalescedOperation[]>()
|
||||
private readQueue = new Map<string, CoalescedOperation[]>()
|
||||
private deleteQueue = new Map<string, CoalescedOperation[]>()
|
||||
|
||||
// Batch configuration
|
||||
private maxBatchSize = 100
|
||||
private maxBatchAge = 100 // ms - flush quickly under load
|
||||
private minBatchSize = 10 // Don't flush until we have enough
|
||||
|
||||
// Flush timers
|
||||
private flushTimer: NodeJS.Timeout | null = null
|
||||
private lastFlush = Date.now()
|
||||
|
||||
// Statistics
|
||||
private stats: BatchStats = {
|
||||
totalOperations: 0,
|
||||
coalescedOperations: 0,
|
||||
deduplicated: 0,
|
||||
batchesProcessed: 0,
|
||||
averageBatchSize: 0
|
||||
}
|
||||
|
||||
// Processor function
|
||||
private processor: (batch: CoalescedOperation[]) => Promise<void>
|
||||
|
||||
constructor(
|
||||
processor: (batch: CoalescedOperation[]) => Promise<void>,
|
||||
options?: {
|
||||
maxBatchSize?: number
|
||||
maxBatchAge?: number
|
||||
minBatchSize?: number
|
||||
}
|
||||
) {
|
||||
this.processor = processor
|
||||
|
||||
if (options) {
|
||||
this.maxBatchSize = options.maxBatchSize || this.maxBatchSize
|
||||
this.maxBatchAge = options.maxBatchAge || this.maxBatchAge
|
||||
this.minBatchSize = options.minBatchSize || this.minBatchSize
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Add a write operation to be coalesced
|
||||
*/
|
||||
public async write(key: string, data: any): Promise<void> {
|
||||
return new Promise((resolve, reject) => {
|
||||
// Check if we already have a pending write for this key
|
||||
const existing = this.writeQueue.get(key)
|
||||
|
||||
if (existing && existing.length > 0) {
|
||||
// Replace the data but resolve all promises
|
||||
const last = existing[existing.length - 1]
|
||||
last.data = data // Use latest data
|
||||
|
||||
// Add this promise to be resolved
|
||||
existing.push({
|
||||
type: 'write',
|
||||
key,
|
||||
data,
|
||||
resolve,
|
||||
reject,
|
||||
timestamp: Date.now()
|
||||
})
|
||||
|
||||
this.stats.deduplicated++
|
||||
} else {
|
||||
// New write operation
|
||||
this.writeQueue.set(key, [{
|
||||
type: 'write',
|
||||
key,
|
||||
data,
|
||||
resolve,
|
||||
reject,
|
||||
timestamp: Date.now()
|
||||
}])
|
||||
}
|
||||
|
||||
this.stats.totalOperations++
|
||||
this.checkFlush()
|
||||
})
|
||||
}
|
||||
|
||||
/**
|
||||
* Add a read operation to be coalesced
|
||||
*/
|
||||
public async read(key: string): Promise<any> {
|
||||
return new Promise((resolve, reject) => {
|
||||
// Check if we already have a pending read for this key
|
||||
const existing = this.readQueue.get(key)
|
||||
|
||||
if (existing && existing.length > 0) {
|
||||
// Coalesce with existing read
|
||||
existing.push({
|
||||
type: 'read',
|
||||
key,
|
||||
resolve,
|
||||
reject,
|
||||
timestamp: Date.now()
|
||||
})
|
||||
|
||||
this.stats.deduplicated++
|
||||
} else {
|
||||
// New read operation
|
||||
this.readQueue.set(key, [{
|
||||
type: 'read',
|
||||
key,
|
||||
resolve,
|
||||
reject,
|
||||
timestamp: Date.now()
|
||||
}])
|
||||
}
|
||||
|
||||
this.stats.totalOperations++
|
||||
this.checkFlush()
|
||||
})
|
||||
}
|
||||
|
||||
/**
|
||||
* Add a delete operation to be coalesced
|
||||
*/
|
||||
public async delete(key: string): Promise<void> {
|
||||
return new Promise((resolve, reject) => {
|
||||
// Cancel any pending writes for this key
|
||||
if (this.writeQueue.has(key)) {
|
||||
const writes = this.writeQueue.get(key)!
|
||||
writes.forEach(op => op.reject(new Error('Cancelled by delete')))
|
||||
this.writeQueue.delete(key)
|
||||
this.stats.deduplicated += writes.length
|
||||
}
|
||||
|
||||
// Cancel any pending reads for this key
|
||||
if (this.readQueue.has(key)) {
|
||||
const reads = this.readQueue.get(key)!
|
||||
reads.forEach(op => op.resolve(null)) // Return null for deleted items
|
||||
this.readQueue.delete(key)
|
||||
this.stats.deduplicated += reads.length
|
||||
}
|
||||
|
||||
// Check if we already have a pending delete
|
||||
const existing = this.deleteQueue.get(key)
|
||||
|
||||
if (existing && existing.length > 0) {
|
||||
// Coalesce with existing delete
|
||||
existing.push({
|
||||
type: 'delete',
|
||||
key,
|
||||
resolve,
|
||||
reject,
|
||||
timestamp: Date.now()
|
||||
})
|
||||
|
||||
this.stats.deduplicated++
|
||||
} else {
|
||||
// New delete operation
|
||||
this.deleteQueue.set(key, [{
|
||||
type: 'delete',
|
||||
key,
|
||||
resolve,
|
||||
reject,
|
||||
timestamp: Date.now()
|
||||
}])
|
||||
}
|
||||
|
||||
this.stats.totalOperations++
|
||||
this.checkFlush()
|
||||
})
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if we should flush the queues
|
||||
*/
|
||||
private checkFlush(): void {
|
||||
const totalSize = this.writeQueue.size + this.readQueue.size + this.deleteQueue.size
|
||||
const now = Date.now()
|
||||
const age = now - this.lastFlush
|
||||
|
||||
// Immediate flush conditions
|
||||
if (totalSize >= this.maxBatchSize) {
|
||||
this.flush('size_limit')
|
||||
return
|
||||
}
|
||||
|
||||
// Age-based flush
|
||||
if (age >= this.maxBatchAge && totalSize >= this.minBatchSize) {
|
||||
this.flush('age_limit')
|
||||
return
|
||||
}
|
||||
|
||||
// Schedule a flush if not already scheduled
|
||||
if (!this.flushTimer && totalSize > 0) {
|
||||
const delay = Math.max(10, this.maxBatchAge - age)
|
||||
this.flushTimer = setTimeout(() => {
|
||||
this.flush('timer')
|
||||
}, delay)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Flush all queued operations
|
||||
*/
|
||||
public async flush(reason: string = 'manual'): Promise<void> {
|
||||
// Clear timer
|
||||
if (this.flushTimer) {
|
||||
clearTimeout(this.flushTimer)
|
||||
this.flushTimer = null
|
||||
}
|
||||
|
||||
// Collect all operations into a single batch
|
||||
const batch: CoalescedOperation[] = []
|
||||
|
||||
// Process deletes first (highest priority)
|
||||
this.deleteQueue.forEach((ops) => {
|
||||
// Only take the first operation per key (others are duplicates)
|
||||
if (ops.length > 0) {
|
||||
batch.push(ops[0])
|
||||
this.stats.coalescedOperations += ops.length
|
||||
}
|
||||
})
|
||||
|
||||
// Then writes
|
||||
this.writeQueue.forEach((ops) => {
|
||||
if (ops.length > 0) {
|
||||
// Use the last write (most recent data)
|
||||
const lastWrite = ops[ops.length - 1]
|
||||
batch.push(lastWrite)
|
||||
this.stats.coalescedOperations += ops.length
|
||||
}
|
||||
})
|
||||
|
||||
// Then reads
|
||||
this.readQueue.forEach((ops) => {
|
||||
if (ops.length > 0) {
|
||||
batch.push(ops[0])
|
||||
this.stats.coalescedOperations += ops.length
|
||||
}
|
||||
})
|
||||
|
||||
// Clear queues
|
||||
const allOps = [
|
||||
...Array.from(this.deleteQueue.values()).flat(),
|
||||
...Array.from(this.writeQueue.values()).flat(),
|
||||
...Array.from(this.readQueue.values()).flat()
|
||||
]
|
||||
|
||||
this.deleteQueue.clear()
|
||||
this.writeQueue.clear()
|
||||
this.readQueue.clear()
|
||||
|
||||
if (batch.length === 0) {
|
||||
return
|
||||
}
|
||||
|
||||
// Update stats
|
||||
this.stats.batchesProcessed++
|
||||
this.stats.averageBatchSize =
|
||||
(this.stats.averageBatchSize * (this.stats.batchesProcessed - 1) + batch.length) /
|
||||
this.stats.batchesProcessed
|
||||
|
||||
this.logger.debug(`Flushing batch of ${batch.length} operations (${allOps.length} total) - reason: ${reason}`)
|
||||
|
||||
// Process the batch
|
||||
try {
|
||||
await this.processor(batch)
|
||||
|
||||
// Resolve all promises
|
||||
allOps.forEach(op => {
|
||||
if (op.type === 'read') {
|
||||
// Find the result for this read
|
||||
const result = batch.find(b => b.key === op.key && b.type === 'read')
|
||||
op.resolve(result?.data || null)
|
||||
} else {
|
||||
op.resolve(undefined)
|
||||
}
|
||||
})
|
||||
} catch (error) {
|
||||
// Reject all promises
|
||||
allOps.forEach(op => op.reject(error))
|
||||
|
||||
this.logger.error('Batch processing failed:', error)
|
||||
}
|
||||
|
||||
this.lastFlush = Date.now()
|
||||
}
|
||||
|
||||
/**
|
||||
* Get current statistics
|
||||
*/
|
||||
public getStats(): BatchStats {
|
||||
return { ...this.stats }
|
||||
}
|
||||
|
||||
/**
|
||||
* Get current queue sizes
|
||||
*/
|
||||
public getQueueSizes(): {
|
||||
writes: number
|
||||
reads: number
|
||||
deletes: number
|
||||
total: number
|
||||
} {
|
||||
return {
|
||||
writes: this.writeQueue.size,
|
||||
reads: this.readQueue.size,
|
||||
deletes: this.deleteQueue.size,
|
||||
total: this.writeQueue.size + this.readQueue.size + this.deleteQueue.size
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Adjust batch parameters based on load
|
||||
*/
|
||||
public adjustParameters(pending: number): void {
|
||||
if (pending > 10000) {
|
||||
// Extreme load - batch aggressively
|
||||
this.maxBatchSize = 500
|
||||
this.maxBatchAge = 50
|
||||
this.minBatchSize = 50
|
||||
} else if (pending > 1000) {
|
||||
// High load - larger batches
|
||||
this.maxBatchSize = 200
|
||||
this.maxBatchAge = 100
|
||||
this.minBatchSize = 20
|
||||
} else if (pending > 100) {
|
||||
// Moderate load
|
||||
this.maxBatchSize = 100
|
||||
this.maxBatchAge = 200
|
||||
this.minBatchSize = 10
|
||||
} else {
|
||||
// Low load - optimize for latency
|
||||
this.maxBatchSize = 50
|
||||
this.maxBatchAge = 500
|
||||
this.minBatchSize = 5
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Force immediate flush of all operations
|
||||
*/
|
||||
public async forceFlush(): Promise<void> {
|
||||
await this.flush('force')
|
||||
}
|
||||
}
|
||||
|
||||
// Global coalescer instances by storage type
|
||||
const coalescers = new Map<string, RequestCoalescer>()
|
||||
|
||||
/**
|
||||
* Get or create a coalescer for a storage instance
|
||||
*/
|
||||
export function getCoalescer(
|
||||
storageId: string,
|
||||
processor: (batch: any[]) => Promise<void>
|
||||
): RequestCoalescer {
|
||||
if (!coalescers.has(storageId)) {
|
||||
coalescers.set(storageId, new RequestCoalescer(processor))
|
||||
}
|
||||
return coalescers.get(storageId)!
|
||||
}
|
||||
|
||||
/**
|
||||
* Clear all coalescers
|
||||
*/
|
||||
export function clearCoalescers(): void {
|
||||
coalescers.clear()
|
||||
}
|
||||
29
src/utils/requestDeduplicator.ts
Normal file
29
src/utils/requestDeduplicator.ts
Normal file
|
|
@ -0,0 +1,29 @@
|
|||
/**
|
||||
* Request Deduplicator Utility
|
||||
* Provides key generation for request deduplication
|
||||
*/
|
||||
|
||||
export class RequestDeduplicator {
|
||||
/**
|
||||
* Generate a unique key for search requests to enable deduplication
|
||||
*/
|
||||
static getSearchKey(
|
||||
query: string,
|
||||
k: number,
|
||||
options: any
|
||||
): string {
|
||||
// Create a consistent key from search parameters
|
||||
const optionsKey = options ? JSON.stringify({
|
||||
metadata: options.metadata,
|
||||
service: options.service,
|
||||
searchMode: options.searchMode,
|
||||
threshold: options.threshold,
|
||||
includeVectors: options.includeVectors,
|
||||
includeMetadata: options.includeMetadata,
|
||||
sortBy: options.sortBy,
|
||||
cursor: options.cursor
|
||||
}) : '{}'
|
||||
|
||||
return `search:${query}:${k}:${optionsKey}`
|
||||
}
|
||||
}
|
||||
308
src/utils/searchCache.ts
Normal file
308
src/utils/searchCache.ts
Normal file
|
|
@ -0,0 +1,308 @@
|
|||
/**
|
||||
* SearchCache - Caches search results for improved performance
|
||||
*/
|
||||
|
||||
import { SearchResult } from '../coreTypes.js'
|
||||
|
||||
export interface CacheEntry<T = any> {
|
||||
results: SearchResult<T>[]
|
||||
timestamp: number
|
||||
hits: number
|
||||
}
|
||||
|
||||
export interface SearchCacheConfig {
|
||||
maxAge?: number // Maximum age in milliseconds (default: 5 minutes)
|
||||
maxSize?: number // Maximum number of cached queries (default: 100)
|
||||
enabled?: boolean // Whether caching is enabled (default: true)
|
||||
hitCountWeight?: number // Weight for hit count in eviction policy (default: 0.3)
|
||||
}
|
||||
|
||||
export class SearchCache<T = any> {
|
||||
private cache = new Map<string, CacheEntry<T>>()
|
||||
private maxAge: number
|
||||
private maxSize: number
|
||||
private enabled: boolean
|
||||
private hitCountWeight: number
|
||||
|
||||
// Cache statistics
|
||||
private hits = 0
|
||||
private misses = 0
|
||||
private evictions = 0
|
||||
|
||||
constructor(config: SearchCacheConfig = {}) {
|
||||
this.maxAge = config.maxAge ?? 5 * 60 * 1000 // 5 minutes
|
||||
this.maxSize = config.maxSize ?? 100
|
||||
this.enabled = config.enabled ?? true
|
||||
this.hitCountWeight = config.hitCountWeight ?? 0.3
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate cache key from search parameters
|
||||
*/
|
||||
getCacheKey(
|
||||
query: any,
|
||||
k: number,
|
||||
options: Record<string, any> = {}
|
||||
): string {
|
||||
// Create a normalized key that ignores order of options
|
||||
const normalizedOptions = Object.keys(options)
|
||||
.sort()
|
||||
.reduce((acc, key) => {
|
||||
// Skip cache-related options
|
||||
if (key === 'skipCache' || key === 'useStreaming') return acc
|
||||
acc[key] = options[key]
|
||||
return acc
|
||||
}, {} as Record<string, any>)
|
||||
|
||||
return JSON.stringify({
|
||||
query: typeof query === 'object' ? JSON.stringify(query) : query,
|
||||
k,
|
||||
...normalizedOptions
|
||||
})
|
||||
}
|
||||
|
||||
/**
|
||||
* Get cached results if available and not expired
|
||||
*/
|
||||
get(key: string): SearchResult<T>[] | null {
|
||||
if (!this.enabled) return null
|
||||
|
||||
const entry = this.cache.get(key)
|
||||
if (!entry) {
|
||||
this.misses++
|
||||
return null
|
||||
}
|
||||
|
||||
// Check if expired
|
||||
if (Date.now() - entry.timestamp > this.maxAge) {
|
||||
this.cache.delete(key)
|
||||
this.misses++
|
||||
return null
|
||||
}
|
||||
|
||||
// Update hit count and statistics
|
||||
entry.hits++
|
||||
this.hits++
|
||||
return entry.results
|
||||
}
|
||||
|
||||
/**
|
||||
* Cache search results
|
||||
*/
|
||||
set(key: string, results: SearchResult<T>[]): void {
|
||||
if (!this.enabled) return
|
||||
|
||||
// Evict if cache is full
|
||||
if (this.cache.size >= this.maxSize) {
|
||||
this.evictOldest()
|
||||
}
|
||||
|
||||
this.cache.set(key, {
|
||||
results: [...results], // Deep copy to prevent mutations
|
||||
timestamp: Date.now(),
|
||||
hits: 0
|
||||
})
|
||||
}
|
||||
|
||||
/**
|
||||
* Evict the oldest entry based on timestamp and hit count
|
||||
*/
|
||||
private evictOldest(): void {
|
||||
let oldestKey: string | null = null
|
||||
let oldestScore = Infinity
|
||||
|
||||
const now = Date.now()
|
||||
|
||||
for (const [key, entry] of this.cache.entries()) {
|
||||
// Score combines age and inverse hit count
|
||||
const age = now - entry.timestamp
|
||||
const hitScore = entry.hits > 0 ? 1 / entry.hits : 1
|
||||
const score = age + (hitScore * this.hitCountWeight * this.maxAge)
|
||||
|
||||
if (score < oldestScore) {
|
||||
oldestScore = score
|
||||
oldestKey = key
|
||||
}
|
||||
}
|
||||
|
||||
if (oldestKey) {
|
||||
this.cache.delete(oldestKey)
|
||||
this.evictions++
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Clear all cached results
|
||||
*/
|
||||
clear(): void {
|
||||
this.cache.clear()
|
||||
this.hits = 0
|
||||
this.misses = 0
|
||||
this.evictions = 0
|
||||
}
|
||||
|
||||
/**
|
||||
* Invalidate cache entries that might be affected by data changes
|
||||
*/
|
||||
invalidate(pattern?: string | RegExp): void {
|
||||
if (!pattern) {
|
||||
this.clear()
|
||||
return
|
||||
}
|
||||
|
||||
const keysToDelete: string[] = []
|
||||
|
||||
for (const key of this.cache.keys()) {
|
||||
const shouldDelete = typeof pattern === 'string'
|
||||
? key.includes(pattern)
|
||||
: pattern.test(key)
|
||||
|
||||
if (shouldDelete) {
|
||||
keysToDelete.push(key)
|
||||
}
|
||||
}
|
||||
|
||||
keysToDelete.forEach(key => this.cache.delete(key))
|
||||
}
|
||||
|
||||
/**
|
||||
* Smart invalidation for real-time data updates
|
||||
* Only clears cache if it's getting stale or if data changes significantly
|
||||
*/
|
||||
invalidateOnDataChange(changeType?: 'add' | 'update' | 'delete'): void {
|
||||
// For now, clear all caches on data changes to ensure consistency
|
||||
// In the future, we could implement more sophisticated invalidation
|
||||
// based on the type of change and affected data
|
||||
this.clear()
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if cache entries have expired and remove them
|
||||
* This is especially important in distributed scenarios where
|
||||
* real-time updates might be delayed or missed
|
||||
*/
|
||||
cleanupExpiredEntries(): number {
|
||||
const now = Date.now()
|
||||
const keysToDelete: string[] = []
|
||||
|
||||
for (const [key, entry] of this.cache.entries()) {
|
||||
if (now - entry.timestamp > this.maxAge) {
|
||||
keysToDelete.push(key)
|
||||
}
|
||||
}
|
||||
|
||||
keysToDelete.forEach(key => this.cache.delete(key))
|
||||
return keysToDelete.length
|
||||
}
|
||||
|
||||
/**
|
||||
* Get cache statistics
|
||||
*/
|
||||
getStats() {
|
||||
const total = this.hits + this.misses
|
||||
return {
|
||||
hits: this.hits,
|
||||
misses: this.misses,
|
||||
evictions: this.evictions,
|
||||
hitRate: total > 0 ? this.hits / total : 0,
|
||||
size: this.cache.size,
|
||||
maxSize: this.maxSize,
|
||||
enabled: this.enabled
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Enable or disable caching
|
||||
*/
|
||||
setEnabled(enabled: boolean): void {
|
||||
Object.defineProperty(this, 'enabled', { value: enabled, writable: false })
|
||||
if (!enabled) {
|
||||
this.clear()
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get memory usage estimate in bytes
|
||||
*/
|
||||
getMemoryUsage(): number {
|
||||
let totalSize = 0
|
||||
|
||||
for (const [key, entry] of this.cache.entries()) {
|
||||
// Estimate key size
|
||||
totalSize += key.length * 2 // UTF-16 characters
|
||||
|
||||
// Estimate entry size
|
||||
totalSize += JSON.stringify(entry.results).length * 2
|
||||
totalSize += 16 // timestamp + hits (8 bytes each)
|
||||
}
|
||||
|
||||
return totalSize
|
||||
}
|
||||
|
||||
/**
|
||||
* Get current cache configuration
|
||||
*/
|
||||
getConfig(): SearchCacheConfig {
|
||||
return {
|
||||
enabled: this.enabled,
|
||||
maxSize: this.maxSize,
|
||||
maxAge: this.maxAge,
|
||||
hitCountWeight: this.hitCountWeight
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Update cache configuration dynamically
|
||||
*/
|
||||
updateConfig(newConfig: Partial<SearchCacheConfig>): void {
|
||||
if (newConfig.enabled !== undefined) {
|
||||
this.enabled = newConfig.enabled
|
||||
}
|
||||
if (newConfig.maxSize !== undefined) {
|
||||
this.maxSize = newConfig.maxSize
|
||||
// Trigger eviction if current size exceeds new limit
|
||||
this.evictIfNeeded()
|
||||
}
|
||||
if (newConfig.maxAge !== undefined) {
|
||||
this.maxAge = newConfig.maxAge
|
||||
// Clean up entries that are now expired with new TTL
|
||||
this.cleanupExpiredEntries()
|
||||
}
|
||||
if (newConfig.hitCountWeight !== undefined) {
|
||||
this.hitCountWeight = newConfig.hitCountWeight
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Evict entries if cache exceeds maxSize
|
||||
*/
|
||||
private evictIfNeeded(): void {
|
||||
if (this.cache.size <= this.maxSize) {
|
||||
return
|
||||
}
|
||||
|
||||
// Calculate eviction score for each entry (same logic as existing eviction)
|
||||
const entries = Array.from(this.cache.entries()).map(([key, entry]) => {
|
||||
const age = Date.now() - entry.timestamp
|
||||
const hitCount = entry.hits
|
||||
|
||||
// Eviction score: lower is more likely to be evicted
|
||||
// Combines age and hit count (weighted by hitCountWeight)
|
||||
const ageScore = age / this.maxAge
|
||||
const hitScore = 1 / (hitCount + 1) // Inverse of hits (more hits = lower score)
|
||||
const score = ageScore * (1 - this.hitCountWeight) + hitScore * this.hitCountWeight
|
||||
|
||||
return { key, entry, score }
|
||||
})
|
||||
|
||||
// Sort by score (lowest first - these will be evicted)
|
||||
entries.sort((a, b) => a.score - b.score)
|
||||
|
||||
// Evict entries until we're under the limit
|
||||
const toEvict = entries.slice(0, this.cache.size - this.maxSize)
|
||||
toEvict.forEach(({ key }) => {
|
||||
this.cache.delete(key)
|
||||
this.evictions++
|
||||
})
|
||||
}
|
||||
}
|
||||
44
src/utils/statistics.ts
Normal file
44
src/utils/statistics.ts
Normal file
|
|
@ -0,0 +1,44 @@
|
|||
/**
|
||||
* Utility functions for retrieving statistics from Brainy
|
||||
*/
|
||||
|
||||
import { BrainyData } from '../brainyData.js'
|
||||
|
||||
/**
|
||||
* Get statistics about the current state of a BrainyData instance
|
||||
* This function provides access to statistics at the root level of the library
|
||||
*
|
||||
* @param instance A BrainyData instance to get statistics from
|
||||
* @param options Additional options for retrieving statistics
|
||||
* @returns Object containing counts of nouns, verbs, metadata entries, and HNSW index size
|
||||
* @throws Error if the instance is not provided or if statistics retrieval fails
|
||||
*/
|
||||
export async function getStatistics(
|
||||
instance: BrainyData,
|
||||
options: {
|
||||
service?: string | string[] // Filter statistics by service(s)
|
||||
} = {}
|
||||
): Promise<{
|
||||
nounCount: number
|
||||
verbCount: number
|
||||
metadataCount: number
|
||||
hnswIndexSize: number
|
||||
serviceBreakdown?: {
|
||||
[service: string]: {
|
||||
nounCount: number
|
||||
verbCount: number
|
||||
metadataCount: number
|
||||
}
|
||||
}
|
||||
}> {
|
||||
if (!instance) {
|
||||
throw new Error('BrainyData instance must be provided to getStatistics')
|
||||
}
|
||||
|
||||
try {
|
||||
return await instance.getStatistics(options)
|
||||
} catch (error) {
|
||||
console.error('Failed to get statistics:', error)
|
||||
throw new Error(`Failed to get statistics: ${error}`)
|
||||
}
|
||||
}
|
||||
452
src/utils/statisticsCollector.ts
Normal file
452
src/utils/statisticsCollector.ts
Normal file
|
|
@ -0,0 +1,452 @@
|
|||
/**
|
||||
* Lightweight statistics collector for Brainy
|
||||
* Designed to have minimal performance impact even with millions of entries
|
||||
*/
|
||||
|
||||
import { StatisticsData } from '../coreTypes.js'
|
||||
|
||||
interface TimeSeriesData {
|
||||
timestamp: number
|
||||
count: number
|
||||
}
|
||||
|
||||
export class StatisticsCollector {
|
||||
// Content type tracking (lightweight counters)
|
||||
private contentTypes: Map<string, number> = new Map()
|
||||
|
||||
// Data freshness tracking (only track timestamps, not full data)
|
||||
private oldestTimestamp: number = Date.now()
|
||||
private newestTimestamp: number = Date.now()
|
||||
private updateTimestamps: TimeSeriesData[] = []
|
||||
|
||||
// Search performance tracking (rolling window)
|
||||
private searchMetrics = {
|
||||
totalSearches: 0,
|
||||
totalSearchTimeMs: 0,
|
||||
searchTimestamps: [] as TimeSeriesData[],
|
||||
topSearchTerms: new Map<string, number>()
|
||||
}
|
||||
|
||||
// Verb type tracking
|
||||
private verbTypes: Map<string, number> = new Map()
|
||||
|
||||
// Storage size estimates (updated periodically, not on every operation)
|
||||
private storageSizeCache = {
|
||||
lastUpdated: 0,
|
||||
sizes: {
|
||||
nouns: 0,
|
||||
verbs: 0,
|
||||
metadata: 0,
|
||||
index: 0
|
||||
}
|
||||
}
|
||||
|
||||
// Throttling metrics
|
||||
private throttlingMetrics = {
|
||||
currentlyThrottled: false,
|
||||
lastThrottleTime: 0,
|
||||
consecutiveThrottleEvents: 0,
|
||||
currentBackoffMs: 1000,
|
||||
totalThrottleEvents: 0,
|
||||
throttleEventsByHour: new Array(24).fill(0),
|
||||
throttleReasons: new Map<string, number>(),
|
||||
delayedOperations: 0,
|
||||
retriedOperations: 0,
|
||||
failedDueToThrottling: 0,
|
||||
totalDelayMs: 0,
|
||||
serviceThrottling: new Map<string, {
|
||||
throttleCount: number
|
||||
lastThrottle: number
|
||||
status: 'normal' | 'throttled' | 'recovering'
|
||||
}>()
|
||||
}
|
||||
|
||||
private readonly MAX_TIMESTAMPS = 1000 // Keep last 1000 timestamps
|
||||
private readonly MAX_SEARCH_TERMS = 100 // Track top 100 search terms
|
||||
private readonly SIZE_UPDATE_INTERVAL = 60000 // Update sizes every minute
|
||||
|
||||
/**
|
||||
* Track content type (very lightweight)
|
||||
*/
|
||||
trackContentType(type: string): void {
|
||||
this.contentTypes.set(type, (this.contentTypes.get(type) || 0) + 1)
|
||||
}
|
||||
|
||||
/**
|
||||
* Track data update timestamp (lightweight)
|
||||
*/
|
||||
trackUpdate(timestamp?: number): void {
|
||||
const ts = timestamp || Date.now()
|
||||
|
||||
// Update oldest/newest
|
||||
if (ts < this.oldestTimestamp) this.oldestTimestamp = ts
|
||||
if (ts > this.newestTimestamp) this.newestTimestamp = ts
|
||||
|
||||
// Add to rolling window
|
||||
this.updateTimestamps.push({ timestamp: ts, count: 1 })
|
||||
|
||||
// Keep window size limited
|
||||
if (this.updateTimestamps.length > this.MAX_TIMESTAMPS) {
|
||||
this.updateTimestamps.shift()
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Track search performance (lightweight)
|
||||
*/
|
||||
trackSearch(searchTerm: string, durationMs: number): void {
|
||||
this.searchMetrics.totalSearches++
|
||||
this.searchMetrics.totalSearchTimeMs += durationMs
|
||||
|
||||
// Add to rolling window
|
||||
this.searchMetrics.searchTimestamps.push({
|
||||
timestamp: Date.now(),
|
||||
count: 1
|
||||
})
|
||||
|
||||
// Keep window size limited
|
||||
if (this.searchMetrics.searchTimestamps.length > this.MAX_TIMESTAMPS) {
|
||||
this.searchMetrics.searchTimestamps.shift()
|
||||
}
|
||||
|
||||
// Track search term (limit to top N)
|
||||
const termCount = (this.searchMetrics.topSearchTerms.get(searchTerm) || 0) + 1
|
||||
this.searchMetrics.topSearchTerms.set(searchTerm, termCount)
|
||||
|
||||
// Prune if too many terms
|
||||
if (this.searchMetrics.topSearchTerms.size > this.MAX_SEARCH_TERMS * 2) {
|
||||
this.pruneSearchTerms()
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Track verb type (lightweight)
|
||||
*/
|
||||
trackVerbType(type: string): void {
|
||||
this.verbTypes.set(type, (this.verbTypes.get(type) || 0) + 1)
|
||||
}
|
||||
|
||||
/**
|
||||
* Update storage size estimates (called periodically, not on every operation)
|
||||
*/
|
||||
updateStorageSizes(sizes: {
|
||||
nouns: number
|
||||
verbs: number
|
||||
metadata: number
|
||||
index: number
|
||||
}): void {
|
||||
this.storageSizeCache = {
|
||||
lastUpdated: Date.now(),
|
||||
sizes
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Track a throttling event
|
||||
*/
|
||||
trackThrottlingEvent(reason: string, service?: string): void {
|
||||
this.throttlingMetrics.currentlyThrottled = true
|
||||
this.throttlingMetrics.consecutiveThrottleEvents++
|
||||
this.throttlingMetrics.lastThrottleTime = Date.now()
|
||||
this.throttlingMetrics.totalThrottleEvents++
|
||||
|
||||
// Track by hour
|
||||
const hourIndex = new Date().getHours()
|
||||
this.throttlingMetrics.throttleEventsByHour[hourIndex]++
|
||||
|
||||
// Track reason
|
||||
const reasonCount = this.throttlingMetrics.throttleReasons.get(reason) || 0
|
||||
this.throttlingMetrics.throttleReasons.set(reason, reasonCount + 1)
|
||||
|
||||
// Track service-level throttling
|
||||
if (service) {
|
||||
const serviceInfo = this.throttlingMetrics.serviceThrottling.get(service) || {
|
||||
throttleCount: 0,
|
||||
lastThrottle: 0,
|
||||
status: 'normal' as const
|
||||
}
|
||||
|
||||
serviceInfo.throttleCount++
|
||||
serviceInfo.lastThrottle = Date.now()
|
||||
serviceInfo.status = 'throttled'
|
||||
|
||||
this.throttlingMetrics.serviceThrottling.set(service, serviceInfo)
|
||||
}
|
||||
|
||||
// Exponential backoff
|
||||
this.throttlingMetrics.currentBackoffMs = Math.min(
|
||||
this.throttlingMetrics.currentBackoffMs * 2,
|
||||
30000 // Max 30 seconds
|
||||
)
|
||||
}
|
||||
|
||||
/**
|
||||
* Clear throttling state after successful operations
|
||||
*/
|
||||
clearThrottlingState(): void {
|
||||
if (this.throttlingMetrics.consecutiveThrottleEvents > 0) {
|
||||
this.throttlingMetrics.consecutiveThrottleEvents = 0
|
||||
this.throttlingMetrics.currentBackoffMs = 1000 // Reset to initial backoff
|
||||
this.throttlingMetrics.currentlyThrottled = false
|
||||
|
||||
// Update service statuses
|
||||
for (const [, info] of this.throttlingMetrics.serviceThrottling) {
|
||||
if (info.status === 'throttled') {
|
||||
info.status = 'recovering'
|
||||
} else if (info.status === 'recovering') {
|
||||
const timeSinceThrottle = Date.now() - info.lastThrottle
|
||||
if (timeSinceThrottle > 60000) { // 1 minute recovery period
|
||||
info.status = 'normal'
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Track delayed operation
|
||||
*/
|
||||
trackDelayedOperation(delayMs: number): void {
|
||||
this.throttlingMetrics.delayedOperations++
|
||||
this.throttlingMetrics.totalDelayMs += delayMs
|
||||
}
|
||||
|
||||
/**
|
||||
* Track retried operation
|
||||
*/
|
||||
trackRetriedOperation(): void {
|
||||
this.throttlingMetrics.retriedOperations++
|
||||
}
|
||||
|
||||
/**
|
||||
* Track operation failed due to throttling
|
||||
*/
|
||||
trackFailedDueToThrottling(): void {
|
||||
this.throttlingMetrics.failedDueToThrottling++
|
||||
}
|
||||
|
||||
/**
|
||||
* Update throttling metrics from storage adapter
|
||||
*/
|
||||
updateThrottlingMetrics(metrics: {
|
||||
currentlyThrottled: boolean
|
||||
lastThrottleTime: number
|
||||
consecutiveThrottleEvents: number
|
||||
currentBackoffMs: number
|
||||
totalThrottleEvents: number
|
||||
throttleEventsByHour: number[]
|
||||
throttleReasons: Record<string, number>
|
||||
delayedOperations: number
|
||||
retriedOperations: number
|
||||
failedDueToThrottling: number
|
||||
totalDelayMs: number
|
||||
}): void {
|
||||
this.throttlingMetrics.currentlyThrottled = metrics.currentlyThrottled
|
||||
this.throttlingMetrics.lastThrottleTime = metrics.lastThrottleTime
|
||||
this.throttlingMetrics.consecutiveThrottleEvents = metrics.consecutiveThrottleEvents
|
||||
this.throttlingMetrics.currentBackoffMs = metrics.currentBackoffMs
|
||||
this.throttlingMetrics.totalThrottleEvents = metrics.totalThrottleEvents
|
||||
this.throttlingMetrics.throttleEventsByHour = [...metrics.throttleEventsByHour]
|
||||
|
||||
// Update throttle reasons map
|
||||
this.throttlingMetrics.throttleReasons.clear()
|
||||
for (const [reason, count] of Object.entries(metrics.throttleReasons)) {
|
||||
this.throttlingMetrics.throttleReasons.set(reason, count)
|
||||
}
|
||||
|
||||
this.throttlingMetrics.delayedOperations = metrics.delayedOperations
|
||||
this.throttlingMetrics.retriedOperations = metrics.retriedOperations
|
||||
this.throttlingMetrics.failedDueToThrottling = metrics.failedDueToThrottling
|
||||
this.throttlingMetrics.totalDelayMs = metrics.totalDelayMs
|
||||
}
|
||||
|
||||
/**
|
||||
* Get comprehensive statistics
|
||||
*/
|
||||
getStatistics(): Partial<StatisticsData> {
|
||||
const now = Date.now()
|
||||
const hourAgo = now - 3600000
|
||||
const dayAgo = now - 86400000
|
||||
const weekAgo = now - 604800000
|
||||
const monthAgo = now - 2592000000
|
||||
|
||||
// Calculate data freshness
|
||||
const updatesLastHour = this.updateTimestamps.filter(t => t.timestamp > hourAgo).length
|
||||
const updatesLastDay = this.updateTimestamps.filter(t => t.timestamp > dayAgo).length
|
||||
|
||||
// Calculate age distribution
|
||||
const ageDistribution = {
|
||||
last24h: 0,
|
||||
last7d: 0,
|
||||
last30d: 0,
|
||||
older: 0
|
||||
}
|
||||
|
||||
// Estimate based on update patterns (not scanning all data)
|
||||
const totalUpdates = this.updateTimestamps.length
|
||||
if (totalUpdates > 0) {
|
||||
const recentUpdates = this.updateTimestamps.filter(t => t.timestamp > dayAgo).length
|
||||
const weekUpdates = this.updateTimestamps.filter(t => t.timestamp > weekAgo).length
|
||||
const monthUpdates = this.updateTimestamps.filter(t => t.timestamp > monthAgo).length
|
||||
|
||||
ageDistribution.last24h = Math.round((recentUpdates / totalUpdates) * 100)
|
||||
ageDistribution.last7d = Math.round(((weekUpdates - recentUpdates) / totalUpdates) * 100)
|
||||
ageDistribution.last30d = Math.round(((monthUpdates - weekUpdates) / totalUpdates) * 100)
|
||||
ageDistribution.older = 100 - ageDistribution.last24h - ageDistribution.last7d - ageDistribution.last30d
|
||||
}
|
||||
|
||||
// Calculate search metrics
|
||||
const searchesLastHour = this.searchMetrics.searchTimestamps.filter(t => t.timestamp > hourAgo).length
|
||||
const searchesLastDay = this.searchMetrics.searchTimestamps.filter(t => t.timestamp > dayAgo).length
|
||||
const avgSearchTime = this.searchMetrics.totalSearches > 0
|
||||
? this.searchMetrics.totalSearchTimeMs / this.searchMetrics.totalSearches
|
||||
: 0
|
||||
|
||||
// Get top search terms
|
||||
const topSearchTerms = Array.from(this.searchMetrics.topSearchTerms.entries())
|
||||
.sort((a, b) => b[1] - a[1])
|
||||
.slice(0, 10)
|
||||
.map(([term]) => term)
|
||||
|
||||
// Calculate storage metrics
|
||||
const totalSize = Object.values(this.storageSizeCache.sizes).reduce((a, b) => a + b, 0)
|
||||
|
||||
// Calculate average delay for throttling
|
||||
const averageDelayMs = this.throttlingMetrics.delayedOperations > 0
|
||||
? this.throttlingMetrics.totalDelayMs / this.throttlingMetrics.delayedOperations
|
||||
: 0
|
||||
|
||||
// Convert service throttling map to record
|
||||
const serviceThrottlingRecord: Record<string, {
|
||||
throttleCount: number
|
||||
lastThrottle: string
|
||||
status: 'normal' | 'throttled' | 'recovering'
|
||||
}> = {}
|
||||
|
||||
for (const [service, info] of this.throttlingMetrics.serviceThrottling) {
|
||||
serviceThrottlingRecord[service] = {
|
||||
throttleCount: info.throttleCount,
|
||||
lastThrottle: new Date(info.lastThrottle).toISOString(),
|
||||
status: info.status
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
contentTypes: Object.fromEntries(this.contentTypes),
|
||||
|
||||
dataFreshness: {
|
||||
oldestEntry: new Date(this.oldestTimestamp).toISOString(),
|
||||
newestEntry: new Date(this.newestTimestamp).toISOString(),
|
||||
updatesLastHour,
|
||||
updatesLastDay,
|
||||
ageDistribution
|
||||
},
|
||||
|
||||
storageMetrics: {
|
||||
totalSizeBytes: totalSize,
|
||||
nounsSizeBytes: this.storageSizeCache.sizes.nouns,
|
||||
verbsSizeBytes: this.storageSizeCache.sizes.verbs,
|
||||
metadataSizeBytes: this.storageSizeCache.sizes.metadata,
|
||||
indexSizeBytes: this.storageSizeCache.sizes.index
|
||||
},
|
||||
|
||||
searchMetrics: {
|
||||
totalSearches: this.searchMetrics.totalSearches,
|
||||
averageSearchTimeMs: avgSearchTime,
|
||||
searchesLastHour,
|
||||
searchesLastDay,
|
||||
topSearchTerms
|
||||
},
|
||||
|
||||
verbStatistics: {
|
||||
totalVerbs: Array.from(this.verbTypes.values()).reduce((a, b) => a + b, 0),
|
||||
verbTypes: Object.fromEntries(this.verbTypes),
|
||||
averageConnectionsPerVerb: 2 // Verbs connect 2 nouns
|
||||
},
|
||||
|
||||
throttlingMetrics: {
|
||||
storage: {
|
||||
currentlyThrottled: this.throttlingMetrics.currentlyThrottled,
|
||||
lastThrottleTime: this.throttlingMetrics.lastThrottleTime > 0
|
||||
? new Date(this.throttlingMetrics.lastThrottleTime).toISOString()
|
||||
: undefined,
|
||||
consecutiveThrottleEvents: this.throttlingMetrics.consecutiveThrottleEvents,
|
||||
currentBackoffMs: this.throttlingMetrics.currentBackoffMs,
|
||||
totalThrottleEvents: this.throttlingMetrics.totalThrottleEvents,
|
||||
throttleEventsByHour: [...this.throttlingMetrics.throttleEventsByHour],
|
||||
throttleReasons: Object.fromEntries(this.throttlingMetrics.throttleReasons)
|
||||
},
|
||||
operationImpact: {
|
||||
delayedOperations: this.throttlingMetrics.delayedOperations,
|
||||
retriedOperations: this.throttlingMetrics.retriedOperations,
|
||||
failedDueToThrottling: this.throttlingMetrics.failedDueToThrottling,
|
||||
averageDelayMs,
|
||||
totalDelayMs: this.throttlingMetrics.totalDelayMs
|
||||
},
|
||||
serviceThrottling: Object.keys(serviceThrottlingRecord).length > 0
|
||||
? serviceThrottlingRecord
|
||||
: undefined
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Merge statistics from storage (for distributed systems)
|
||||
*/
|
||||
mergeFromStorage(stored: Partial<StatisticsData>): void {
|
||||
// Merge content types
|
||||
if (stored.contentTypes) {
|
||||
for (const [type, count] of Object.entries(stored.contentTypes)) {
|
||||
this.contentTypes.set(type, count)
|
||||
}
|
||||
}
|
||||
|
||||
// Merge verb types
|
||||
if (stored.verbStatistics?.verbTypes) {
|
||||
for (const [type, count] of Object.entries(stored.verbStatistics.verbTypes)) {
|
||||
this.verbTypes.set(type, count)
|
||||
}
|
||||
}
|
||||
|
||||
// Merge search metrics
|
||||
if (stored.searchMetrics) {
|
||||
this.searchMetrics.totalSearches = stored.searchMetrics.totalSearches || 0
|
||||
this.searchMetrics.totalSearchTimeMs = (stored.searchMetrics.averageSearchTimeMs || 0) * this.searchMetrics.totalSearches
|
||||
}
|
||||
|
||||
// Merge data freshness
|
||||
if (stored.dataFreshness) {
|
||||
this.oldestTimestamp = new Date(stored.dataFreshness.oldestEntry).getTime()
|
||||
this.newestTimestamp = new Date(stored.dataFreshness.newestEntry).getTime()
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Reset statistics (for testing)
|
||||
*/
|
||||
reset(): void {
|
||||
this.contentTypes.clear()
|
||||
this.verbTypes.clear()
|
||||
this.updateTimestamps = []
|
||||
this.searchMetrics = {
|
||||
totalSearches: 0,
|
||||
totalSearchTimeMs: 0,
|
||||
searchTimestamps: [],
|
||||
topSearchTerms: new Map()
|
||||
}
|
||||
this.oldestTimestamp = Date.now()
|
||||
this.newestTimestamp = Date.now()
|
||||
}
|
||||
|
||||
private pruneSearchTerms(): void {
|
||||
// Keep only top N search terms
|
||||
const sorted = Array.from(this.searchMetrics.topSearchTerms.entries())
|
||||
.sort((a, b) => b[1] - a[1])
|
||||
.slice(0, this.MAX_SEARCH_TERMS)
|
||||
|
||||
this.searchMetrics.topSearchTerms.clear()
|
||||
for (const [term, count] of sorted) {
|
||||
this.searchMetrics.topSearchTerms.set(term, count)
|
||||
}
|
||||
}
|
||||
}
|
||||
74
src/utils/textEncoding.ts
Normal file
74
src/utils/textEncoding.ts
Normal file
|
|
@ -0,0 +1,74 @@
|
|||
import { isNode } from './environment.js'
|
||||
|
||||
// Simplified TextEncoder/TextDecoder utilities for Node.js compatibility
|
||||
// No longer needs complex TensorFlow.js patches - only basic TextEncoder/TextDecoder
|
||||
|
||||
/**
|
||||
* Flag to track if the patch has been applied
|
||||
*/
|
||||
let patchApplied = false
|
||||
|
||||
/**
|
||||
* Apply TextEncoder/TextDecoder patches for Node.js compatibility
|
||||
* Simplified version for Transformers.js/ONNX Runtime
|
||||
*/
|
||||
export async function applyTensorFlowPatch(): Promise<void> {
|
||||
// Apply patches for all non-browser environments that might need TextEncoder/TextDecoder
|
||||
const isBrowserEnv = typeof window !== 'undefined' && typeof document !== 'undefined'
|
||||
if (isBrowserEnv || patchApplied) {
|
||||
return // Browser environments don't need these patches, and don't patch twice
|
||||
}
|
||||
|
||||
if (!isNode()) {
|
||||
return // Only patch Node.js environments
|
||||
}
|
||||
|
||||
try {
|
||||
console.log('Brainy: Applying TextEncoder/TextDecoder patch for Node.js')
|
||||
|
||||
// Get the appropriate global object
|
||||
const globalObj = (() => {
|
||||
if (typeof globalThis !== 'undefined') return globalThis
|
||||
if (typeof global !== 'undefined') return global
|
||||
return {} as any
|
||||
})()
|
||||
|
||||
// Make sure TextEncoder and TextDecoder are available globally
|
||||
if (!globalObj.TextEncoder) {
|
||||
globalObj.TextEncoder = TextEncoder
|
||||
}
|
||||
if (!globalObj.TextDecoder) {
|
||||
globalObj.TextDecoder = TextDecoder
|
||||
}
|
||||
|
||||
// Also set them on the global object for older code
|
||||
if (typeof global !== 'undefined') {
|
||||
if (!global.TextEncoder) {
|
||||
global.TextEncoder = TextEncoder
|
||||
}
|
||||
if (!global.TextDecoder) {
|
||||
global.TextDecoder = TextDecoder
|
||||
}
|
||||
}
|
||||
|
||||
patchApplied = true
|
||||
console.log('Brainy: TextEncoder/TextDecoder patches applied successfully')
|
||||
} catch (error) {
|
||||
console.warn('Brainy: Failed to apply TextEncoder/TextDecoder patch:', error)
|
||||
}
|
||||
}
|
||||
|
||||
export function getTextEncoder(): TextEncoder {
|
||||
return new TextEncoder()
|
||||
}
|
||||
|
||||
export function getTextDecoder(): TextDecoder {
|
||||
return new TextDecoder()
|
||||
}
|
||||
|
||||
// Apply patch immediately if in Node.js
|
||||
if (isNode()) {
|
||||
applyTensorFlowPatch().catch((error) => {
|
||||
console.warn('Failed to apply TextEncoder/TextDecoder patch at module load:', error)
|
||||
})
|
||||
}
|
||||
44
src/utils/typeUtils.ts
Normal file
44
src/utils/typeUtils.ts
Normal file
|
|
@ -0,0 +1,44 @@
|
|||
/**
|
||||
* Type Utilities
|
||||
*
|
||||
* This module provides utility functions for working with the Brainy type system,
|
||||
* particularly for accessing lists of noun and verb types.
|
||||
*/
|
||||
|
||||
import { NounType, VerbType } from '../types/graphTypes.js'
|
||||
|
||||
/**
|
||||
* Returns an array of all available noun types
|
||||
*
|
||||
* @returns {string[]} Array of all noun type values
|
||||
*/
|
||||
export function getNounTypes(): string[] {
|
||||
return Object.values(NounType)
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns an array of all available verb types
|
||||
*
|
||||
* @returns {string[]} Array of all verb type values
|
||||
*/
|
||||
export function getVerbTypes(): string[] {
|
||||
return Object.values(VerbType)
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns a map of noun type keys to their string values
|
||||
*
|
||||
* @returns {Record<string, string>} Map of noun type keys to values
|
||||
*/
|
||||
export function getNounTypeMap(): Record<string, string> {
|
||||
return { ...NounType }
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns a map of verb type keys to their string values
|
||||
*
|
||||
* @returns {Record<string, string>} Map of verb type keys to values
|
||||
*/
|
||||
export function getVerbTypeMap(): Record<string, string> {
|
||||
return { ...VerbType }
|
||||
}
|
||||
386
src/utils/unifiedCache.ts
Normal file
386
src/utils/unifiedCache.ts
Normal file
|
|
@ -0,0 +1,386 @@
|
|||
/**
|
||||
* UnifiedCache - Single cache for both HNSW and MetadataIndex
|
||||
* Prevents resource competition with cost-aware eviction
|
||||
*/
|
||||
|
||||
import { prodLog } from './logger.js'
|
||||
|
||||
export interface CacheItem {
|
||||
key: string
|
||||
type: 'hnsw' | 'metadata' | 'embedding' | 'other'
|
||||
data: any
|
||||
size: number
|
||||
rebuildCost: number // milliseconds to rebuild
|
||||
lastAccess: number
|
||||
accessCount: number
|
||||
}
|
||||
|
||||
export interface UnifiedCacheConfig {
|
||||
maxSize?: number // bytes
|
||||
enableRequestCoalescing?: boolean
|
||||
enableFairnessCheck?: boolean
|
||||
fairnessCheckInterval?: number // ms
|
||||
persistPatterns?: boolean
|
||||
}
|
||||
|
||||
export class UnifiedCache {
|
||||
private cache = new Map<string, CacheItem>()
|
||||
private access = new Map<string, number>() // Access counts
|
||||
private loadingPromises = new Map<string, Promise<any>>()
|
||||
private typeAccessCounts = { hnsw: 0, metadata: 0, embedding: 0, other: 0 }
|
||||
private totalAccessCount = 0
|
||||
private currentSize = 0
|
||||
private readonly maxSize: number
|
||||
private readonly config: UnifiedCacheConfig
|
||||
|
||||
constructor(config: UnifiedCacheConfig = {}) {
|
||||
this.maxSize = config.maxSize || 2 * 1024 * 1024 * 1024 // 2GB default
|
||||
this.config = {
|
||||
enableRequestCoalescing: true,
|
||||
enableFairnessCheck: true,
|
||||
fairnessCheckInterval: 60000, // Check fairness every minute
|
||||
persistPatterns: true,
|
||||
...config
|
||||
}
|
||||
|
||||
if (this.config.enableFairnessCheck) {
|
||||
this.startFairnessMonitor()
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get item from cache with request coalescing
|
||||
*/
|
||||
async get(key: string, loadFn?: () => Promise<any>): Promise<any> {
|
||||
// Update access tracking
|
||||
this.access.set(key, (this.access.get(key) || 0) + 1)
|
||||
this.totalAccessCount++
|
||||
|
||||
// Check if in cache
|
||||
const item = this.cache.get(key)
|
||||
if (item) {
|
||||
item.lastAccess = Date.now()
|
||||
item.accessCount++
|
||||
this.typeAccessCounts[item.type]++
|
||||
return item.data
|
||||
}
|
||||
|
||||
// If no load function, return undefined
|
||||
if (!loadFn) {
|
||||
return undefined
|
||||
}
|
||||
|
||||
// Request coalescing - prevent stampede
|
||||
if (this.config.enableRequestCoalescing && this.loadingPromises.has(key)) {
|
||||
prodLog.debug('Request coalescing for key:', key)
|
||||
return this.loadingPromises.get(key)
|
||||
}
|
||||
|
||||
// Load data
|
||||
const loadPromise = loadFn()
|
||||
if (this.config.enableRequestCoalescing) {
|
||||
this.loadingPromises.set(key, loadPromise)
|
||||
}
|
||||
|
||||
try {
|
||||
const data = await loadPromise
|
||||
return data
|
||||
} finally {
|
||||
if (this.config.enableRequestCoalescing) {
|
||||
this.loadingPromises.delete(key)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Set item in cache with cost-aware eviction
|
||||
*/
|
||||
set(
|
||||
key: string,
|
||||
data: any,
|
||||
type: 'hnsw' | 'metadata' | 'embedding' | 'other',
|
||||
size: number,
|
||||
rebuildCost: number = 1
|
||||
): void {
|
||||
// Make room if needed
|
||||
while (this.currentSize + size > this.maxSize && this.cache.size > 0) {
|
||||
this.evictLowestValue()
|
||||
}
|
||||
|
||||
// Add to cache
|
||||
const item: CacheItem = {
|
||||
key,
|
||||
type,
|
||||
data,
|
||||
size,
|
||||
rebuildCost,
|
||||
lastAccess: Date.now(),
|
||||
accessCount: 1
|
||||
}
|
||||
|
||||
// Update or add
|
||||
const existing = this.cache.get(key)
|
||||
if (existing) {
|
||||
this.currentSize -= existing.size
|
||||
}
|
||||
|
||||
this.cache.set(key, item)
|
||||
this.currentSize += size
|
||||
this.typeAccessCounts[type]++
|
||||
this.totalAccessCount++
|
||||
}
|
||||
|
||||
/**
|
||||
* Evict item with lowest value (access count / rebuild cost)
|
||||
*/
|
||||
private evictLowestValue(): void {
|
||||
let victim: string | null = null
|
||||
let lowestScore = Infinity
|
||||
|
||||
for (const [key, item] of this.cache) {
|
||||
// Calculate value score: access frequency / rebuild cost
|
||||
const accessScore = (this.access.get(key) || 1)
|
||||
const score = accessScore / Math.max(item.rebuildCost, 1)
|
||||
|
||||
if (score < lowestScore) {
|
||||
lowestScore = score
|
||||
victim = key
|
||||
}
|
||||
}
|
||||
|
||||
if (victim) {
|
||||
const item = this.cache.get(victim)!
|
||||
prodLog.debug(`Evicting ${victim} (type: ${item.type}, score: ${lowestScore})`)
|
||||
|
||||
this.currentSize -= item.size
|
||||
this.cache.delete(victim)
|
||||
// Keep access count for a while to prevent re-caching cold items
|
||||
// this.access.delete(victim) // Don't delete immediately
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Size-aware eviction - try to match needed size
|
||||
*/
|
||||
evictForSize(bytesNeeded: number): boolean {
|
||||
const candidates: Array<[string, number, CacheItem]> = []
|
||||
|
||||
for (const [key, item] of this.cache) {
|
||||
const score = (this.access.get(key) || 1) / item.rebuildCost
|
||||
candidates.push([key, score, item])
|
||||
}
|
||||
|
||||
// Sort by score (lower is worse)
|
||||
candidates.sort((a, b) => a[1] - b[1])
|
||||
|
||||
let freedBytes = 0
|
||||
const toEvict: string[] = []
|
||||
|
||||
// Try to free exactly what we need
|
||||
for (const [key, , item] of candidates) {
|
||||
toEvict.push(key)
|
||||
freedBytes += item.size
|
||||
if (freedBytes >= bytesNeeded) {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
// Evict selected items
|
||||
for (const key of toEvict) {
|
||||
const item = this.cache.get(key)!
|
||||
this.currentSize -= item.size
|
||||
this.cache.delete(key)
|
||||
}
|
||||
|
||||
return freedBytes >= bytesNeeded
|
||||
}
|
||||
|
||||
/**
|
||||
* Fairness monitoring - prevent one type from hogging cache
|
||||
*/
|
||||
private startFairnessMonitor(): void {
|
||||
setInterval(() => {
|
||||
this.checkFairness()
|
||||
}, this.config.fairnessCheckInterval!)
|
||||
}
|
||||
|
||||
private checkFairness(): void {
|
||||
// Calculate type ratios in cache
|
||||
const typeSizes = { hnsw: 0, metadata: 0, embedding: 0, other: 0 }
|
||||
const typeCounts = { hnsw: 0, metadata: 0, embedding: 0, other: 0 }
|
||||
|
||||
for (const item of this.cache.values()) {
|
||||
typeSizes[item.type] += item.size
|
||||
typeCounts[item.type]++
|
||||
}
|
||||
|
||||
// Calculate access ratios
|
||||
const totalAccess = this.totalAccessCount || 1
|
||||
const accessRatios = {
|
||||
hnsw: this.typeAccessCounts.hnsw / totalAccess,
|
||||
metadata: this.typeAccessCounts.metadata / totalAccess,
|
||||
embedding: this.typeAccessCounts.embedding / totalAccess,
|
||||
other: this.typeAccessCounts.other / totalAccess
|
||||
}
|
||||
|
||||
// Calculate size ratios
|
||||
const totalSize = this.currentSize || 1
|
||||
const sizeRatios = {
|
||||
hnsw: typeSizes.hnsw / totalSize,
|
||||
metadata: typeSizes.metadata / totalSize,
|
||||
embedding: typeSizes.embedding / totalSize,
|
||||
other: typeSizes.other / totalSize
|
||||
}
|
||||
|
||||
// Check for starvation (90% cache but <10% accesses)
|
||||
for (const type of ['hnsw', 'metadata', 'embedding', 'other'] as const) {
|
||||
if (sizeRatios[type] > 0.9 && accessRatios[type] < 0.1) {
|
||||
prodLog.warn(`Type ${type} is hogging cache (${(sizeRatios[type] * 100).toFixed(1)}% size, ${(accessRatios[type] * 100).toFixed(1)}% access)`)
|
||||
this.evictType(type)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Force evict items of a specific type
|
||||
*/
|
||||
private evictType(type: 'hnsw' | 'metadata' | 'embedding' | 'other'): void {
|
||||
const candidates: Array<[string, number, CacheItem]> = []
|
||||
|
||||
for (const [key, item] of this.cache) {
|
||||
if (item.type === type) {
|
||||
const score = (this.access.get(key) || 1) / item.rebuildCost
|
||||
candidates.push([key, score, item])
|
||||
}
|
||||
}
|
||||
|
||||
// Sort by score (lower is worse)
|
||||
candidates.sort((a, b) => a[1] - b[1])
|
||||
|
||||
// Evict bottom 20% of this type
|
||||
const evictCount = Math.max(1, Math.floor(candidates.length * 0.2))
|
||||
|
||||
for (let i = 0; i < evictCount && i < candidates.length; i++) {
|
||||
const [key, , item] = candidates[i]
|
||||
this.currentSize -= item.size
|
||||
this.cache.delete(key)
|
||||
prodLog.debug(`Fairness eviction: ${key} (type: ${type})`)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Delete specific item from cache
|
||||
*/
|
||||
delete(key: string): boolean {
|
||||
const item = this.cache.get(key)
|
||||
if (item) {
|
||||
this.currentSize -= item.size
|
||||
this.cache.delete(key)
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
/**
|
||||
* Clear cache or specific type
|
||||
*/
|
||||
clear(type?: 'hnsw' | 'metadata' | 'embedding' | 'other'): void {
|
||||
if (!type) {
|
||||
this.cache.clear()
|
||||
this.currentSize = 0
|
||||
return
|
||||
}
|
||||
|
||||
for (const [key, item] of this.cache) {
|
||||
if (item.type === type) {
|
||||
this.currentSize -= item.size
|
||||
this.cache.delete(key)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get cache statistics
|
||||
*/
|
||||
getStats() {
|
||||
const typeSizes = { hnsw: 0, metadata: 0, embedding: 0, other: 0 }
|
||||
const typeCounts = { hnsw: 0, metadata: 0, embedding: 0, other: 0 }
|
||||
|
||||
for (const item of this.cache.values()) {
|
||||
typeSizes[item.type] += item.size
|
||||
typeCounts[item.type]++
|
||||
}
|
||||
|
||||
return {
|
||||
totalSize: this.currentSize,
|
||||
maxSize: this.maxSize,
|
||||
utilization: this.currentSize / this.maxSize,
|
||||
itemCount: this.cache.size,
|
||||
typeSizes,
|
||||
typeCounts,
|
||||
typeAccessCounts: this.typeAccessCounts,
|
||||
totalAccessCount: this.totalAccessCount,
|
||||
hitRate: this.cache.size > 0 ?
|
||||
Array.from(this.cache.values()).reduce((sum, item) => sum + item.accessCount, 0) / this.totalAccessCount : 0
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Save access patterns for cold start optimization
|
||||
*/
|
||||
async saveAccessPatterns(): Promise<any> {
|
||||
if (!this.config.persistPatterns) return
|
||||
|
||||
const patterns = Array.from(this.cache.entries())
|
||||
.map(([key, item]) => ({
|
||||
key,
|
||||
type: item.type,
|
||||
accessCount: this.access.get(key) || 0,
|
||||
size: item.size,
|
||||
rebuildCost: item.rebuildCost
|
||||
}))
|
||||
.sort((a, b) => b.accessCount - a.accessCount)
|
||||
|
||||
return {
|
||||
patterns,
|
||||
typeAccessCounts: this.typeAccessCounts,
|
||||
timestamp: Date.now()
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Load access patterns for warm start
|
||||
*/
|
||||
async loadAccessPatterns(patterns: any): Promise<void> {
|
||||
if (!patterns?.patterns) return
|
||||
|
||||
// Pre-populate access counts
|
||||
for (const pattern of patterns.patterns) {
|
||||
this.access.set(pattern.key, pattern.accessCount)
|
||||
}
|
||||
|
||||
// Restore type access counts
|
||||
if (patterns.typeAccessCounts) {
|
||||
this.typeAccessCounts = patterns.typeAccessCounts
|
||||
}
|
||||
|
||||
prodLog.debug('Loaded access patterns:', patterns.patterns.length, 'items')
|
||||
}
|
||||
}
|
||||
|
||||
// Export singleton for global coordination
|
||||
let globalCache: UnifiedCache | null = null
|
||||
|
||||
export function getGlobalCache(config?: UnifiedCacheConfig): UnifiedCache {
|
||||
if (!globalCache) {
|
||||
globalCache = new UnifiedCache(config)
|
||||
}
|
||||
return globalCache
|
||||
}
|
||||
|
||||
export function clearGlobalCache(): void {
|
||||
if (globalCache) {
|
||||
globalCache.clear()
|
||||
globalCache = null
|
||||
}
|
||||
}
|
||||
26
src/utils/version.ts
Normal file
26
src/utils/version.ts
Normal file
|
|
@ -0,0 +1,26 @@
|
|||
/**
|
||||
* Version utilities for Brainy
|
||||
*/
|
||||
|
||||
// Package version - this should be updated during the build process
|
||||
const BRAINY_VERSION = '0.41.0'
|
||||
|
||||
/**
|
||||
* Get the current Brainy package version
|
||||
* @returns The current version string
|
||||
*/
|
||||
export function getBrainyVersion(): string {
|
||||
return BRAINY_VERSION
|
||||
}
|
||||
|
||||
/**
|
||||
* Get version information for augmentation metadata
|
||||
* @param service The service/augmentation name
|
||||
* @returns Version metadata object
|
||||
*/
|
||||
export function getAugmentationVersion(service: string): { augmentation: string; version: string } {
|
||||
return {
|
||||
augmentation: service,
|
||||
version: getBrainyVersion()
|
||||
}
|
||||
}
|
||||
512
src/utils/workerUtils.ts
Normal file
512
src/utils/workerUtils.ts
Normal file
|
|
@ -0,0 +1,512 @@
|
|||
/**
|
||||
* Utility functions for executing functions in Worker Threads (Node.js) or Web Workers (Browser)
|
||||
* This implementation leverages Node.js 24's improved Worker Threads API for better performance
|
||||
*/
|
||||
|
||||
import { isBrowser, isNode } from './environment.js'
|
||||
import { prodLog } from './logger.js'
|
||||
|
||||
// Worker pool to reuse workers
|
||||
const workerPool: Map<string, any> = new Map()
|
||||
const MAX_POOL_SIZE = 4 // Adjust based on system capabilities
|
||||
|
||||
/**
|
||||
* Execute a function in a separate thread
|
||||
*
|
||||
* @param fnString The function to execute as a string
|
||||
* @param args The arguments to pass to the function
|
||||
* @returns A promise that resolves with the result of the function
|
||||
*/
|
||||
export function executeInThread<T>(fnString: string, args: any): Promise<T> {
|
||||
if (isNode()) {
|
||||
return executeInNodeWorker<T>(fnString, args)
|
||||
} else if (isBrowser() && typeof window !== 'undefined' && window.Worker) {
|
||||
return executeInWebWorker<T>(fnString, args)
|
||||
} else {
|
||||
// Fallback to main thread execution
|
||||
try {
|
||||
// Try different approaches to create a function from string
|
||||
let fn
|
||||
try {
|
||||
// First try with 'return' prefix
|
||||
fn = new Function('return ' + fnString)()
|
||||
} catch (functionError) {
|
||||
console.warn(
|
||||
'Fallback: Error creating function with return syntax, trying alternative approaches',
|
||||
functionError
|
||||
)
|
||||
|
||||
try {
|
||||
// Try wrapping in parentheses for function expressions
|
||||
fn = new Function('return (' + fnString + ')')()
|
||||
} catch (wrapError) {
|
||||
console.warn(
|
||||
'Fallback: Error creating function with parentheses wrapping',
|
||||
wrapError
|
||||
)
|
||||
|
||||
try {
|
||||
// Try direct approach for named functions
|
||||
fn = new Function(fnString)()
|
||||
} catch (directError) {
|
||||
console.warn(
|
||||
'Fallback: Direct approach failed, trying with function wrapper',
|
||||
directError
|
||||
)
|
||||
|
||||
try {
|
||||
// Try wrapping in a function that returns the function expression
|
||||
fn = new Function(
|
||||
'return function(args) { return (' + fnString + ')(args); }'
|
||||
)()
|
||||
} catch (wrapperError) {
|
||||
console.error(
|
||||
'Fallback: All approaches to create function failed',
|
||||
wrapperError
|
||||
)
|
||||
throw new Error(
|
||||
'Failed to create function from string: ' +
|
||||
(functionError as Error).message
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return Promise.resolve(fn(args) as T)
|
||||
} catch (error) {
|
||||
return Promise.reject(error)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Execute a function in a Node.js Worker Thread
|
||||
* Optimized for Node.js 24 with improved Worker Threads performance
|
||||
*/
|
||||
function executeInNodeWorker<T>(fnString: string, args: any): Promise<T> {
|
||||
return new Promise<T>((resolve, reject) => {
|
||||
try {
|
||||
// Dynamically import worker_threads (Node.js only)
|
||||
import('node:worker_threads')
|
||||
.then(({ Worker, isMainThread, parentPort, workerData }) => {
|
||||
if (!isMainThread && parentPort) {
|
||||
// We're inside a worker, execute the function
|
||||
const fn = new Function('return ' + workerData.fnString)()
|
||||
const result = fn(workerData.args)
|
||||
parentPort.postMessage({ result })
|
||||
return
|
||||
}
|
||||
|
||||
// Get a worker from the pool or create a new one
|
||||
const workerId = `worker-${Math.random().toString(36).substring(2, 9)}`
|
||||
let worker: any
|
||||
|
||||
if (workerPool.size < MAX_POOL_SIZE) {
|
||||
// Create a new worker
|
||||
worker = new Worker(
|
||||
`
|
||||
import { parentPort, workerData } from 'node:worker_threads';
|
||||
|
||||
// Add TensorFlow.js platform patch for Node.js
|
||||
if (typeof global !== 'undefined') {
|
||||
try {
|
||||
// Define a custom PlatformNode class
|
||||
class PlatformNode {
|
||||
constructor() {
|
||||
// Create a util object with necessary methods
|
||||
this.util = {
|
||||
// Add isFloat32Array and isTypedArray directly to util
|
||||
isFloat32Array: (arr) => {
|
||||
return !!(
|
||||
arr instanceof Float32Array ||
|
||||
(arr &&
|
||||
Object.prototype.toString.call(arr) === '[object Float32Array]')
|
||||
);
|
||||
},
|
||||
isTypedArray: (arr) => {
|
||||
return !!(ArrayBuffer.isView(arr) && !(arr instanceof DataView));
|
||||
},
|
||||
// Use native TextEncoder and TextDecoder
|
||||
TextEncoder: TextEncoder,
|
||||
TextDecoder: TextDecoder
|
||||
};
|
||||
|
||||
// Initialize encoders using native constructors
|
||||
this.textEncoder = new TextEncoder();
|
||||
this.textDecoder = new TextDecoder();
|
||||
}
|
||||
|
||||
// Define isFloat32Array directly on the instance
|
||||
isFloat32Array(arr) {
|
||||
return !!(
|
||||
arr instanceof Float32Array ||
|
||||
(arr && Object.prototype.toString.call(arr) === '[object Float32Array]')
|
||||
);
|
||||
}
|
||||
|
||||
// Define isTypedArray directly on the instance
|
||||
isTypedArray(arr) {
|
||||
return !!(ArrayBuffer.isView(arr) && !(arr instanceof DataView));
|
||||
}
|
||||
}
|
||||
|
||||
// Assign the PlatformNode class to the global object
|
||||
global.PlatformNode = PlatformNode;
|
||||
|
||||
// Also create an instance and assign it to global.platformNode
|
||||
global.platformNode = new PlatformNode();
|
||||
|
||||
// Ensure global.util exists and has the necessary methods
|
||||
if (!global.util) {
|
||||
global.util = {};
|
||||
}
|
||||
|
||||
// Add isFloat32Array method if it doesn't exist
|
||||
if (!global.util.isFloat32Array) {
|
||||
global.util.isFloat32Array = (arr) => {
|
||||
return !!(
|
||||
arr instanceof Float32Array ||
|
||||
(arr && Object.prototype.toString.call(arr) === '[object Float32Array]')
|
||||
);
|
||||
};
|
||||
}
|
||||
|
||||
// Add isTypedArray method if it doesn't exist
|
||||
if (!global.util.isTypedArray) {
|
||||
global.util.isTypedArray = (arr) => {
|
||||
return !!(ArrayBuffer.isView(arr) && !(arr instanceof DataView));
|
||||
};
|
||||
}
|
||||
} catch (error) {
|
||||
console.warn('Failed to apply TensorFlow.js platform patch:', error);
|
||||
}
|
||||
}
|
||||
|
||||
const fn = new Function('return ' + workerData.fnString)();
|
||||
const result = fn(workerData.args);
|
||||
parentPort.postMessage({ result });
|
||||
`,
|
||||
{
|
||||
eval: true,
|
||||
workerData: { fnString, args }
|
||||
}
|
||||
)
|
||||
|
||||
workerPool.set(workerId, worker)
|
||||
} else {
|
||||
// Reuse an existing worker
|
||||
const poolKeys = Array.from(workerPool.keys())
|
||||
const randomKey =
|
||||
poolKeys[Math.floor(Math.random() * poolKeys.length)]
|
||||
worker = workerPool.get(randomKey)
|
||||
|
||||
// Terminate and recreate if the worker is busy
|
||||
if (worker._busy) {
|
||||
worker.terminate()
|
||||
worker = new Worker(
|
||||
`
|
||||
import { parentPort, workerData } from 'node:worker_threads';
|
||||
|
||||
// Add TensorFlow.js platform patch for Node.js
|
||||
if (typeof global !== 'undefined') {
|
||||
try {
|
||||
// Define a custom PlatformNode class
|
||||
class PlatformNode {
|
||||
constructor() {
|
||||
// Create a util object with necessary methods
|
||||
this.util = {
|
||||
// Use native TextEncoder and TextDecoder
|
||||
TextEncoder: TextEncoder,
|
||||
TextDecoder: TextDecoder
|
||||
};
|
||||
|
||||
// Initialize encoders using native constructors
|
||||
this.textEncoder = new TextEncoder();
|
||||
this.textDecoder = new TextDecoder();
|
||||
}
|
||||
|
||||
// Define isFloat32Array directly on the instance
|
||||
isFloat32Array(arr) {
|
||||
return !!(
|
||||
arr instanceof Float32Array ||
|
||||
(arr && Object.prototype.toString.call(arr) === '[object Float32Array]')
|
||||
);
|
||||
}
|
||||
|
||||
// Define isTypedArray directly on the instance
|
||||
isTypedArray(arr) {
|
||||
return !!(ArrayBuffer.isView(arr) && !(arr instanceof DataView));
|
||||
}
|
||||
}
|
||||
|
||||
// Assign the PlatformNode class to the global object
|
||||
global.PlatformNode = PlatformNode;
|
||||
|
||||
// Also create an instance and assign it to global.platformNode
|
||||
global.platformNode = new PlatformNode();
|
||||
|
||||
// Ensure global.util exists and has the necessary methods
|
||||
if (!global.util) {
|
||||
global.util = {};
|
||||
}
|
||||
|
||||
// Add isFloat32Array method if it doesn't exist
|
||||
if (!global.util.isFloat32Array) {
|
||||
global.util.isFloat32Array = (arr) => {
|
||||
return !!(
|
||||
arr instanceof Float32Array ||
|
||||
(arr && Object.prototype.toString.call(arr) === '[object Float32Array]')
|
||||
);
|
||||
};
|
||||
}
|
||||
|
||||
// Add isTypedArray method if it doesn't exist
|
||||
if (!global.util.isTypedArray) {
|
||||
global.util.isTypedArray = (arr) => {
|
||||
return !!(ArrayBuffer.isView(arr) && !(arr instanceof DataView));
|
||||
};
|
||||
}
|
||||
} catch (error) {
|
||||
console.warn('Failed to apply TensorFlow.js platform patch:', error);
|
||||
}
|
||||
}
|
||||
|
||||
const fn = new Function('return ' + workerData.fnString)();
|
||||
const result = fn(workerData.args);
|
||||
parentPort.postMessage({ result });
|
||||
`,
|
||||
{
|
||||
eval: true,
|
||||
workerData: { fnString, args }
|
||||
}
|
||||
)
|
||||
workerPool.set(randomKey, worker)
|
||||
}
|
||||
|
||||
worker._busy = true
|
||||
}
|
||||
|
||||
worker.on('message', (message: any) => {
|
||||
worker._busy = false
|
||||
resolve(message.result as T)
|
||||
})
|
||||
|
||||
worker.on('error', (err: any) => {
|
||||
worker._busy = false
|
||||
reject(err)
|
||||
})
|
||||
|
||||
worker.on('exit', (code: number) => {
|
||||
if (code !== 0) {
|
||||
worker._busy = false
|
||||
reject(new Error(`Worker stopped with exit code ${code}`))
|
||||
}
|
||||
})
|
||||
})
|
||||
.catch(reject)
|
||||
} catch (error) {
|
||||
reject(error)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
/**
|
||||
* Execute a function in a Web Worker (Browser environment)
|
||||
*/
|
||||
function executeInWebWorker<T>(fnString: string, args: any): Promise<T> {
|
||||
return new Promise<T>((resolve, reject) => {
|
||||
try {
|
||||
// Use the dedicated worker.js file instead of creating a blob
|
||||
// Try different approaches to locate the worker.js file
|
||||
let workerPath = './worker.js'
|
||||
|
||||
try {
|
||||
// First try to use the import.meta.url if available (modern browsers)
|
||||
if (typeof import.meta !== 'undefined' && import.meta.url) {
|
||||
const baseUrl = import.meta.url.substring(
|
||||
0,
|
||||
import.meta.url.lastIndexOf('/') + 1
|
||||
)
|
||||
workerPath = `${baseUrl}worker.js`
|
||||
}
|
||||
// Fallback to a relative path based on the unified.js location
|
||||
else if (typeof document !== 'undefined') {
|
||||
// Find the script tag that loaded unified.js
|
||||
const scripts = document.getElementsByTagName('script')
|
||||
for (let i = 0; i < scripts.length; i++) {
|
||||
const src = scripts[i].src
|
||||
if (src && src.includes('unified.js')) {
|
||||
// Get the directory path
|
||||
workerPath =
|
||||
src.substring(0, src.lastIndexOf('/') + 1) + 'worker.js'
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch (e) {
|
||||
console.warn(
|
||||
'Could not determine worker path from import.meta.url, using relative path',
|
||||
e
|
||||
)
|
||||
}
|
||||
|
||||
// If we couldn't determine the path, try some common locations
|
||||
if (workerPath === './worker.js' && typeof window !== 'undefined') {
|
||||
// Try to find the worker.js in the same directory as the current page
|
||||
const pageUrl = window.location.href
|
||||
const pageDir = pageUrl.substring(0, pageUrl.lastIndexOf('/') + 1)
|
||||
workerPath = `${pageDir}worker.js`
|
||||
|
||||
// Also check for dist/worker.js
|
||||
if (typeof document !== 'undefined') {
|
||||
const distWorkerPath = `${pageDir}dist/worker.js`
|
||||
// Create a test request to see if the file exists
|
||||
const xhr = new XMLHttpRequest()
|
||||
xhr.open('HEAD', distWorkerPath, false)
|
||||
try {
|
||||
xhr.send()
|
||||
if (xhr.status >= 200 && xhr.status < 300) {
|
||||
workerPath = distWorkerPath
|
||||
}
|
||||
} catch (e) {
|
||||
// Ignore errors, we'll use the default path
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
console.log('Using worker path:', workerPath)
|
||||
|
||||
// Try to create a worker, but fall back to inline worker or main thread execution if it fails
|
||||
let worker: Worker
|
||||
try {
|
||||
worker = new Worker(workerPath)
|
||||
} catch (error) {
|
||||
console.warn(
|
||||
'Failed to create Web Worker from file, trying inline worker:',
|
||||
error
|
||||
)
|
||||
|
||||
try {
|
||||
// Create an inline worker using a Blob
|
||||
const workerCode = `
|
||||
// Brainy Inline Worker Script
|
||||
console.log('Brainy Inline Worker: Started');
|
||||
|
||||
self.onmessage = function (e) {
|
||||
try {
|
||||
console.log('Brainy Inline Worker: Received message', e.data ? 'with data' : 'without data');
|
||||
|
||||
if (!e.data || !e.data.fnString) {
|
||||
throw new Error('Invalid message: missing function string');
|
||||
}
|
||||
|
||||
console.log('Brainy Inline Worker: Creating function from string');
|
||||
const fn = new Function('return ' + e.data.fnString)();
|
||||
|
||||
console.log('Brainy Inline Worker: Executing function with args');
|
||||
const result = fn(e.data.args);
|
||||
|
||||
console.log('Brainy Inline Worker: Function executed successfully, posting result');
|
||||
self.postMessage({ result: result });
|
||||
} catch (error) {
|
||||
console.error('Brainy Inline Worker: Error executing function', error);
|
||||
self.postMessage({
|
||||
error: error.message,
|
||||
stack: error.stack
|
||||
});
|
||||
}
|
||||
};
|
||||
`
|
||||
|
||||
const blob = new Blob([workerCode], {
|
||||
type: 'application/javascript'
|
||||
})
|
||||
const blobUrl = URL.createObjectURL(blob)
|
||||
worker = new Worker(blobUrl)
|
||||
|
||||
console.log('Created inline worker using Blob URL')
|
||||
} catch (inlineWorkerError) {
|
||||
console.warn(
|
||||
'Failed to create inline Web Worker, falling back to main thread execution:',
|
||||
inlineWorkerError
|
||||
)
|
||||
// Execute in main thread as fallback
|
||||
try {
|
||||
const fn = new Function('return ' + fnString)()
|
||||
resolve(fn(args) as T)
|
||||
return
|
||||
} catch (mainThreadError) {
|
||||
reject(mainThreadError)
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Set a timeout to prevent hanging
|
||||
const timeoutId = setTimeout(() => {
|
||||
console.warn(
|
||||
'Web Worker execution timed out, falling back to main thread'
|
||||
)
|
||||
worker.terminate()
|
||||
|
||||
// Execute in main thread as fallback
|
||||
try {
|
||||
const fn = new Function('return ' + fnString)()
|
||||
resolve(fn(args) as T)
|
||||
} catch (mainThreadError) {
|
||||
reject(mainThreadError)
|
||||
}
|
||||
}, 25000) // 25 second timeout (less than the 30 second test timeout)
|
||||
|
||||
worker.onmessage = function (e) {
|
||||
clearTimeout(timeoutId)
|
||||
if (e.data.error) {
|
||||
reject(new Error(e.data.error))
|
||||
} else {
|
||||
resolve(e.data.result as T)
|
||||
}
|
||||
worker.terminate()
|
||||
}
|
||||
|
||||
worker.onerror = function (e) {
|
||||
clearTimeout(timeoutId)
|
||||
console.warn(
|
||||
'Web Worker error, falling back to main thread execution:',
|
||||
e.message
|
||||
)
|
||||
worker.terminate()
|
||||
|
||||
// Execute in main thread as fallback
|
||||
try {
|
||||
const fn = new Function('return ' + fnString)()
|
||||
resolve(fn(args) as T)
|
||||
} catch (mainThreadError) {
|
||||
reject(mainThreadError)
|
||||
}
|
||||
}
|
||||
|
||||
worker.postMessage({ fnString, args })
|
||||
} catch (error) {
|
||||
reject(error)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
/**
|
||||
* Clean up all worker pools
|
||||
* This should be called when the application is shutting down
|
||||
*/
|
||||
export function cleanupWorkerPools(): void {
|
||||
if (isNode()) {
|
||||
import('node:worker_threads')
|
||||
.then(({ Worker }) => {
|
||||
for (const worker of workerPool.values()) {
|
||||
worker.terminate()
|
||||
}
|
||||
workerPool.clear()
|
||||
console.log('Worker pools cleaned up')
|
||||
})
|
||||
.catch(console.error)
|
||||
}
|
||||
}
|
||||
411
src/utils/writeBuffer.ts
Normal file
411
src/utils/writeBuffer.ts
Normal file
|
|
@ -0,0 +1,411 @@
|
|||
/**
|
||||
* Write Buffer
|
||||
* Accumulates writes and flushes them in bulk to reduce S3 operations
|
||||
* Implements intelligent deduplication and compression
|
||||
*/
|
||||
|
||||
import { HNSWNoun, HNSWVerb } from '../coreTypes.js'
|
||||
import { createModuleLogger } from './logger.js'
|
||||
import { getGlobalBackpressure } from './adaptiveBackpressure.js'
|
||||
|
||||
interface BufferedWrite<T> {
|
||||
id: string
|
||||
data: T
|
||||
timestamp: number
|
||||
type: 'noun' | 'verb' | 'metadata'
|
||||
retryCount: number
|
||||
}
|
||||
|
||||
interface FlushResult {
|
||||
successful: number
|
||||
failed: number
|
||||
duration: number
|
||||
}
|
||||
|
||||
/**
|
||||
* High-performance write buffer for bulk operations
|
||||
*/
|
||||
export class WriteBuffer<T> {
|
||||
private logger = createModuleLogger('WriteBuffer')
|
||||
|
||||
// Buffer storage
|
||||
private buffer = new Map<string, BufferedWrite<T>>()
|
||||
|
||||
// Configuration - More aggressive for high volume
|
||||
private maxBufferSize = 2000 // Allow larger buffers
|
||||
private flushInterval = 500 // Flush more frequently (0.5 seconds)
|
||||
private minFlushSize = 50 // Lower minimum to flush sooner
|
||||
private maxRetries = 3 // Maximum retry attempts
|
||||
|
||||
// State
|
||||
private flushTimer: NodeJS.Timeout | null = null
|
||||
private isFlushing = false
|
||||
private lastFlush = Date.now()
|
||||
private pendingFlush: Promise<FlushResult> | null = null
|
||||
|
||||
// Statistics
|
||||
private totalWrites = 0
|
||||
private totalFlushes = 0
|
||||
private failedWrites = 0
|
||||
private duplicatesRemoved = 0
|
||||
|
||||
// Write function
|
||||
private writeFunction: (items: Map<string, T>) => Promise<void>
|
||||
private type: 'noun' | 'verb' | 'metadata'
|
||||
|
||||
// Backpressure integration
|
||||
private backpressure = getGlobalBackpressure()
|
||||
|
||||
constructor(
|
||||
type: 'noun' | 'verb' | 'metadata',
|
||||
writeFunction: (items: Map<string, T>) => Promise<void>,
|
||||
options?: {
|
||||
maxBufferSize?: number
|
||||
flushInterval?: number
|
||||
minFlushSize?: number
|
||||
}
|
||||
) {
|
||||
this.type = type
|
||||
this.writeFunction = writeFunction
|
||||
|
||||
if (options) {
|
||||
this.maxBufferSize = options.maxBufferSize || this.maxBufferSize
|
||||
this.flushInterval = options.flushInterval || this.flushInterval
|
||||
this.minFlushSize = options.minFlushSize || this.minFlushSize
|
||||
}
|
||||
|
||||
// Start periodic flush
|
||||
this.startPeriodicFlush()
|
||||
}
|
||||
|
||||
/**
|
||||
* Add item to buffer
|
||||
*/
|
||||
public async add(id: string, data: T): Promise<void> {
|
||||
// Check if we're already at capacity
|
||||
if (this.buffer.size >= this.maxBufferSize) {
|
||||
// Wait for current flush to complete
|
||||
if (this.pendingFlush) {
|
||||
await this.pendingFlush
|
||||
}
|
||||
|
||||
// Force flush if still at capacity
|
||||
if (this.buffer.size >= this.maxBufferSize) {
|
||||
await this.flush('capacity')
|
||||
}
|
||||
}
|
||||
|
||||
// Check for duplicate and update if newer
|
||||
const existing = this.buffer.get(id)
|
||||
if (existing) {
|
||||
// Update with newer data
|
||||
existing.data = data
|
||||
existing.timestamp = Date.now()
|
||||
this.duplicatesRemoved++
|
||||
} else {
|
||||
// Add new item
|
||||
this.buffer.set(id, {
|
||||
id,
|
||||
data,
|
||||
timestamp: Date.now(),
|
||||
type: this.type,
|
||||
retryCount: 0
|
||||
})
|
||||
}
|
||||
|
||||
this.totalWrites++
|
||||
|
||||
// Log buffer growth periodically
|
||||
if (this.totalWrites % 100 === 0) {
|
||||
this.logger.info(`📈 BUFFER GROWTH: ${this.buffer.size} ${this.type} items buffered (${this.totalWrites} total writes, ${this.duplicatesRemoved} deduplicated)`)
|
||||
}
|
||||
|
||||
// Check if we should flush
|
||||
this.checkFlush()
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if we should flush
|
||||
*/
|
||||
private checkFlush(): void {
|
||||
const bufferSize = this.buffer.size
|
||||
const timeSinceFlush = Date.now() - this.lastFlush
|
||||
|
||||
// Immediate flush conditions
|
||||
if (bufferSize >= this.maxBufferSize) {
|
||||
this.flush('size')
|
||||
return
|
||||
}
|
||||
|
||||
// Time-based flush with minimum size
|
||||
if (timeSinceFlush >= this.flushInterval && bufferSize >= this.minFlushSize) {
|
||||
this.flush('time')
|
||||
return
|
||||
}
|
||||
|
||||
// Adaptive flush based on system load
|
||||
const backpressureStatus = this.backpressure.getStatus()
|
||||
if (backpressureStatus.queueLength > 1000 && bufferSize > 10) {
|
||||
// System under pressure - flush smaller batches more frequently
|
||||
this.flush('pressure')
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Flush buffer to storage
|
||||
*/
|
||||
public async flush(reason: string = 'manual'): Promise<FlushResult> {
|
||||
// Prevent concurrent flushes
|
||||
if (this.isFlushing) {
|
||||
if (this.pendingFlush) {
|
||||
return this.pendingFlush
|
||||
}
|
||||
return { successful: 0, failed: 0, duration: 0 }
|
||||
}
|
||||
|
||||
// Nothing to flush
|
||||
if (this.buffer.size === 0) {
|
||||
return { successful: 0, failed: 0, duration: 0 }
|
||||
}
|
||||
|
||||
this.isFlushing = true
|
||||
const startTime = Date.now()
|
||||
|
||||
// Create flush promise
|
||||
this.pendingFlush = this.doFlush(reason, startTime)
|
||||
|
||||
try {
|
||||
const result = await this.pendingFlush
|
||||
return result
|
||||
} finally {
|
||||
this.isFlushing = false
|
||||
this.pendingFlush = null
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Perform the actual flush
|
||||
*/
|
||||
private async doFlush(reason: string, startTime: number): Promise<FlushResult> {
|
||||
const itemsToFlush = new Map<string, T>()
|
||||
const flushingItems = new Map<string, BufferedWrite<T>>()
|
||||
|
||||
// Take items from buffer
|
||||
let count = 0
|
||||
for (const [id, item] of this.buffer.entries()) {
|
||||
itemsToFlush.set(id, item.data)
|
||||
flushingItems.set(id, item)
|
||||
count++
|
||||
|
||||
// Limit batch size for better performance
|
||||
if (count >= 500) {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
// Remove from buffer
|
||||
for (const id of itemsToFlush.keys()) {
|
||||
this.buffer.delete(id)
|
||||
}
|
||||
|
||||
this.logger.warn(`🔄 BUFFERING: Flushing ${itemsToFlush.size} ${this.type} items (buffer had ${this.buffer.size + itemsToFlush.size}) - reason: ${reason}`)
|
||||
|
||||
try {
|
||||
// Request permission from backpressure system
|
||||
const opId = `flush-${Date.now()}`
|
||||
await this.backpressure.requestPermission(opId, 2) // Higher priority
|
||||
|
||||
try {
|
||||
// Perform bulk write
|
||||
await this.writeFunction(itemsToFlush)
|
||||
|
||||
// Success
|
||||
this.backpressure.releasePermission(opId, true)
|
||||
this.totalFlushes++
|
||||
this.lastFlush = Date.now()
|
||||
|
||||
const duration = Date.now() - startTime
|
||||
this.logger.warn(`🚀 BATCH FLUSH: ${itemsToFlush.size} ${this.type} items → 1 bulk S3 operation (${duration}ms, reason: ${reason})`)
|
||||
|
||||
return {
|
||||
successful: itemsToFlush.size,
|
||||
failed: 0,
|
||||
duration
|
||||
}
|
||||
} catch (error) {
|
||||
// Release with error
|
||||
this.backpressure.releasePermission(opId, false)
|
||||
throw error
|
||||
}
|
||||
} catch (error) {
|
||||
this.logger.error(`Flush failed: ${error}`)
|
||||
|
||||
// Put items back with retry count
|
||||
for (const [id, item] of flushingItems.entries()) {
|
||||
item.retryCount++
|
||||
|
||||
if (item.retryCount < this.maxRetries) {
|
||||
// Put back for retry
|
||||
this.buffer.set(id, item)
|
||||
} else {
|
||||
// Max retries exceeded
|
||||
this.failedWrites++
|
||||
this.logger.error(`Max retries exceeded for ${this.type} ${id}`)
|
||||
}
|
||||
}
|
||||
|
||||
const duration = Date.now() - startTime
|
||||
|
||||
return {
|
||||
successful: 0,
|
||||
failed: itemsToFlush.size,
|
||||
duration
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Start periodic flush timer
|
||||
*/
|
||||
private startPeriodicFlush(): void {
|
||||
if (this.flushTimer) {
|
||||
return
|
||||
}
|
||||
|
||||
this.flushTimer = setInterval(() => {
|
||||
if (this.buffer.size > 0) {
|
||||
const timeSinceFlush = Date.now() - this.lastFlush
|
||||
|
||||
// Flush if we have items and enough time has passed
|
||||
if (timeSinceFlush >= this.flushInterval) {
|
||||
this.flush('periodic').catch(error => {
|
||||
this.logger.error('Periodic flush failed:', error)
|
||||
})
|
||||
}
|
||||
}
|
||||
}, Math.min(100, this.flushInterval / 2))
|
||||
}
|
||||
|
||||
/**
|
||||
* Stop periodic flush timer
|
||||
*/
|
||||
public stop(): void {
|
||||
if (this.flushTimer) {
|
||||
clearInterval(this.flushTimer)
|
||||
this.flushTimer = null
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Force flush all pending writes
|
||||
*/
|
||||
public async forceFlush(): Promise<FlushResult> {
|
||||
// Flush everything regardless of size
|
||||
const oldMinSize = this.minFlushSize
|
||||
this.minFlushSize = 0
|
||||
|
||||
try {
|
||||
const result = await this.flush('force')
|
||||
|
||||
// Flush any remaining items
|
||||
while (this.buffer.size > 0) {
|
||||
const additionalResult = await this.flush('force-remaining')
|
||||
result.successful += additionalResult.successful
|
||||
result.failed += additionalResult.failed
|
||||
result.duration += additionalResult.duration
|
||||
}
|
||||
|
||||
return result
|
||||
} finally {
|
||||
this.minFlushSize = oldMinSize
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get buffer statistics
|
||||
*/
|
||||
public getStats(): {
|
||||
bufferSize: number
|
||||
totalWrites: number
|
||||
totalFlushes: number
|
||||
failedWrites: number
|
||||
duplicatesRemoved: number
|
||||
avgFlushSize: number
|
||||
} {
|
||||
return {
|
||||
bufferSize: this.buffer.size,
|
||||
totalWrites: this.totalWrites,
|
||||
totalFlushes: this.totalFlushes,
|
||||
failedWrites: this.failedWrites,
|
||||
duplicatesRemoved: this.duplicatesRemoved,
|
||||
avgFlushSize: this.totalFlushes > 0 ? this.totalWrites / this.totalFlushes : 0
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Adjust parameters based on load
|
||||
*/
|
||||
public adjustForLoad(pendingRequests: number): void {
|
||||
if (pendingRequests > 10000) {
|
||||
// Extreme load - buffer more aggressively
|
||||
this.maxBufferSize = 5000
|
||||
this.flushInterval = 500
|
||||
this.minFlushSize = 500
|
||||
} else if (pendingRequests > 1000) {
|
||||
// High load
|
||||
this.maxBufferSize = 2000
|
||||
this.flushInterval = 1000
|
||||
this.minFlushSize = 200
|
||||
} else if (pendingRequests > 100) {
|
||||
// Moderate load
|
||||
this.maxBufferSize = 1000
|
||||
this.flushInterval = 2000
|
||||
this.minFlushSize = 100
|
||||
} else {
|
||||
// Low load - optimize for latency
|
||||
this.maxBufferSize = 500
|
||||
this.flushInterval = 5000
|
||||
this.minFlushSize = 50
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Global write buffers
|
||||
const writeBuffers = new Map<string, WriteBuffer<any>>()
|
||||
|
||||
/**
|
||||
* Get or create a write buffer
|
||||
*/
|
||||
export function getWriteBuffer<T>(
|
||||
id: string,
|
||||
type: 'noun' | 'verb' | 'metadata',
|
||||
writeFunction: (items: Map<string, T>) => Promise<void>
|
||||
): WriteBuffer<T> {
|
||||
if (!writeBuffers.has(id)) {
|
||||
writeBuffers.set(id, new WriteBuffer<T>(type, writeFunction))
|
||||
}
|
||||
return writeBuffers.get(id)!
|
||||
}
|
||||
|
||||
/**
|
||||
* Flush all write buffers
|
||||
*/
|
||||
export async function flushAllBuffers(): Promise<void> {
|
||||
const promises: Promise<FlushResult>[] = []
|
||||
|
||||
for (const buffer of writeBuffers.values()) {
|
||||
promises.push(buffer.forceFlush())
|
||||
}
|
||||
|
||||
await Promise.all(promises)
|
||||
}
|
||||
|
||||
/**
|
||||
* Clear all write buffers
|
||||
*/
|
||||
export function clearWriteBuffers(): void {
|
||||
for (const buffer of writeBuffers.values()) {
|
||||
buffer.stop()
|
||||
}
|
||||
writeBuffers.clear()
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue