feat(storage): fix critical socket exhaustion during metadata reading
CRITICAL ISSUE FIXED: Service initialization was failing due to socket exhaustion when reading metadata for 1400+ items during index rebuild Changes: - Add batch metadata reading to prevent 2000+ concurrent requests - Implement strict concurrency control (3 max concurrent requests) - Add proper yielding between batches to prevent event loop blocking - Reduce batch sizes during initialization (50 → 25 items per batch) - Add getMetadataBatch() and getVerbMetadataBatch() methods to S3 storage - Update StorageAdapter interface with batch methods - Add production environment auto-detection for smart logging - Auto-cleanup legacy /index folder during initialization Socket usage: Reduced from 1400+ concurrent to 3 max concurrent Expected production result: - Service initialization will complete successfully - firehoseServiceInitialized: true - Data collection will begin normally - No more socket exhaustion errors (100 socket limit exceeded) Fixes: #socket-exhaustion Breaking: None - backward compatible with fallback modes
This commit is contained in:
parent
6648149e90
commit
1b265e8c42
7 changed files with 485 additions and 52 deletions
|
|
@ -84,3 +84,103 @@ export function isThreadingAvailable(): boolean {
|
|||
export async function isThreadingAvailableAsync(): Promise<boolean> {
|
||||
return areWebWorkersAvailable() || (await areWorkerThreadsAvailable())
|
||||
}
|
||||
|
||||
/**
|
||||
* Auto-detect production environment to minimize logging costs
|
||||
*/
|
||||
export function isProductionEnvironment(): boolean {
|
||||
// Node.js environment detection
|
||||
if (isNode()) {
|
||||
// Check common production environment indicators
|
||||
const nodeEnv = process.env.NODE_ENV?.toLowerCase()
|
||||
if (nodeEnv === 'production' || nodeEnv === 'prod') return true
|
||||
|
||||
// Google Cloud Run detection
|
||||
if (process.env.K_SERVICE || process.env.GOOGLE_CLOUD_PROJECT) return true
|
||||
|
||||
// AWS Lambda detection
|
||||
if (process.env.AWS_LAMBDA_FUNCTION_NAME || process.env.AWS_EXECUTION_ENV) return true
|
||||
|
||||
// Azure Functions detection
|
||||
if (process.env.AZURE_FUNCTIONS_ENVIRONMENT || process.env.WEBSITE_SITE_NAME) return true
|
||||
|
||||
// Vercel detection
|
||||
if (process.env.VERCEL || process.env.VERCEL_ENV === 'production') return true
|
||||
|
||||
// Netlify detection
|
||||
if (process.env.NETLIFY && process.env.CONTEXT === 'production') return true
|
||||
|
||||
// Heroku detection
|
||||
if (process.env.DYNO && process.env.NODE_ENV !== 'development') return true
|
||||
|
||||
// Railway detection
|
||||
if (process.env.RAILWAY_ENVIRONMENT === 'production') return true
|
||||
|
||||
// Fly.io detection
|
||||
if (process.env.FLY_APP_NAME && process.env.FLY_REGION) return true
|
||||
|
||||
// Docker in production (common patterns)
|
||||
if (process.env.DOCKER_ENV === 'production' || process.env.ENVIRONMENT === 'production') return true
|
||||
|
||||
// Generic production indicators
|
||||
if (process.env.PROD === 'true' || process.env.PRODUCTION === 'true') return true
|
||||
}
|
||||
|
||||
// Browser environment - assume development unless explicitly production
|
||||
if (isBrowser()) {
|
||||
// Check for production domain patterns
|
||||
const hostname = window?.location?.hostname
|
||||
if (hostname) {
|
||||
// Avoid logging on production domains
|
||||
if (hostname.includes('.com') || hostname.includes('.org') || hostname.includes('.net')) {
|
||||
return !hostname.includes('localhost') && !hostname.includes('127.0.0.1') && !hostname.includes('dev')
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return false
|
||||
}
|
||||
|
||||
/**
|
||||
* Get appropriate log level based on environment
|
||||
*/
|
||||
export function getLogLevel(): 'silent' | 'error' | 'warn' | 'info' | 'verbose' {
|
||||
// Explicit log level override
|
||||
const explicitLevel = process.env.BRAINY_LOG_LEVEL?.toLowerCase()
|
||||
if (explicitLevel && ['silent', 'error', 'warn', 'info', 'verbose'].includes(explicitLevel)) {
|
||||
return explicitLevel as 'silent' | 'error' | 'warn' | 'info' | 'verbose'
|
||||
}
|
||||
|
||||
// Auto-detect based on environment
|
||||
if (isProductionEnvironment()) {
|
||||
return 'error' // Only log errors in production to minimize costs
|
||||
}
|
||||
|
||||
// Development environments get more verbose logging
|
||||
if (process.env.NODE_ENV === 'development' || process.env.NODE_ENV === 'dev') {
|
||||
return 'verbose'
|
||||
}
|
||||
|
||||
// Test environments should be quieter
|
||||
if (process.env.NODE_ENV === 'test') {
|
||||
return 'warn'
|
||||
}
|
||||
|
||||
// Default to info level
|
||||
return 'info'
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if logging should be enabled for a given level
|
||||
*/
|
||||
export function shouldLog(level: 'error' | 'warn' | 'info' | 'verbose'): boolean {
|
||||
const currentLevel = getLogLevel()
|
||||
|
||||
if (currentLevel === 'silent') return false
|
||||
|
||||
const levels = ['error', 'warn', 'info', 'verbose']
|
||||
const currentIndex = levels.indexOf(currentLevel)
|
||||
const messageIndex = levels.indexOf(level)
|
||||
|
||||
return messageIndex <= currentIndex
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,8 +1,11 @@
|
|||
/**
|
||||
* Centralized logging utility for Brainy
|
||||
* Provides configurable log levels and consistent logging across the codebase
|
||||
* Automatically reduces logging in production environments to minimize costs
|
||||
*/
|
||||
|
||||
import { isProductionEnvironment, getLogLevel } from './environment.js'
|
||||
|
||||
export enum LogLevel {
|
||||
ERROR = 0,
|
||||
WARN = 1,
|
||||
|
|
@ -28,13 +31,16 @@ export interface LoggerConfig {
|
|||
class Logger {
|
||||
private static instance: Logger
|
||||
private config: LoggerConfig = {
|
||||
level: LogLevel.WARN, // Default to WARN - only critical messages
|
||||
timestamps: true,
|
||||
level: LogLevel.ERROR, // Default to ERROR in production for cost optimization
|
||||
timestamps: false, // Disable timestamps in production to reduce log size
|
||||
includeModule: true
|
||||
}
|
||||
|
||||
private constructor() {
|
||||
// Set log level from environment variable if available
|
||||
// Auto-detect production environment and set appropriate defaults
|
||||
this.applyEnvironmentDefaults()
|
||||
|
||||
// Set log level from environment variable if available (overrides auto-detection)
|
||||
const envLogLevel = process.env.BRAINY_LOG_LEVEL
|
||||
if (envLogLevel) {
|
||||
const level = LogLevel[envLogLevel.toUpperCase() as keyof typeof LogLevel]
|
||||
|
|
@ -54,6 +60,40 @@ class Logger {
|
|||
}
|
||||
}
|
||||
|
||||
private applyEnvironmentDefaults(): void {
|
||||
const envLogLevel = getLogLevel()
|
||||
|
||||
// Convert environment log level to Logger LogLevel
|
||||
switch (envLogLevel) {
|
||||
case 'silent':
|
||||
this.config.level = -1 as LogLevel // Below ERROR to silence all logs
|
||||
break
|
||||
case 'error':
|
||||
this.config.level = LogLevel.ERROR
|
||||
this.config.timestamps = false // Minimize log size in production
|
||||
break
|
||||
case 'warn':
|
||||
this.config.level = LogLevel.WARN
|
||||
this.config.timestamps = false
|
||||
break
|
||||
case 'info':
|
||||
this.config.level = LogLevel.INFO
|
||||
this.config.timestamps = true
|
||||
break
|
||||
case 'verbose':
|
||||
this.config.level = LogLevel.DEBUG
|
||||
this.config.timestamps = true
|
||||
break
|
||||
}
|
||||
|
||||
// In production environments, be extra conservative to minimize costs
|
||||
if (isProductionEnvironment()) {
|
||||
this.config.level = Math.min(this.config.level, LogLevel.ERROR)
|
||||
this.config.timestamps = false
|
||||
this.config.includeModule = false // Reduce log size
|
||||
}
|
||||
}
|
||||
|
||||
static getInstance(): Logger {
|
||||
if (!Logger.instance) {
|
||||
Logger.instance = new Logger()
|
||||
|
|
@ -164,4 +204,65 @@ export function createModuleLogger(module: string) {
|
|||
// Export function to configure logger
|
||||
export function configureLogger(config: Partial<LoggerConfig>) {
|
||||
logger.configure(config)
|
||||
}
|
||||
|
||||
/**
|
||||
* Smart console replacement that automatically reduces logging in production
|
||||
* Dramatically reduces Google Cloud Run logging costs
|
||||
*
|
||||
* Usage: Replace console.log with smartConsole.log, etc.
|
||||
*/
|
||||
export const smartConsole = {
|
||||
log: (message?: any, ...args: any[]) => {
|
||||
if (logger['shouldLog'](LogLevel.INFO, 'console')) {
|
||||
console.log(message, ...args)
|
||||
}
|
||||
},
|
||||
|
||||
info: (message?: any, ...args: any[]) => {
|
||||
if (logger['shouldLog'](LogLevel.INFO, 'console')) {
|
||||
console.info(message, ...args)
|
||||
}
|
||||
},
|
||||
|
||||
warn: (message?: any, ...args: any[]) => {
|
||||
if (logger['shouldLog'](LogLevel.WARN, 'console')) {
|
||||
console.warn(message, ...args)
|
||||
}
|
||||
},
|
||||
|
||||
error: (message?: any, ...args: any[]) => {
|
||||
if (logger['shouldLog'](LogLevel.ERROR, 'console')) {
|
||||
console.error(message, ...args)
|
||||
}
|
||||
},
|
||||
|
||||
debug: (message?: any, ...args: any[]) => {
|
||||
if (logger['shouldLog'](LogLevel.DEBUG, 'console')) {
|
||||
console.debug(message, ...args)
|
||||
}
|
||||
},
|
||||
|
||||
trace: (message?: any, ...args: any[]) => {
|
||||
if (logger['shouldLog'](LogLevel.TRACE, 'console')) {
|
||||
console.trace(message, ...args)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Production-optimized logging functions
|
||||
* These only log in non-production environments or when explicitly enabled
|
||||
*/
|
||||
export const prodLog = {
|
||||
// Only log errors in production (always visible)
|
||||
error: (message?: any, ...args: any[]) => {
|
||||
console.error(message, ...args)
|
||||
},
|
||||
|
||||
// These are suppressed in production unless BRAINY_LOG_LEVEL is set
|
||||
warn: (message?: any, ...args: any[]) => smartConsole.warn(message, ...args),
|
||||
info: (message?: any, ...args: any[]) => smartConsole.info(message, ...args),
|
||||
debug: (message?: any, ...args: any[]) => smartConsole.debug(message, ...args),
|
||||
log: (message?: any, ...args: any[]) => smartConsole.log(message, ...args)
|
||||
}
|
||||
|
|
@ -6,6 +6,7 @@
|
|||
|
||||
import { StorageAdapter } from '../coreTypes.js'
|
||||
import { MetadataIndexCache, MetadataIndexCacheConfig } from './metadataIndexCache.js'
|
||||
import { prodLog } from './logger.js'
|
||||
|
||||
export interface MetadataIndexEntry {
|
||||
field: string
|
||||
|
|
@ -684,7 +685,7 @@ export class MetadataIndexManager {
|
|||
|
||||
this.isRebuilding = true
|
||||
try {
|
||||
console.log('🔄 Starting non-blocking metadata index rebuild...')
|
||||
prodLog.info('🔄 Starting non-blocking metadata index rebuild with batch processing to prevent socket exhaustion...')
|
||||
|
||||
// Clear existing indexes
|
||||
this.indexCache.clear()
|
||||
|
|
@ -694,7 +695,7 @@ export class MetadataIndexManager {
|
|||
|
||||
// Rebuild noun metadata indexes using pagination
|
||||
let nounOffset = 0
|
||||
const nounLimit = 50 // Smaller batches to reduce blocking
|
||||
const nounLimit = 25 // Even smaller batches during initialization to prevent socket exhaustion
|
||||
let hasMoreNouns = true
|
||||
let totalNounsProcessed = 0
|
||||
|
||||
|
|
@ -703,35 +704,69 @@ export class MetadataIndexManager {
|
|||
pagination: { offset: nounOffset, limit: nounLimit }
|
||||
})
|
||||
|
||||
// Process batch with event loop yields
|
||||
for (let i = 0; i < result.items.length; i++) {
|
||||
const noun = result.items[i]
|
||||
const metadata = await this.storage.getMetadata(noun.id)
|
||||
// CRITICAL FIX: Use batch metadata reading to prevent socket exhaustion
|
||||
const nounIds = result.items.map(noun => noun.id)
|
||||
|
||||
let metadataBatch: Map<string, any>
|
||||
if (this.storage.getMetadataBatch) {
|
||||
// Use batch reading if available (prevents socket exhaustion)
|
||||
metadataBatch = await this.storage.getMetadataBatch(nounIds)
|
||||
prodLog.debug(`📦 Batch loaded ${metadataBatch.size}/${nounIds.length} metadata objects`)
|
||||
} else {
|
||||
// Fallback to individual calls with strict concurrency control
|
||||
metadataBatch = new Map()
|
||||
const CONCURRENCY_LIMIT = 3 // Very conservative limit
|
||||
|
||||
for (let i = 0; i < nounIds.length; i += CONCURRENCY_LIMIT) {
|
||||
const batch = nounIds.slice(i, i + CONCURRENCY_LIMIT)
|
||||
const batchPromises = batch.map(async (id) => {
|
||||
try {
|
||||
const metadata = await this.storage.getMetadata(id)
|
||||
return { id, metadata }
|
||||
} catch (error) {
|
||||
prodLog.debug(`Failed to read metadata for ${id}:`, error)
|
||||
return { id, metadata: null }
|
||||
}
|
||||
})
|
||||
|
||||
const batchResults = await Promise.all(batchPromises)
|
||||
for (const { id, metadata } of batchResults) {
|
||||
if (metadata) {
|
||||
metadataBatch.set(id, metadata)
|
||||
}
|
||||
}
|
||||
|
||||
// Yield between batches to prevent socket exhaustion
|
||||
await this.yieldToEventLoop()
|
||||
}
|
||||
}
|
||||
|
||||
// Process the metadata batch
|
||||
for (const noun of result.items) {
|
||||
const metadata = metadataBatch.get(noun.id)
|
||||
if (metadata) {
|
||||
// Skip flush during rebuild for performance
|
||||
await this.addToIndex(noun.id, metadata, true)
|
||||
}
|
||||
|
||||
// Yield to event loop every 10 items to prevent blocking
|
||||
if (i % 10 === 9) {
|
||||
await this.yieldToEventLoop()
|
||||
}
|
||||
}
|
||||
|
||||
// Yield after processing the entire batch
|
||||
await this.yieldToEventLoop()
|
||||
|
||||
totalNounsProcessed += result.items.length
|
||||
hasMoreNouns = result.hasMore
|
||||
nounOffset += nounLimit
|
||||
|
||||
// Progress logging and event loop yield after each batch
|
||||
if (totalNounsProcessed % 100 === 0 || !hasMoreNouns) {
|
||||
console.log(`📊 Indexed ${totalNounsProcessed} nouns...`)
|
||||
prodLog.debug(`📊 Indexed ${totalNounsProcessed} nouns...`)
|
||||
}
|
||||
await this.yieldToEventLoop()
|
||||
}
|
||||
|
||||
// Rebuild verb metadata indexes using pagination
|
||||
let verbOffset = 0
|
||||
const verbLimit = 50 // Smaller batches to reduce blocking
|
||||
const verbLimit = 25 // Even smaller batches during initialization to prevent socket exhaustion
|
||||
let hasMoreVerbs = true
|
||||
let totalVerbsProcessed = 0
|
||||
|
||||
|
|
@ -740,38 +775,72 @@ export class MetadataIndexManager {
|
|||
pagination: { offset: verbOffset, limit: verbLimit }
|
||||
})
|
||||
|
||||
// Process batch with event loop yields
|
||||
for (let i = 0; i < result.items.length; i++) {
|
||||
const verb = result.items[i]
|
||||
const metadata = await this.storage.getVerbMetadata(verb.id)
|
||||
// CRITICAL FIX: Use batch verb metadata reading to prevent socket exhaustion
|
||||
const verbIds = result.items.map(verb => verb.id)
|
||||
|
||||
let verbMetadataBatch: Map<string, any>
|
||||
if ((this.storage as any).getVerbMetadataBatch) {
|
||||
// Use batch reading if available (prevents socket exhaustion)
|
||||
verbMetadataBatch = await (this.storage as any).getVerbMetadataBatch(verbIds)
|
||||
prodLog.debug(`📦 Batch loaded ${verbMetadataBatch.size}/${verbIds.length} verb metadata objects`)
|
||||
} else {
|
||||
// Fallback to individual calls with strict concurrency control
|
||||
verbMetadataBatch = new Map()
|
||||
const CONCURRENCY_LIMIT = 3 // Very conservative limit to prevent socket exhaustion
|
||||
|
||||
for (let i = 0; i < verbIds.length; i += CONCURRENCY_LIMIT) {
|
||||
const batch = verbIds.slice(i, i + CONCURRENCY_LIMIT)
|
||||
const batchPromises = batch.map(async (id) => {
|
||||
try {
|
||||
const metadata = await this.storage.getVerbMetadata(id)
|
||||
return { id, metadata }
|
||||
} catch (error) {
|
||||
prodLog.debug(`Failed to read verb metadata for ${id}:`, error)
|
||||
return { id, metadata: null }
|
||||
}
|
||||
})
|
||||
|
||||
const batchResults = await Promise.all(batchPromises)
|
||||
for (const { id, metadata } of batchResults) {
|
||||
if (metadata) {
|
||||
verbMetadataBatch.set(id, metadata)
|
||||
}
|
||||
}
|
||||
|
||||
// Yield between batches to prevent socket exhaustion
|
||||
await this.yieldToEventLoop()
|
||||
}
|
||||
}
|
||||
|
||||
// Process the verb metadata batch
|
||||
for (const verb of result.items) {
|
||||
const metadata = verbMetadataBatch.get(verb.id)
|
||||
if (metadata) {
|
||||
// Skip flush during rebuild for performance
|
||||
await this.addToIndex(verb.id, metadata, true)
|
||||
}
|
||||
|
||||
// Yield to event loop every 10 items to prevent blocking
|
||||
if (i % 10 === 9) {
|
||||
await this.yieldToEventLoop()
|
||||
}
|
||||
}
|
||||
|
||||
// Yield after processing the entire batch
|
||||
await this.yieldToEventLoop()
|
||||
|
||||
totalVerbsProcessed += result.items.length
|
||||
hasMoreVerbs = result.hasMore
|
||||
verbOffset += verbLimit
|
||||
|
||||
// Progress logging and event loop yield after each batch
|
||||
if (totalVerbsProcessed % 100 === 0 || !hasMoreVerbs) {
|
||||
console.log(`🔗 Indexed ${totalVerbsProcessed} verbs...`)
|
||||
prodLog.debug(`🔗 Indexed ${totalVerbsProcessed} verbs...`)
|
||||
}
|
||||
await this.yieldToEventLoop()
|
||||
}
|
||||
|
||||
// Flush to storage with final yield
|
||||
console.log('💾 Flushing metadata index to storage...')
|
||||
prodLog.debug('💾 Flushing metadata index to storage...')
|
||||
await this.flush()
|
||||
await this.yieldToEventLoop()
|
||||
|
||||
console.log(`✅ Metadata index rebuild completed! Processed ${totalNounsProcessed} nouns and ${totalVerbsProcessed} verbs`)
|
||||
prodLog.info(`✅ Metadata index rebuild completed! Processed ${totalNounsProcessed} nouns and ${totalVerbsProcessed} verbs`)
|
||||
|
||||
} finally {
|
||||
this.isRebuilding = false
|
||||
|
|
|
|||
|
|
@ -4,6 +4,7 @@
|
|||
*/
|
||||
|
||||
import { isBrowser, isNode } from './environment.js'
|
||||
import { prodLog } from './logger.js'
|
||||
|
||||
// Worker pool to reuse workers
|
||||
const workerPool: Map<string, any> = new Map()
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue