feat(storage): fix critical socket exhaustion during metadata reading
CRITICAL ISSUE FIXED: Service initialization was failing due to socket exhaustion when reading metadata for 1400+ items during index rebuild Changes: - Add batch metadata reading to prevent 2000+ concurrent requests - Implement strict concurrency control (3 max concurrent requests) - Add proper yielding between batches to prevent event loop blocking - Reduce batch sizes during initialization (50 → 25 items per batch) - Add getMetadataBatch() and getVerbMetadataBatch() methods to S3 storage - Update StorageAdapter interface with batch methods - Add production environment auto-detection for smart logging - Auto-cleanup legacy /index folder during initialization Socket usage: Reduced from 1400+ concurrent to 3 max concurrent Expected production result: - Service initialization will complete successfully - firehoseServiceInitialized: true - Data collection will begin normally - No more socket exhaustion errors (100 socket limit exceeded) Fixes: #socket-exhaustion Breaking: None - backward compatible with fallback modes
This commit is contained in:
parent
6648149e90
commit
1b265e8c42
7 changed files with 485 additions and 52 deletions
|
|
@ -47,6 +47,7 @@ import {
|
|||
import { IntelligentVerbScoring } from './augmentations/intelligentVerbScoring.js'
|
||||
import { BrainyDataInterface } from './types/brainyDataInterface.js'
|
||||
import { augmentationPipeline } from './augmentationPipeline.js'
|
||||
import { prodLog } from './utils/logger.js'
|
||||
import {
|
||||
prepareJsonForVectorization,
|
||||
extractFieldFromJson
|
||||
|
|
@ -683,7 +684,7 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
|
|||
}
|
||||
|
||||
if (this.loggingConfig?.verbose) {
|
||||
console.log(this.cacheAutoConfigurator.getConfigExplanation(autoConfig))
|
||||
prodLog.info(this.cacheAutoConfigurator.getConfigExplanation(autoConfig))
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -751,7 +752,7 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
|
|||
// If the database is frozen, do not start real-time updates
|
||||
if (this.frozen) {
|
||||
if (this.loggingConfig?.verbose) {
|
||||
console.log('Real-time updates disabled: database is frozen')
|
||||
prodLog.info('Real-time updates disabled: database is frozen')
|
||||
}
|
||||
return
|
||||
}
|
||||
|
|
@ -767,7 +768,7 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
|
|||
this.lastKnownNounCount = count
|
||||
})
|
||||
.catch((error) => {
|
||||
console.warn(
|
||||
prodLog.warn(
|
||||
'Failed to get initial noun count for real-time updates:',
|
||||
error
|
||||
)
|
||||
|
|
@ -776,12 +777,12 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
|
|||
// Start the update timer
|
||||
this.updateTimerId = setInterval(() => {
|
||||
this.checkForUpdates().catch((error) => {
|
||||
console.warn('Error during real-time update check:', error)
|
||||
prodLog.warn('Error during real-time update check:', error)
|
||||
})
|
||||
}, this.realtimeUpdateConfig.interval)
|
||||
|
||||
if (this.loggingConfig?.verbose) {
|
||||
console.log(
|
||||
prodLog.info(
|
||||
`Real-time updates started with interval: ${this.realtimeUpdateConfig.interval}ms`
|
||||
)
|
||||
}
|
||||
|
|
@ -801,7 +802,7 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
|
|||
this.updateTimerId = null
|
||||
|
||||
if (this.loggingConfig?.verbose) {
|
||||
console.log('Real-time updates stopped')
|
||||
prodLog.info('Real-time updates stopped')
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -849,7 +850,7 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
|
|||
try {
|
||||
await this.metadataIndex!.flush()
|
||||
} catch (error) {
|
||||
console.warn('Error flushing metadata index:', error)
|
||||
prodLog.warn('Error flushing metadata index:', error)
|
||||
}
|
||||
}, 30000) // Flush every 30 seconds
|
||||
|
||||
|
|
@ -922,7 +923,7 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
|
|||
// Cleanup expired cache entries (defensive mechanism for distributed scenarios)
|
||||
const expiredCount = this.searchCache.cleanupExpiredEntries()
|
||||
if (expiredCount > 0 && this.loggingConfig?.verbose) {
|
||||
console.log(`Cleaned up ${expiredCount} expired cache entries`)
|
||||
prodLog.debug(`Cleaned up ${expiredCount} expired cache entries`)
|
||||
}
|
||||
|
||||
// Adapt cache configuration based on performance (every few updates)
|
||||
|
|
@ -940,10 +941,10 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
|
|||
|
||||
if (this.loggingConfig?.verbose) {
|
||||
const duration = this.lastUpdateTime - startTime
|
||||
console.log(`Real-time update completed in ${duration}ms`)
|
||||
prodLog.debug(`Real-time update completed in ${duration}ms`)
|
||||
}
|
||||
} catch (error) {
|
||||
console.error('Failed to check for updates:', error)
|
||||
prodLog.error('Failed to check for updates:', error)
|
||||
// Don't rethrow the error to avoid disrupting the update timer
|
||||
}
|
||||
}
|
||||
|
|
@ -977,7 +978,7 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
|
|||
|
||||
// Check if the vector dimensions match the expected dimensions
|
||||
if (noun.vector.length !== this._dimensions) {
|
||||
console.warn(
|
||||
prodLog.warn(
|
||||
`Skipping noun ${noun.id} due to dimension mismatch: expected ${this._dimensions}, got ${noun.vector.length}`
|
||||
)
|
||||
continue
|
||||
|
|
@ -996,7 +997,7 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
|
|||
}
|
||||
|
||||
if (this.loggingConfig?.verbose) {
|
||||
console.log(
|
||||
prodLog.debug(
|
||||
`${change.operation === 'add' ? 'Added' : 'Updated'} noun ${noun.id} in index during real-time update`
|
||||
)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -431,6 +431,13 @@ export interface StorageAdapter {
|
|||
|
||||
getMetadata(id: string): Promise<any | null>
|
||||
|
||||
/**
|
||||
* Get multiple metadata objects in batches (prevents socket exhaustion)
|
||||
* @param ids Array of IDs to get metadata for
|
||||
* @returns Promise that resolves to a Map of id -> metadata
|
||||
*/
|
||||
getMetadataBatch?(ids: string[]): Promise<Map<string, any>>
|
||||
|
||||
/**
|
||||
* Save verb metadata to storage
|
||||
* @param id The ID of the verb
|
||||
|
|
|
|||
|
|
@ -22,7 +22,7 @@ import {
|
|||
} from '../../utils/operationUtils.js'
|
||||
import { BrainyError } from '../../errors/brainyError.js'
|
||||
import { CacheManager } from '../cacheManager.js'
|
||||
import { createModuleLogger } from '../../utils/logger.js'
|
||||
import { createModuleLogger, prodLog } from '../../utils/logger.js'
|
||||
import { getGlobalSocketManager } from '../../utils/adaptiveSocketManager.js'
|
||||
import { getGlobalBackpressure } from '../../utils/adaptiveBackpressure.js'
|
||||
import { getWriteBuffer, WriteBuffer } from '../../utils/writeBuffer.js'
|
||||
|
|
@ -335,6 +335,9 @@ export class S3CompatibleStorage extends BaseStorage {
|
|||
// Initialize request coalescer
|
||||
this.initializeCoalescer()
|
||||
|
||||
// Auto-cleanup legacy /index folder on initialization
|
||||
await this.cleanupLegacyIndexFolder()
|
||||
|
||||
this.isInitialized = true
|
||||
this.logger.info(`Initialized ${this.serviceType} storage with bucket ${this.bucketName}`)
|
||||
} catch (error) {
|
||||
|
|
@ -345,6 +348,72 @@ export class S3CompatibleStorage extends BaseStorage {
|
|||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Auto-cleanup legacy /index folder during initialization
|
||||
* This removes old index data that has been migrated to _system
|
||||
*/
|
||||
private async cleanupLegacyIndexFolder(): Promise<void> {
|
||||
try {
|
||||
// Check if there are any objects in the legacy index folder
|
||||
const { ListObjectsV2Command } = await import('@aws-sdk/client-s3')
|
||||
|
||||
const listResponse = await this.s3Client!.send(
|
||||
new ListObjectsV2Command({
|
||||
Bucket: this.bucketName,
|
||||
Prefix: this.indexPrefix,
|
||||
MaxKeys: 1 // Just check if anything exists
|
||||
})
|
||||
)
|
||||
|
||||
// If there are objects in the legacy index folder, clean them up
|
||||
if (listResponse.Contents && listResponse.Contents.length > 0) {
|
||||
prodLog.info(`🧹 Cleaning up legacy /index folder during initialization...`)
|
||||
|
||||
// Use the existing deleteObjectsWithPrefix function logic
|
||||
const { ListObjectsV2Command, DeleteObjectsCommand } = await import('@aws-sdk/client-s3')
|
||||
|
||||
let continuationToken: string | undefined = undefined
|
||||
let totalDeleted = 0
|
||||
|
||||
do {
|
||||
const listResponseBatch: any = await this.s3Client!.send(
|
||||
new ListObjectsV2Command({
|
||||
Bucket: this.bucketName,
|
||||
Prefix: this.indexPrefix,
|
||||
ContinuationToken: continuationToken
|
||||
})
|
||||
)
|
||||
|
||||
if (listResponseBatch.Contents && listResponseBatch.Contents.length > 0) {
|
||||
const objectsToDelete = listResponseBatch.Contents.map((obj: any) => ({
|
||||
Key: obj.Key!
|
||||
}))
|
||||
|
||||
await this.s3Client!.send(
|
||||
new DeleteObjectsCommand({
|
||||
Bucket: this.bucketName,
|
||||
Delete: {
|
||||
Objects: objectsToDelete
|
||||
}
|
||||
})
|
||||
)
|
||||
|
||||
totalDeleted += objectsToDelete.length
|
||||
}
|
||||
|
||||
continuationToken = listResponseBatch.NextContinuationToken
|
||||
} while (continuationToken)
|
||||
|
||||
prodLog.info(`✅ Cleaned up ${totalDeleted} legacy index objects`)
|
||||
} else {
|
||||
prodLog.debug('No legacy /index folder found - already clean')
|
||||
}
|
||||
} catch (error) {
|
||||
// Don't fail initialization if cleanup fails
|
||||
prodLog.warn('Failed to cleanup legacy /index folder:', error)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Initialize write buffers for high-volume scenarios
|
||||
*/
|
||||
|
|
@ -1846,6 +1915,89 @@ export class S3CompatibleStorage extends BaseStorage {
|
|||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get multiple metadata objects in batches (CRITICAL: Prevents socket exhaustion)
|
||||
* This is the solution to the metadata reading socket exhaustion during initialization
|
||||
*/
|
||||
public async getMetadataBatch(ids: string[]): Promise<Map<string, any>> {
|
||||
await this.ensureInitialized()
|
||||
|
||||
const results = new Map<string, any>()
|
||||
const batchSize = Math.min(this.getBatchSize(), 10) // Smaller batches for metadata to prevent socket exhaustion
|
||||
|
||||
// Process in smaller batches to avoid socket exhaustion
|
||||
for (let i = 0; i < ids.length; i += batchSize) {
|
||||
const batch = ids.slice(i, i + batchSize)
|
||||
|
||||
// Process batch with concurrency control
|
||||
const batchPromises = batch.map(async (id) => {
|
||||
try {
|
||||
const metadata = await this.getMetadata(id)
|
||||
return { id, metadata }
|
||||
} catch (error) {
|
||||
// Don't fail entire batch if one metadata read fails
|
||||
this.logger.debug(`Failed to read metadata for ${id}:`, error)
|
||||
return { id, metadata: null }
|
||||
}
|
||||
})
|
||||
|
||||
const batchResults = await Promise.all(batchPromises)
|
||||
|
||||
// Add results to map
|
||||
for (const { id, metadata } of batchResults) {
|
||||
if (metadata !== null) {
|
||||
results.set(id, metadata)
|
||||
}
|
||||
}
|
||||
|
||||
// Yield to prevent socket exhaustion between batches
|
||||
await new Promise(resolve => setImmediate(resolve))
|
||||
}
|
||||
|
||||
return results
|
||||
}
|
||||
|
||||
/**
|
||||
* Get multiple verb metadata objects in batches (prevents socket exhaustion)
|
||||
*/
|
||||
public async getVerbMetadataBatch(ids: string[]): Promise<Map<string, any>> {
|
||||
await this.ensureInitialized()
|
||||
|
||||
const results = new Map<string, any>()
|
||||
const batchSize = Math.min(this.getBatchSize(), 10) // Smaller batches for metadata to prevent socket exhaustion
|
||||
|
||||
// Process in smaller batches to avoid socket exhaustion
|
||||
for (let i = 0; i < ids.length; i += batchSize) {
|
||||
const batch = ids.slice(i, i + batchSize)
|
||||
|
||||
// Process batch with concurrency control
|
||||
const batchPromises = batch.map(async (id) => {
|
||||
try {
|
||||
const metadata = await this.getVerbMetadata(id)
|
||||
return { id, metadata }
|
||||
} catch (error) {
|
||||
// Don't fail entire batch if one metadata read fails
|
||||
this.logger.debug(`Failed to read verb metadata for ${id}:`, error)
|
||||
return { id, metadata: null }
|
||||
}
|
||||
})
|
||||
|
||||
const batchResults = await Promise.all(batchPromises)
|
||||
|
||||
// Add results to map
|
||||
for (const { id, metadata } of batchResults) {
|
||||
if (metadata !== null) {
|
||||
results.set(id, metadata)
|
||||
}
|
||||
}
|
||||
|
||||
// Yield to prevent socket exhaustion between batches
|
||||
await new Promise(resolve => setImmediate(resolve))
|
||||
}
|
||||
|
||||
return results
|
||||
}
|
||||
|
||||
/**
|
||||
* Get noun metadata from storage
|
||||
*/
|
||||
|
|
@ -1915,9 +2067,9 @@ export class S3CompatibleStorage extends BaseStorage {
|
|||
// Import the GetObjectCommand only when needed
|
||||
const { GetObjectCommand } = await import('@aws-sdk/client-s3')
|
||||
|
||||
console.log(`Getting metadata for ${id} from bucket ${this.bucketName}`)
|
||||
prodLog.debug(`Getting metadata for ${id} from bucket ${this.bucketName}`)
|
||||
const key = `${this.metadataPrefix}${id}.json`
|
||||
console.log(`Looking for metadata at key: ${key}`)
|
||||
prodLog.debug(`Looking for metadata at key: ${key}`)
|
||||
|
||||
// Try to get the metadata from the metadata directory
|
||||
const response = await this.s3Client!.send(
|
||||
|
|
@ -1929,24 +2081,24 @@ export class S3CompatibleStorage extends BaseStorage {
|
|||
|
||||
// Check if response is null or undefined (can happen in mock implementations)
|
||||
if (!response || !response.Body) {
|
||||
console.log(`No metadata found for ${id}`)
|
||||
prodLog.debug(`No metadata found for ${id}`)
|
||||
return null
|
||||
}
|
||||
|
||||
// Convert the response body to a string
|
||||
const bodyContents = await response.Body.transformToString()
|
||||
console.log(`Retrieved metadata body: ${bodyContents}`)
|
||||
prodLog.debug(`Retrieved metadata body: ${bodyContents}`)
|
||||
|
||||
// Parse the JSON string
|
||||
try {
|
||||
const parsedMetadata = JSON.parse(bodyContents)
|
||||
console.log(
|
||||
prodLog.debug(
|
||||
`Successfully retrieved metadata for ${id}:`,
|
||||
parsedMetadata
|
||||
)
|
||||
return parsedMetadata
|
||||
} catch (parseError) {
|
||||
console.error(`Failed to parse metadata for ${id}:`, parseError)
|
||||
prodLog.error(`Failed to parse metadata for ${id}:`, parseError)
|
||||
return null
|
||||
}
|
||||
} catch (error: any) {
|
||||
|
|
@ -1960,7 +2112,7 @@ export class S3CompatibleStorage extends BaseStorage {
|
|||
error.message.includes('not found') ||
|
||||
error.message.includes('does not exist')))
|
||||
) {
|
||||
console.log(`Metadata not found for ${id}`)
|
||||
prodLog.debug(`Metadata not found for ${id}`)
|
||||
return null
|
||||
}
|
||||
|
||||
|
|
@ -2033,7 +2185,7 @@ export class S3CompatibleStorage extends BaseStorage {
|
|||
this.statisticsCache = null
|
||||
this.statisticsModified = false
|
||||
} catch (error) {
|
||||
console.error('Failed to clear storage:', error)
|
||||
prodLog.error('Failed to clear storage:', error)
|
||||
throw new Error(`Failed to clear storage: ${error}`)
|
||||
}
|
||||
}
|
||||
|
|
@ -2159,8 +2311,9 @@ export class S3CompatibleStorage extends BaseStorage {
|
|||
}
|
||||
|
||||
/**
|
||||
* Get the legacy statistics key (for backward compatibility)
|
||||
* Get the legacy statistics key (DEPRECATED - /index folder is auto-cleaned)
|
||||
* @returns The legacy statistics key
|
||||
* @deprecated Legacy /index folder is automatically cleaned on initialization
|
||||
*/
|
||||
private getLegacyStatisticsKey(): string {
|
||||
return `${this.indexPrefix}${STATISTICS_KEY}.json`
|
||||
|
|
@ -2433,11 +2586,12 @@ export class S3CompatibleStorage extends BaseStorage {
|
|||
const { GetObjectCommand } = await import('@aws-sdk/client-s3')
|
||||
|
||||
// Try statistics locations in order of preference (but with timeout)
|
||||
// NOTE: Legacy /index folder is auto-cleaned on init, so only check _system
|
||||
const keys = [
|
||||
this.getCurrentStatisticsKey(),
|
||||
// Only try yesterday if it's within 2 hours of midnight to avoid unnecessary calls
|
||||
...(this.shouldTryYesterday() ? [this.getStatisticsKeyForDate(this.getYesterday())] : []),
|
||||
this.getLegacyStatisticsKey()
|
||||
...(this.shouldTryYesterday() ? [this.getStatisticsKeyForDate(this.getYesterday())] : [])
|
||||
// Legacy fallback removed - /index folder is auto-cleaned on initialization
|
||||
]
|
||||
|
||||
let statistics: StatisticsData | null = null
|
||||
|
|
|
|||
|
|
@ -84,3 +84,103 @@ export function isThreadingAvailable(): boolean {
|
|||
export async function isThreadingAvailableAsync(): Promise<boolean> {
|
||||
return areWebWorkersAvailable() || (await areWorkerThreadsAvailable())
|
||||
}
|
||||
|
||||
/**
|
||||
* Auto-detect production environment to minimize logging costs
|
||||
*/
|
||||
export function isProductionEnvironment(): boolean {
|
||||
// Node.js environment detection
|
||||
if (isNode()) {
|
||||
// Check common production environment indicators
|
||||
const nodeEnv = process.env.NODE_ENV?.toLowerCase()
|
||||
if (nodeEnv === 'production' || nodeEnv === 'prod') return true
|
||||
|
||||
// Google Cloud Run detection
|
||||
if (process.env.K_SERVICE || process.env.GOOGLE_CLOUD_PROJECT) return true
|
||||
|
||||
// AWS Lambda detection
|
||||
if (process.env.AWS_LAMBDA_FUNCTION_NAME || process.env.AWS_EXECUTION_ENV) return true
|
||||
|
||||
// Azure Functions detection
|
||||
if (process.env.AZURE_FUNCTIONS_ENVIRONMENT || process.env.WEBSITE_SITE_NAME) return true
|
||||
|
||||
// Vercel detection
|
||||
if (process.env.VERCEL || process.env.VERCEL_ENV === 'production') return true
|
||||
|
||||
// Netlify detection
|
||||
if (process.env.NETLIFY && process.env.CONTEXT === 'production') return true
|
||||
|
||||
// Heroku detection
|
||||
if (process.env.DYNO && process.env.NODE_ENV !== 'development') return true
|
||||
|
||||
// Railway detection
|
||||
if (process.env.RAILWAY_ENVIRONMENT === 'production') return true
|
||||
|
||||
// Fly.io detection
|
||||
if (process.env.FLY_APP_NAME && process.env.FLY_REGION) return true
|
||||
|
||||
// Docker in production (common patterns)
|
||||
if (process.env.DOCKER_ENV === 'production' || process.env.ENVIRONMENT === 'production') return true
|
||||
|
||||
// Generic production indicators
|
||||
if (process.env.PROD === 'true' || process.env.PRODUCTION === 'true') return true
|
||||
}
|
||||
|
||||
// Browser environment - assume development unless explicitly production
|
||||
if (isBrowser()) {
|
||||
// Check for production domain patterns
|
||||
const hostname = window?.location?.hostname
|
||||
if (hostname) {
|
||||
// Avoid logging on production domains
|
||||
if (hostname.includes('.com') || hostname.includes('.org') || hostname.includes('.net')) {
|
||||
return !hostname.includes('localhost') && !hostname.includes('127.0.0.1') && !hostname.includes('dev')
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return false
|
||||
}
|
||||
|
||||
/**
|
||||
* Get appropriate log level based on environment
|
||||
*/
|
||||
export function getLogLevel(): 'silent' | 'error' | 'warn' | 'info' | 'verbose' {
|
||||
// Explicit log level override
|
||||
const explicitLevel = process.env.BRAINY_LOG_LEVEL?.toLowerCase()
|
||||
if (explicitLevel && ['silent', 'error', 'warn', 'info', 'verbose'].includes(explicitLevel)) {
|
||||
return explicitLevel as 'silent' | 'error' | 'warn' | 'info' | 'verbose'
|
||||
}
|
||||
|
||||
// Auto-detect based on environment
|
||||
if (isProductionEnvironment()) {
|
||||
return 'error' // Only log errors in production to minimize costs
|
||||
}
|
||||
|
||||
// Development environments get more verbose logging
|
||||
if (process.env.NODE_ENV === 'development' || process.env.NODE_ENV === 'dev') {
|
||||
return 'verbose'
|
||||
}
|
||||
|
||||
// Test environments should be quieter
|
||||
if (process.env.NODE_ENV === 'test') {
|
||||
return 'warn'
|
||||
}
|
||||
|
||||
// Default to info level
|
||||
return 'info'
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if logging should be enabled for a given level
|
||||
*/
|
||||
export function shouldLog(level: 'error' | 'warn' | 'info' | 'verbose'): boolean {
|
||||
const currentLevel = getLogLevel()
|
||||
|
||||
if (currentLevel === 'silent') return false
|
||||
|
||||
const levels = ['error', 'warn', 'info', 'verbose']
|
||||
const currentIndex = levels.indexOf(currentLevel)
|
||||
const messageIndex = levels.indexOf(level)
|
||||
|
||||
return messageIndex <= currentIndex
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,8 +1,11 @@
|
|||
/**
|
||||
* Centralized logging utility for Brainy
|
||||
* Provides configurable log levels and consistent logging across the codebase
|
||||
* Automatically reduces logging in production environments to minimize costs
|
||||
*/
|
||||
|
||||
import { isProductionEnvironment, getLogLevel } from './environment.js'
|
||||
|
||||
export enum LogLevel {
|
||||
ERROR = 0,
|
||||
WARN = 1,
|
||||
|
|
@ -28,13 +31,16 @@ export interface LoggerConfig {
|
|||
class Logger {
|
||||
private static instance: Logger
|
||||
private config: LoggerConfig = {
|
||||
level: LogLevel.WARN, // Default to WARN - only critical messages
|
||||
timestamps: true,
|
||||
level: LogLevel.ERROR, // Default to ERROR in production for cost optimization
|
||||
timestamps: false, // Disable timestamps in production to reduce log size
|
||||
includeModule: true
|
||||
}
|
||||
|
||||
private constructor() {
|
||||
// Set log level from environment variable if available
|
||||
// Auto-detect production environment and set appropriate defaults
|
||||
this.applyEnvironmentDefaults()
|
||||
|
||||
// Set log level from environment variable if available (overrides auto-detection)
|
||||
const envLogLevel = process.env.BRAINY_LOG_LEVEL
|
||||
if (envLogLevel) {
|
||||
const level = LogLevel[envLogLevel.toUpperCase() as keyof typeof LogLevel]
|
||||
|
|
@ -54,6 +60,40 @@ class Logger {
|
|||
}
|
||||
}
|
||||
|
||||
private applyEnvironmentDefaults(): void {
|
||||
const envLogLevel = getLogLevel()
|
||||
|
||||
// Convert environment log level to Logger LogLevel
|
||||
switch (envLogLevel) {
|
||||
case 'silent':
|
||||
this.config.level = -1 as LogLevel // Below ERROR to silence all logs
|
||||
break
|
||||
case 'error':
|
||||
this.config.level = LogLevel.ERROR
|
||||
this.config.timestamps = false // Minimize log size in production
|
||||
break
|
||||
case 'warn':
|
||||
this.config.level = LogLevel.WARN
|
||||
this.config.timestamps = false
|
||||
break
|
||||
case 'info':
|
||||
this.config.level = LogLevel.INFO
|
||||
this.config.timestamps = true
|
||||
break
|
||||
case 'verbose':
|
||||
this.config.level = LogLevel.DEBUG
|
||||
this.config.timestamps = true
|
||||
break
|
||||
}
|
||||
|
||||
// In production environments, be extra conservative to minimize costs
|
||||
if (isProductionEnvironment()) {
|
||||
this.config.level = Math.min(this.config.level, LogLevel.ERROR)
|
||||
this.config.timestamps = false
|
||||
this.config.includeModule = false // Reduce log size
|
||||
}
|
||||
}
|
||||
|
||||
static getInstance(): Logger {
|
||||
if (!Logger.instance) {
|
||||
Logger.instance = new Logger()
|
||||
|
|
@ -164,4 +204,65 @@ export function createModuleLogger(module: string) {
|
|||
// Export function to configure logger
|
||||
export function configureLogger(config: Partial<LoggerConfig>) {
|
||||
logger.configure(config)
|
||||
}
|
||||
|
||||
/**
|
||||
* Smart console replacement that automatically reduces logging in production
|
||||
* Dramatically reduces Google Cloud Run logging costs
|
||||
*
|
||||
* Usage: Replace console.log with smartConsole.log, etc.
|
||||
*/
|
||||
export const smartConsole = {
|
||||
log: (message?: any, ...args: any[]) => {
|
||||
if (logger['shouldLog'](LogLevel.INFO, 'console')) {
|
||||
console.log(message, ...args)
|
||||
}
|
||||
},
|
||||
|
||||
info: (message?: any, ...args: any[]) => {
|
||||
if (logger['shouldLog'](LogLevel.INFO, 'console')) {
|
||||
console.info(message, ...args)
|
||||
}
|
||||
},
|
||||
|
||||
warn: (message?: any, ...args: any[]) => {
|
||||
if (logger['shouldLog'](LogLevel.WARN, 'console')) {
|
||||
console.warn(message, ...args)
|
||||
}
|
||||
},
|
||||
|
||||
error: (message?: any, ...args: any[]) => {
|
||||
if (logger['shouldLog'](LogLevel.ERROR, 'console')) {
|
||||
console.error(message, ...args)
|
||||
}
|
||||
},
|
||||
|
||||
debug: (message?: any, ...args: any[]) => {
|
||||
if (logger['shouldLog'](LogLevel.DEBUG, 'console')) {
|
||||
console.debug(message, ...args)
|
||||
}
|
||||
},
|
||||
|
||||
trace: (message?: any, ...args: any[]) => {
|
||||
if (logger['shouldLog'](LogLevel.TRACE, 'console')) {
|
||||
console.trace(message, ...args)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Production-optimized logging functions
|
||||
* These only log in non-production environments or when explicitly enabled
|
||||
*/
|
||||
export const prodLog = {
|
||||
// Only log errors in production (always visible)
|
||||
error: (message?: any, ...args: any[]) => {
|
||||
console.error(message, ...args)
|
||||
},
|
||||
|
||||
// These are suppressed in production unless BRAINY_LOG_LEVEL is set
|
||||
warn: (message?: any, ...args: any[]) => smartConsole.warn(message, ...args),
|
||||
info: (message?: any, ...args: any[]) => smartConsole.info(message, ...args),
|
||||
debug: (message?: any, ...args: any[]) => smartConsole.debug(message, ...args),
|
||||
log: (message?: any, ...args: any[]) => smartConsole.log(message, ...args)
|
||||
}
|
||||
|
|
@ -6,6 +6,7 @@
|
|||
|
||||
import { StorageAdapter } from '../coreTypes.js'
|
||||
import { MetadataIndexCache, MetadataIndexCacheConfig } from './metadataIndexCache.js'
|
||||
import { prodLog } from './logger.js'
|
||||
|
||||
export interface MetadataIndexEntry {
|
||||
field: string
|
||||
|
|
@ -684,7 +685,7 @@ export class MetadataIndexManager {
|
|||
|
||||
this.isRebuilding = true
|
||||
try {
|
||||
console.log('🔄 Starting non-blocking metadata index rebuild...')
|
||||
prodLog.info('🔄 Starting non-blocking metadata index rebuild with batch processing to prevent socket exhaustion...')
|
||||
|
||||
// Clear existing indexes
|
||||
this.indexCache.clear()
|
||||
|
|
@ -694,7 +695,7 @@ export class MetadataIndexManager {
|
|||
|
||||
// Rebuild noun metadata indexes using pagination
|
||||
let nounOffset = 0
|
||||
const nounLimit = 50 // Smaller batches to reduce blocking
|
||||
const nounLimit = 25 // Even smaller batches during initialization to prevent socket exhaustion
|
||||
let hasMoreNouns = true
|
||||
let totalNounsProcessed = 0
|
||||
|
||||
|
|
@ -703,35 +704,69 @@ export class MetadataIndexManager {
|
|||
pagination: { offset: nounOffset, limit: nounLimit }
|
||||
})
|
||||
|
||||
// Process batch with event loop yields
|
||||
for (let i = 0; i < result.items.length; i++) {
|
||||
const noun = result.items[i]
|
||||
const metadata = await this.storage.getMetadata(noun.id)
|
||||
// CRITICAL FIX: Use batch metadata reading to prevent socket exhaustion
|
||||
const nounIds = result.items.map(noun => noun.id)
|
||||
|
||||
let metadataBatch: Map<string, any>
|
||||
if (this.storage.getMetadataBatch) {
|
||||
// Use batch reading if available (prevents socket exhaustion)
|
||||
metadataBatch = await this.storage.getMetadataBatch(nounIds)
|
||||
prodLog.debug(`📦 Batch loaded ${metadataBatch.size}/${nounIds.length} metadata objects`)
|
||||
} else {
|
||||
// Fallback to individual calls with strict concurrency control
|
||||
metadataBatch = new Map()
|
||||
const CONCURRENCY_LIMIT = 3 // Very conservative limit
|
||||
|
||||
for (let i = 0; i < nounIds.length; i += CONCURRENCY_LIMIT) {
|
||||
const batch = nounIds.slice(i, i + CONCURRENCY_LIMIT)
|
||||
const batchPromises = batch.map(async (id) => {
|
||||
try {
|
||||
const metadata = await this.storage.getMetadata(id)
|
||||
return { id, metadata }
|
||||
} catch (error) {
|
||||
prodLog.debug(`Failed to read metadata for ${id}:`, error)
|
||||
return { id, metadata: null }
|
||||
}
|
||||
})
|
||||
|
||||
const batchResults = await Promise.all(batchPromises)
|
||||
for (const { id, metadata } of batchResults) {
|
||||
if (metadata) {
|
||||
metadataBatch.set(id, metadata)
|
||||
}
|
||||
}
|
||||
|
||||
// Yield between batches to prevent socket exhaustion
|
||||
await this.yieldToEventLoop()
|
||||
}
|
||||
}
|
||||
|
||||
// Process the metadata batch
|
||||
for (const noun of result.items) {
|
||||
const metadata = metadataBatch.get(noun.id)
|
||||
if (metadata) {
|
||||
// Skip flush during rebuild for performance
|
||||
await this.addToIndex(noun.id, metadata, true)
|
||||
}
|
||||
|
||||
// Yield to event loop every 10 items to prevent blocking
|
||||
if (i % 10 === 9) {
|
||||
await this.yieldToEventLoop()
|
||||
}
|
||||
}
|
||||
|
||||
// Yield after processing the entire batch
|
||||
await this.yieldToEventLoop()
|
||||
|
||||
totalNounsProcessed += result.items.length
|
||||
hasMoreNouns = result.hasMore
|
||||
nounOffset += nounLimit
|
||||
|
||||
// Progress logging and event loop yield after each batch
|
||||
if (totalNounsProcessed % 100 === 0 || !hasMoreNouns) {
|
||||
console.log(`📊 Indexed ${totalNounsProcessed} nouns...`)
|
||||
prodLog.debug(`📊 Indexed ${totalNounsProcessed} nouns...`)
|
||||
}
|
||||
await this.yieldToEventLoop()
|
||||
}
|
||||
|
||||
// Rebuild verb metadata indexes using pagination
|
||||
let verbOffset = 0
|
||||
const verbLimit = 50 // Smaller batches to reduce blocking
|
||||
const verbLimit = 25 // Even smaller batches during initialization to prevent socket exhaustion
|
||||
let hasMoreVerbs = true
|
||||
let totalVerbsProcessed = 0
|
||||
|
||||
|
|
@ -740,38 +775,72 @@ export class MetadataIndexManager {
|
|||
pagination: { offset: verbOffset, limit: verbLimit }
|
||||
})
|
||||
|
||||
// Process batch with event loop yields
|
||||
for (let i = 0; i < result.items.length; i++) {
|
||||
const verb = result.items[i]
|
||||
const metadata = await this.storage.getVerbMetadata(verb.id)
|
||||
// CRITICAL FIX: Use batch verb metadata reading to prevent socket exhaustion
|
||||
const verbIds = result.items.map(verb => verb.id)
|
||||
|
||||
let verbMetadataBatch: Map<string, any>
|
||||
if ((this.storage as any).getVerbMetadataBatch) {
|
||||
// Use batch reading if available (prevents socket exhaustion)
|
||||
verbMetadataBatch = await (this.storage as any).getVerbMetadataBatch(verbIds)
|
||||
prodLog.debug(`📦 Batch loaded ${verbMetadataBatch.size}/${verbIds.length} verb metadata objects`)
|
||||
} else {
|
||||
// Fallback to individual calls with strict concurrency control
|
||||
verbMetadataBatch = new Map()
|
||||
const CONCURRENCY_LIMIT = 3 // Very conservative limit to prevent socket exhaustion
|
||||
|
||||
for (let i = 0; i < verbIds.length; i += CONCURRENCY_LIMIT) {
|
||||
const batch = verbIds.slice(i, i + CONCURRENCY_LIMIT)
|
||||
const batchPromises = batch.map(async (id) => {
|
||||
try {
|
||||
const metadata = await this.storage.getVerbMetadata(id)
|
||||
return { id, metadata }
|
||||
} catch (error) {
|
||||
prodLog.debug(`Failed to read verb metadata for ${id}:`, error)
|
||||
return { id, metadata: null }
|
||||
}
|
||||
})
|
||||
|
||||
const batchResults = await Promise.all(batchPromises)
|
||||
for (const { id, metadata } of batchResults) {
|
||||
if (metadata) {
|
||||
verbMetadataBatch.set(id, metadata)
|
||||
}
|
||||
}
|
||||
|
||||
// Yield between batches to prevent socket exhaustion
|
||||
await this.yieldToEventLoop()
|
||||
}
|
||||
}
|
||||
|
||||
// Process the verb metadata batch
|
||||
for (const verb of result.items) {
|
||||
const metadata = verbMetadataBatch.get(verb.id)
|
||||
if (metadata) {
|
||||
// Skip flush during rebuild for performance
|
||||
await this.addToIndex(verb.id, metadata, true)
|
||||
}
|
||||
|
||||
// Yield to event loop every 10 items to prevent blocking
|
||||
if (i % 10 === 9) {
|
||||
await this.yieldToEventLoop()
|
||||
}
|
||||
}
|
||||
|
||||
// Yield after processing the entire batch
|
||||
await this.yieldToEventLoop()
|
||||
|
||||
totalVerbsProcessed += result.items.length
|
||||
hasMoreVerbs = result.hasMore
|
||||
verbOffset += verbLimit
|
||||
|
||||
// Progress logging and event loop yield after each batch
|
||||
if (totalVerbsProcessed % 100 === 0 || !hasMoreVerbs) {
|
||||
console.log(`🔗 Indexed ${totalVerbsProcessed} verbs...`)
|
||||
prodLog.debug(`🔗 Indexed ${totalVerbsProcessed} verbs...`)
|
||||
}
|
||||
await this.yieldToEventLoop()
|
||||
}
|
||||
|
||||
// Flush to storage with final yield
|
||||
console.log('💾 Flushing metadata index to storage...')
|
||||
prodLog.debug('💾 Flushing metadata index to storage...')
|
||||
await this.flush()
|
||||
await this.yieldToEventLoop()
|
||||
|
||||
console.log(`✅ Metadata index rebuild completed! Processed ${totalNounsProcessed} nouns and ${totalVerbsProcessed} verbs`)
|
||||
prodLog.info(`✅ Metadata index rebuild completed! Processed ${totalNounsProcessed} nouns and ${totalVerbsProcessed} verbs`)
|
||||
|
||||
} finally {
|
||||
this.isRebuilding = false
|
||||
|
|
|
|||
|
|
@ -4,6 +4,7 @@
|
|||
*/
|
||||
|
||||
import { isBrowser, isNode } from './environment.js'
|
||||
import { prodLog } from './logger.js'
|
||||
|
||||
// Worker pool to reuse workers
|
||||
const workerPool: Map<string, any> = new Map()
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue