feat(storage): fix critical socket exhaustion during metadata reading

CRITICAL ISSUE FIXED: Service initialization was failing due to socket exhaustion
when reading metadata for 1400+ items during index rebuild

Changes:
- Add batch metadata reading to prevent 2000+ concurrent requests
- Implement strict concurrency control (3 max concurrent requests)
- Add proper yielding between batches to prevent event loop blocking
- Reduce batch sizes during initialization (50 → 25 items per batch)
- Add getMetadataBatch() and getVerbMetadataBatch() methods to S3 storage
- Update StorageAdapter interface with batch methods
- Add production environment auto-detection for smart logging
- Auto-cleanup legacy /index folder during initialization

Socket usage: Reduced from 1400+ concurrent to 3 max concurrent

Expected production result:
- Service initialization will complete successfully
- firehoseServiceInitialized: true
- Data collection will begin normally
- No more socket exhaustion errors (100 socket limit exceeded)

Fixes: #socket-exhaustion
Breaking: None - backward compatible with fallback modes

src/utils/logger.ts

@@ -1,8 +1,11 @@
 /**
  * Centralized logging utility for Brainy
  * Provides configurable log levels and consistent logging across the codebase
+ * Automatically reduces logging in production environments to minimize costs
  */
 
+import { isProductionEnvironment, getLogLevel } from './environment.js'
+
 export enum LogLevel {
   ERROR = 0,
   WARN = 1,
@@ -28,13 +31,16 @@ export interface LoggerConfig {
 class Logger {
   private static instance: Logger
   private config: LoggerConfig = {
-    level: LogLevel.WARN, // Default to WARN - only critical messages
-    timestamps: true,
+    level: LogLevel.ERROR, // Default to ERROR in production for cost optimization
+    timestamps: false, // Disable timestamps in production to reduce log size
     includeModule: true
   }
   
   private constructor() {
-    // Set log level from environment variable if available
+    // Auto-detect production environment and set appropriate defaults
+    this.applyEnvironmentDefaults()
+    
+    // Set log level from environment variable if available (overrides auto-detection)
     const envLogLevel = process.env.BRAINY_LOG_LEVEL
     if (envLogLevel) {
       const level = LogLevel[envLogLevel.toUpperCase() as keyof typeof LogLevel]
@@ -54,6 +60,40 @@ class Logger {
     }
   }
   
+  private applyEnvironmentDefaults(): void {
+    const envLogLevel = getLogLevel()
+    
+    // Convert environment log level to Logger LogLevel
+    switch (envLogLevel) {
+      case 'silent':
+        this.config.level = -1 as LogLevel // Below ERROR to silence all logs
+        break
+      case 'error':
+        this.config.level = LogLevel.ERROR
+        this.config.timestamps = false // Minimize log size in production
+        break
+      case 'warn':
+        this.config.level = LogLevel.WARN
+        this.config.timestamps = false
+        break
+      case 'info':
+        this.config.level = LogLevel.INFO
+        this.config.timestamps = true
+        break
+      case 'verbose':
+        this.config.level = LogLevel.DEBUG
+        this.config.timestamps = true
+        break
+    }
+    
+    // In production environments, be extra conservative to minimize costs
+    if (isProductionEnvironment()) {
+      this.config.level = Math.min(this.config.level, LogLevel.ERROR)
+      this.config.timestamps = false
+      this.config.includeModule = false // Reduce log size
+    }
+  }
+  
   static getInstance(): Logger {
     if (!Logger.instance) {
       Logger.instance = new Logger()
@@ -164,4 +204,65 @@ export function createModuleLogger(module: string) {
 // Export function to configure logger
 export function configureLogger(config: Partial<LoggerConfig>) {
   logger.configure(config)
+}
+
+/**
+ * Smart console replacement that automatically reduces logging in production
+ * Dramatically reduces Google Cloud Run logging costs
+ * 
+ * Usage: Replace console.log with smartConsole.log, etc.
+ */
+export const smartConsole = {
+  log: (message?: any, ...args: any[]) => {
+    if (logger['shouldLog'](LogLevel.INFO, 'console')) {
+      console.log(message, ...args)
+    }
+  },
+  
+  info: (message?: any, ...args: any[]) => {
+    if (logger['shouldLog'](LogLevel.INFO, 'console')) {
+      console.info(message, ...args)
+    }
+  },
+  
+  warn: (message?: any, ...args: any[]) => {
+    if (logger['shouldLog'](LogLevel.WARN, 'console')) {
+      console.warn(message, ...args)
+    }
+  },
+  
+  error: (message?: any, ...args: any[]) => {
+    if (logger['shouldLog'](LogLevel.ERROR, 'console')) {
+      console.error(message, ...args)
+    }
+  },
+  
+  debug: (message?: any, ...args: any[]) => {
+    if (logger['shouldLog'](LogLevel.DEBUG, 'console')) {
+      console.debug(message, ...args)
+    }
+  },
+  
+  trace: (message?: any, ...args: any[]) => {
+    if (logger['shouldLog'](LogLevel.TRACE, 'console')) {
+      console.trace(message, ...args)
+    }
+  }
+}
+
+/**
+ * Production-optimized logging functions
+ * These only log in non-production environments or when explicitly enabled
+ */
+export const prodLog = {
+  // Only log errors in production (always visible)
+  error: (message?: any, ...args: any[]) => {
+    console.error(message, ...args)
+  },
+  
+  // These are suppressed in production unless BRAINY_LOG_LEVEL is set
+  warn: (message?: any, ...args: any[]) => smartConsole.warn(message, ...args),
+  info: (message?: any, ...args: any[]) => smartConsole.info(message, ...args),
+  debug: (message?: any, ...args: any[]) => smartConsole.debug(message, ...args),
+  log: (message?: any, ...args: any[]) => smartConsole.log(message, ...args)
 }