feat(storage): fix critical socket exhaustion during metadata reading

CRITICAL ISSUE FIXED: Service initialization was failing due to socket exhaustion
when reading metadata for 1400+ items during index rebuild

Changes:
- Add batch metadata reading to prevent 2000+ concurrent requests
- Implement strict concurrency control (3 max concurrent requests)
- Add proper yielding between batches to prevent event loop blocking
- Reduce batch sizes during initialization (50 → 25 items per batch)
- Add getMetadataBatch() and getVerbMetadataBatch() methods to S3 storage
- Update StorageAdapter interface with batch methods
- Add production environment auto-detection for smart logging
- Auto-cleanup legacy /index folder during initialization

Socket usage: Reduced from 1400+ concurrent to 3 max concurrent

Expected production result:
- Service initialization will complete successfully
- firehoseServiceInitialized: true
- Data collection will begin normally
- No more socket exhaustion errors (100 socket limit exceeded)

Fixes: #socket-exhaustion
Breaking: None - backward compatible with fallback modes

src/utils/environment.ts

@@ -84,3 +84,103 @@ export function isThreadingAvailable(): boolean {
 export async function isThreadingAvailableAsync(): Promise<boolean> {
   return areWebWorkersAvailable() || (await areWorkerThreadsAvailable())
 }
+
+/**
+ * Auto-detect production environment to minimize logging costs
+ */
+export function isProductionEnvironment(): boolean {
+  // Node.js environment detection
+  if (isNode()) {
+    // Check common production environment indicators
+    const nodeEnv = process.env.NODE_ENV?.toLowerCase()
+    if (nodeEnv === 'production' || nodeEnv === 'prod') return true
+    
+    // Google Cloud Run detection
+    if (process.env.K_SERVICE || process.env.GOOGLE_CLOUD_PROJECT) return true
+    
+    // AWS Lambda detection  
+    if (process.env.AWS_LAMBDA_FUNCTION_NAME || process.env.AWS_EXECUTION_ENV) return true
+    
+    // Azure Functions detection
+    if (process.env.AZURE_FUNCTIONS_ENVIRONMENT || process.env.WEBSITE_SITE_NAME) return true
+    
+    // Vercel detection
+    if (process.env.VERCEL || process.env.VERCEL_ENV === 'production') return true
+    
+    // Netlify detection
+    if (process.env.NETLIFY && process.env.CONTEXT === 'production') return true
+    
+    // Heroku detection
+    if (process.env.DYNO && process.env.NODE_ENV !== 'development') return true
+    
+    // Railway detection
+    if (process.env.RAILWAY_ENVIRONMENT === 'production') return true
+    
+    // Fly.io detection
+    if (process.env.FLY_APP_NAME && process.env.FLY_REGION) return true
+    
+    // Docker in production (common patterns)
+    if (process.env.DOCKER_ENV === 'production' || process.env.ENVIRONMENT === 'production') return true
+    
+    // Generic production indicators
+    if (process.env.PROD === 'true' || process.env.PRODUCTION === 'true') return true
+  }
+  
+  // Browser environment - assume development unless explicitly production
+  if (isBrowser()) {
+    // Check for production domain patterns
+    const hostname = window?.location?.hostname
+    if (hostname) {
+      // Avoid logging on production domains
+      if (hostname.includes('.com') || hostname.includes('.org') || hostname.includes('.net')) {
+        return !hostname.includes('localhost') && !hostname.includes('127.0.0.1') && !hostname.includes('dev')
+      }
+    }
+  }
+  
+  return false
+}
+
+/**
+ * Get appropriate log level based on environment
+ */
+export function getLogLevel(): 'silent' | 'error' | 'warn' | 'info' | 'verbose' {
+  // Explicit log level override
+  const explicitLevel = process.env.BRAINY_LOG_LEVEL?.toLowerCase()
+  if (explicitLevel && ['silent', 'error', 'warn', 'info', 'verbose'].includes(explicitLevel)) {
+    return explicitLevel as 'silent' | 'error' | 'warn' | 'info' | 'verbose'
+  }
+  
+  // Auto-detect based on environment
+  if (isProductionEnvironment()) {
+    return 'error' // Only log errors in production to minimize costs
+  }
+  
+  // Development environments get more verbose logging
+  if (process.env.NODE_ENV === 'development' || process.env.NODE_ENV === 'dev') {
+    return 'verbose'
+  }
+  
+  // Test environments should be quieter
+  if (process.env.NODE_ENV === 'test') {
+    return 'warn'
+  }
+  
+  // Default to info level
+  return 'info'
+}
+
+/**
+ * Check if logging should be enabled for a given level
+ */
+export function shouldLog(level: 'error' | 'warn' | 'info' | 'verbose'): boolean {
+  const currentLevel = getLogLevel()
+  
+  if (currentLevel === 'silent') return false
+  
+  const levels = ['error', 'warn', 'info', 'verbose']
+  const currentIndex = levels.indexOf(currentLevel)
+  const messageIndex = levels.indexOf(level)
+  
+  return messageIndex <= currentIndex
+}