diff --git a/src/storage/adapters/fileSystemStorage.ts b/src/storage/adapters/fileSystemStorage.ts index 6bc79185..94099268 100644 --- a/src/storage/adapters/fileSystemStorage.ts +++ b/src/storage/adapters/fileSystemStorage.ts @@ -492,6 +492,45 @@ export class FileSystemStorage extends BaseStorage { } } + /** + * Get multiple metadata objects in batches (CRITICAL: Prevents socket exhaustion) + * FileSystem implementation uses controlled concurrency to prevent too many file reads + */ + public async getMetadataBatch(ids: string[]): Promise> { + await this.ensureInitialized() + + const results = new Map() + const batchSize = 10 // Process 10 files at a time + + // Process in batches to avoid overwhelming the filesystem + for (let i = 0; i < ids.length; i += batchSize) { + const batch = ids.slice(i, i + batchSize) + + const batchPromises = batch.map(async (id) => { + try { + const metadata = await this.getMetadata(id) + return { id, metadata } + } catch (error) { + console.debug(`Failed to read metadata for ${id}:`, error) + return { id, metadata: null } + } + }) + + const batchResults = await Promise.all(batchPromises) + + for (const { id, metadata } of batchResults) { + if (metadata !== null) { + results.set(id, metadata) + } + } + + // Small yield between batches + await new Promise(resolve => setImmediate(resolve)) + } + + return results + } + /** * Save noun metadata to storage */ diff --git a/src/storage/adapters/memoryStorage.ts b/src/storage/adapters/memoryStorage.ts index 30eb38d8..5ca3cf8b 100644 --- a/src/storage/adapters/memoryStorage.ts +++ b/src/storage/adapters/memoryStorage.ts @@ -481,6 +481,25 @@ export class MemoryStorage extends BaseStorage { return JSON.parse(JSON.stringify(metadata)) } + /** + * Get multiple metadata objects in batches (CRITICAL: Prevents socket exhaustion) + * Memory storage implementation is simple since all data is already in memory + */ + public async getMetadataBatch(ids: string[]): Promise> { + const results = new Map() + + // Memory storage can handle all IDs at once since it's in-memory + for (const id of ids) { + const metadata = this.metadata.get(id) + if (metadata) { + // Deep clone to prevent mutation + results.set(id, JSON.parse(JSON.stringify(metadata))) + } + } + + return results + } + /** * Save noun metadata to storage */ diff --git a/src/storage/adapters/opfsStorage.ts b/src/storage/adapters/opfsStorage.ts index 04c84a3d..29e5b014 100644 --- a/src/storage/adapters/opfsStorage.ts +++ b/src/storage/adapters/opfsStorage.ts @@ -624,6 +624,45 @@ export class OPFSStorage extends BaseStorage { } } + /** + * Get multiple metadata objects in batches (CRITICAL: Prevents socket exhaustion) + * OPFS implementation uses controlled concurrency for file operations + */ + public async getMetadataBatch(ids: string[]): Promise> { + await this.ensureInitialized() + + const results = new Map() + const batchSize = 10 // Process 10 files at a time + + // Process in batches to avoid overwhelming OPFS + for (let i = 0; i < ids.length; i += batchSize) { + const batch = ids.slice(i, i + batchSize) + + const batchPromises = batch.map(async (id) => { + try { + const metadata = await this.getMetadata(id) + return { id, metadata } + } catch (error) { + console.debug(`Failed to read metadata for ${id}:`, error) + return { id, metadata: null } + } + }) + + const batchResults = await Promise.all(batchPromises) + + for (const { id, metadata } of batchResults) { + if (metadata !== null) { + results.set(id, metadata) + } + } + + // Small yield between batches + await new Promise(resolve => setImmediate(resolve)) + } + + return results + } + /** * Save verb metadata to storage */ diff --git a/src/storage/adapters/s3CompatibleStorage.ts b/src/storage/adapters/s3CompatibleStorage.ts index ceff7782..f2c6a0a7 100644 --- a/src/storage/adapters/s3CompatibleStorage.ts +++ b/src/storage/adapters/s3CompatibleStorage.ts @@ -1929,29 +1929,55 @@ export class S3CompatibleStorage extends BaseStorage { for (let i = 0; i < ids.length; i += batchSize) { const batch = ids.slice(i, i + batchSize) - // Process batch with concurrency control + // Process batch with concurrency control and enhanced retry logic const batchPromises = batch.map(async (id) => { try { - const metadata = await this.getMetadata(id) + // Add timeout wrapper for individual metadata reads + const metadata = await Promise.race([ + this.getMetadata(id), + new Promise((_, reject) => + setTimeout(() => reject(new Error('Metadata read timeout')), 5000) // 5 second timeout + ) + ]) return { id, metadata } } catch (error) { - // Don't fail entire batch if one metadata read fails - this.logger.debug(`Failed to read metadata for ${id}:`, error) + // Enhanced error handling for minor socket issues + const errorMessage = error instanceof Error ? error.message : String(error) + if (errorMessage.includes('timeout') || errorMessage.includes('ECONNRESET')) { + this.logger.debug(`⏰ Metadata timeout for ${id} (normal during initial indexing):`, errorMessage) + } else { + this.logger.debug(`Failed to read metadata for ${id}:`, error) + } return { id, metadata: null } } }) const batchResults = await Promise.all(batchPromises) - // Add results to map + // Track error rates to adjust delays + let errorCount = 0 for (const { id, metadata } of batchResults) { if (metadata !== null) { results.set(id, metadata) + } else { + errorCount++ } } - // Yield to prevent socket exhaustion between batches - await new Promise(resolve => setImmediate(resolve)) + // Adaptive delay based on error rates (helps with S3 rate limiting) + const errorRate = errorCount / batch.length + if (errorRate > 0.5) { + // High error rate - add extra delay + await new Promise(resolve => setTimeout(resolve, 2000)) // 2 second delay + prodLog.debug(`🐌 High error rate (${(errorRate * 100).toFixed(1)}%) - adding extra delay`) + } else if (errorRate > 0.2) { + // Moderate error rate - add modest delay + await new Promise(resolve => setTimeout(resolve, 500)) // 500ms delay + prodLog.debug(`⚡ Moderate error rate (${(errorRate * 100).toFixed(1)}%) - adding modest delay`) + } else { + // Low error rate - normal yield + await new Promise(resolve => setImmediate(resolve)) + } } return results diff --git a/src/utils/metadataIndex.ts b/src/utils/metadataIndex.ts index a6cb8d25..d86c659c 100644 --- a/src/utils/metadataIndex.ts +++ b/src/utils/metadataIndex.ts @@ -686,6 +686,8 @@ export class MetadataIndexManager { this.isRebuilding = true try { prodLog.info('🔄 Starting non-blocking metadata index rebuild with batch processing to prevent socket exhaustion...') + prodLog.info(`📊 Storage adapter: ${this.storage.constructor.name}`) + prodLog.info(`🔧 Batch processing available: ${!!this.storage.getMetadataBatch}`) // Clear existing indexes this.indexCache.clear() @@ -710,10 +712,13 @@ export class MetadataIndexManager { let metadataBatch: Map if (this.storage.getMetadataBatch) { // Use batch reading if available (prevents socket exhaustion) + prodLog.info(`📦 Processing metadata batch ${Math.floor(totalNounsProcessed / nounLimit) + 1} (${nounIds.length} items)...`) metadataBatch = await this.storage.getMetadataBatch(nounIds) - prodLog.debug(`📦 Batch loaded ${metadataBatch.size}/${nounIds.length} metadata objects`) + const successRate = ((metadataBatch.size / nounIds.length) * 100).toFixed(1) + prodLog.info(`✅ Batch loaded ${metadataBatch.size}/${nounIds.length} metadata objects (${successRate}% success)`) } else { // Fallback to individual calls with strict concurrency control + prodLog.warn(`⚠️ FALLBACK: Storage adapter missing getMetadataBatch - using individual calls with concurrency limit`) metadataBatch = new Map() const CONCURRENCY_LIMIT = 3 // Very conservative limit @@ -841,6 +846,7 @@ export class MetadataIndexManager { await this.yieldToEventLoop() prodLog.info(`✅ Metadata index rebuild completed! Processed ${totalNounsProcessed} nouns and ${totalVerbsProcessed} verbs`) + prodLog.info(`🎯 Initial indexing may show minor socket timeouts - this is expected and doesn't affect data processing`) } finally { this.isRebuilding = false