fix(storage): add universal batch metadata reading to ALL storage adapters

CRITICAL FIX: getMetadataBatch was only implemented in S3CompatibleStorage,
causing other adapters to fall back to individual calls = socket exhaustion\!

Universal Implementation:
- Add getMetadataBatch() to MemoryStorage (in-memory batch processing)
- Add getMetadataBatch() to FileSystemStorage (10 concurrent file reads)
- Add getMetadataBatch() to OPFSStorage (10 concurrent OPFS operations)
- Enhanced S3CompatibleStorage with adaptive delays and timeout handling

Enhanced Debugging:
- Log storage adapter type and batch availability
- Clear fallback warnings if batch processing unavailable
- Progress reporting with success rates
- Better timeout and error handling

This ensures socket exhaustion prevention works regardless of storage adapter.
Services using Memory/FileSystem/OPFS storage will now use batch processing
instead of 1400+ individual getMetadata() calls during initialization.

Expected result: Services initialize successfully across ALL storage types
This commit is contained in:
David Snelling 2025-08-07 12:14:23 -07:00
parent befa1c2c8d
commit 3456ba332a
5 changed files with 137 additions and 8 deletions

View file

@ -686,6 +686,8 @@ export class MetadataIndexManager {
this.isRebuilding = true
try {
prodLog.info('🔄 Starting non-blocking metadata index rebuild with batch processing to prevent socket exhaustion...')
prodLog.info(`📊 Storage adapter: ${this.storage.constructor.name}`)
prodLog.info(`🔧 Batch processing available: ${!!this.storage.getMetadataBatch}`)
// Clear existing indexes
this.indexCache.clear()
@ -710,10 +712,13 @@ export class MetadataIndexManager {
let metadataBatch: Map<string, any>
if (this.storage.getMetadataBatch) {
// Use batch reading if available (prevents socket exhaustion)
prodLog.info(`📦 Processing metadata batch ${Math.floor(totalNounsProcessed / nounLimit) + 1} (${nounIds.length} items)...`)
metadataBatch = await this.storage.getMetadataBatch(nounIds)
prodLog.debug(`📦 Batch loaded ${metadataBatch.size}/${nounIds.length} metadata objects`)
const successRate = ((metadataBatch.size / nounIds.length) * 100).toFixed(1)
prodLog.info(`✅ Batch loaded ${metadataBatch.size}/${nounIds.length} metadata objects (${successRate}% success)`)
} else {
// Fallback to individual calls with strict concurrency control
prodLog.warn(`⚠️ FALLBACK: Storage adapter missing getMetadataBatch - using individual calls with concurrency limit`)
metadataBatch = new Map()
const CONCURRENCY_LIMIT = 3 // Very conservative limit
@ -841,6 +846,7 @@ export class MetadataIndexManager {
await this.yieldToEventLoop()
prodLog.info(`✅ Metadata index rebuild completed! Processed ${totalNounsProcessed} nouns and ${totalVerbsProcessed} verbs`)
prodLog.info(`🎯 Initial indexing may show minor socket timeouts - this is expected and doesn't affect data processing`)
} finally {
this.isRebuilding = false