fix(storage): add universal batch metadata reading to ALL storage adapters
CRITICAL FIX: getMetadataBatch was only implemented in S3CompatibleStorage, causing other adapters to fall back to individual calls = socket exhaustion\! Universal Implementation: - Add getMetadataBatch() to MemoryStorage (in-memory batch processing) - Add getMetadataBatch() to FileSystemStorage (10 concurrent file reads) - Add getMetadataBatch() to OPFSStorage (10 concurrent OPFS operations) - Enhanced S3CompatibleStorage with adaptive delays and timeout handling Enhanced Debugging: - Log storage adapter type and batch availability - Clear fallback warnings if batch processing unavailable - Progress reporting with success rates - Better timeout and error handling This ensures socket exhaustion prevention works regardless of storage adapter. Services using Memory/FileSystem/OPFS storage will now use batch processing instead of 1400+ individual getMetadata() calls during initialization. Expected result: Services initialize successfully across ALL storage types
This commit is contained in:
parent
befa1c2c8d
commit
3456ba332a
5 changed files with 137 additions and 8 deletions
|
|
@ -686,6 +686,8 @@ export class MetadataIndexManager {
|
|||
this.isRebuilding = true
|
||||
try {
|
||||
prodLog.info('🔄 Starting non-blocking metadata index rebuild with batch processing to prevent socket exhaustion...')
|
||||
prodLog.info(`📊 Storage adapter: ${this.storage.constructor.name}`)
|
||||
prodLog.info(`🔧 Batch processing available: ${!!this.storage.getMetadataBatch}`)
|
||||
|
||||
// Clear existing indexes
|
||||
this.indexCache.clear()
|
||||
|
|
@ -710,10 +712,13 @@ export class MetadataIndexManager {
|
|||
let metadataBatch: Map<string, any>
|
||||
if (this.storage.getMetadataBatch) {
|
||||
// Use batch reading if available (prevents socket exhaustion)
|
||||
prodLog.info(`📦 Processing metadata batch ${Math.floor(totalNounsProcessed / nounLimit) + 1} (${nounIds.length} items)...`)
|
||||
metadataBatch = await this.storage.getMetadataBatch(nounIds)
|
||||
prodLog.debug(`📦 Batch loaded ${metadataBatch.size}/${nounIds.length} metadata objects`)
|
||||
const successRate = ((metadataBatch.size / nounIds.length) * 100).toFixed(1)
|
||||
prodLog.info(`✅ Batch loaded ${metadataBatch.size}/${nounIds.length} metadata objects (${successRate}% success)`)
|
||||
} else {
|
||||
// Fallback to individual calls with strict concurrency control
|
||||
prodLog.warn(`⚠️ FALLBACK: Storage adapter missing getMetadataBatch - using individual calls with concurrency limit`)
|
||||
metadataBatch = new Map()
|
||||
const CONCURRENCY_LIMIT = 3 // Very conservative limit
|
||||
|
||||
|
|
@ -841,6 +846,7 @@ export class MetadataIndexManager {
|
|||
await this.yieldToEventLoop()
|
||||
|
||||
prodLog.info(`✅ Metadata index rebuild completed! Processed ${totalNounsProcessed} nouns and ${totalVerbsProcessed} verbs`)
|
||||
prodLog.info(`🎯 Initial indexing may show minor socket timeouts - this is expected and doesn't affect data processing`)
|
||||
|
||||
} finally {
|
||||
this.isRebuilding = false
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue