diff --git a/src/storage/adapters/baseStorageAdapter.ts b/src/storage/adapters/baseStorageAdapter.ts index 77e5de7f..bb6b5043 100644 --- a/src/storage/adapters/baseStorageAdapter.ts +++ b/src/storage/adapters/baseStorageAdapter.ts @@ -923,10 +923,10 @@ export abstract class BaseStorageAdapter implements StorageAdapter { const mutex = getGlobalMutex() await mutex.runExclusive(`count-entity-${type}`, async () => { this.incrementEntityCount(type) - // Persist counts periodically - if (this.totalNounCount % 10 === 0) { - await this.persistCounts() - } + // CRITICAL FIX: Persist counts on EVERY change for cloud storage adapters + // This ensures counts survive container restarts (GCS, S3, etc.) + // For memory/file storage, this is fast; for cloud storage, it's essential + await this.persistCounts() }) } @@ -958,9 +958,8 @@ export abstract class BaseStorageAdapter implements StorageAdapter { const mutex = getGlobalMutex() await mutex.runExclusive(`count-entity-${type}`, async () => { this.decrementEntityCount(type) - if (this.totalNounCount % 10 === 0) { - await this.persistCounts() - } + // CRITICAL FIX: Persist counts on EVERY change for cloud storage adapters + await this.persistCounts() }) } @@ -979,10 +978,8 @@ export abstract class BaseStorageAdapter implements StorageAdapter { timestamp: Date.now() }) - // Persist counts immediately for consistency - if (this.totalVerbCount % 10 === 0) { - await this.persistCounts() - } + // CRITICAL FIX: Persist counts on EVERY change for cloud storage adapters + await this.persistCounts() }) } @@ -1008,10 +1005,8 @@ export abstract class BaseStorageAdapter implements StorageAdapter { timestamp: Date.now() }) - // Persist counts immediately for consistency - if (this.totalVerbCount % 10 === 0) { - await this.persistCounts() - } + // CRITICAL FIX: Persist counts on EVERY change for cloud storage adapters + await this.persistCounts() }) } diff --git a/src/storage/adapters/gcsStorage.ts b/src/storage/adapters/gcsStorage.ts index 78028fa2..331deb7f 100644 --- a/src/storage/adapters/gcsStorage.ts +++ b/src/storage/adapters/gcsStorage.ts @@ -1435,9 +1435,9 @@ export class GcsStorage extends BaseStorage { * Initialize counts from storage */ protected async initializeCounts(): Promise { - try { - const key = `${this.systemPrefix}counts.json` + const key = `${this.systemPrefix}counts.json` + try { const file = this.bucket!.file(key) const [contents] = await file.download() @@ -1448,14 +1448,20 @@ export class GcsStorage extends BaseStorage { this.entityCounts = new Map(Object.entries(counts.entityCounts || {})) as Map this.verbCounts = new Map(Object.entries(counts.verbCounts || {})) as Map - prodLog.info(`📊 Loaded counts: ${this.totalNounCount} nouns, ${this.totalVerbCount} verbs`) + prodLog.info(`📊 Loaded counts from storage: ${this.totalNounCount} nouns, ${this.totalVerbCount} verbs`) } catch (error: any) { if (error.code === 404) { - // No counts file yet - initialize from scan - prodLog.info('📊 No counts file found - initializing from storage scan...') + // No counts file yet - initialize from scan (first-time setup or counts not persisted) + prodLog.info('📊 No counts file found - this is normal for first init or if <10 entities were added') await this.initializeCountsFromScan() } else { - this.logger.error('Error loading counts:', error) + // CRITICAL FIX: Don't silently fail on network/permission errors + this.logger.error('❌ CRITICAL: Failed to load counts from GCS:', error) + prodLog.error(`❌ Error loading ${key}: ${error.message}`) + + // Try to recover by scanning the bucket + prodLog.warn('⚠️ Attempting recovery by scanning GCS bucket...') + await this.initializeCountsFromScan() } } } @@ -1465,6 +1471,8 @@ export class GcsStorage extends BaseStorage { */ private async initializeCountsFromScan(): Promise { try { + prodLog.info('📊 Scanning GCS bucket to initialize counts...') + // Count nouns const [nounFiles] = await this.bucket!.getFiles({ prefix: this.nounPrefix }) this.totalNounCount = nounFiles?.filter((f: any) => f.name?.endsWith('.json')).length || 0 @@ -1476,9 +1484,11 @@ export class GcsStorage extends BaseStorage { // Save initial counts await this.persistCounts() - prodLog.info(`✅ Initialized counts: ${this.totalNounCount} nouns, ${this.totalVerbCount} verbs`) + prodLog.info(`✅ Initialized counts from scan: ${this.totalNounCount} nouns, ${this.totalVerbCount} verbs`) } catch (error) { - this.logger.error('Error initializing counts from scan:', error) + // CRITICAL FIX: Don't silently fail - this prevents data loss scenarios + this.logger.error('❌ CRITICAL: Failed to initialize counts from GCS bucket scan:', error) + throw new Error(`Failed to initialize GCS storage counts: ${error}. This prevents container restarts from working correctly.`) } }