From 39525aff26c973cb60be8556d1fef810e5d32cdd Mon Sep 17 00:00:00 2001 From: David Snelling Date: Thu, 9 Oct 2025 17:12:06 -0700 Subject: [PATCH] fix: resolve critical count persistence bugs affecting container restarts MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fixes two critical production bugs in GCS storage adapter: 1. brain.find({ where: {...} }) returned empty array after restart - Root cause: Counts only persisted every 10 operations - If <10 entities added before restart, counts were lost - After restart: totalNounCount = 0, causing empty results 2. brain.init() returned 0 entities after container restart - Same root cause as bug #1 - Counts file never written for small datasets - getStats() returned 0 despite data in GCS bucket Changes: - baseStorageAdapter.ts: Persist counts on EVERY operation (not every 10) - incrementEntityCountSafe(): Now persists immediately - decrementEntityCountSafe(): Now persists immediately - incrementVerbCount(): Now persists immediately - decrementVerbCount(): Now persists immediately - gcsStorage.ts: Better error handling for count initialization - initializeCounts(): Fail loudly on network/permission errors - initializeCountsFromScan(): Throw on scan failures instead of silent fail - Added recovery logic with bucket scan fallback Impact: Critical for serverless/containerized deployments (Cloud Run, Fargate, Lambda) where containers restart frequently. The basic writeโ†’restartโ†’read scenario now works. --- src/storage/adapters/baseStorageAdapter.ts | 25 +++++++++------------ src/storage/adapters/gcsStorage.ts | 26 +++++++++++++++------- 2 files changed, 28 insertions(+), 23 deletions(-) diff --git a/src/storage/adapters/baseStorageAdapter.ts b/src/storage/adapters/baseStorageAdapter.ts index 77e5de7f..bb6b5043 100644 --- a/src/storage/adapters/baseStorageAdapter.ts +++ b/src/storage/adapters/baseStorageAdapter.ts @@ -923,10 +923,10 @@ export abstract class BaseStorageAdapter implements StorageAdapter { const mutex = getGlobalMutex() await mutex.runExclusive(`count-entity-${type}`, async () => { this.incrementEntityCount(type) - // Persist counts periodically - if (this.totalNounCount % 10 === 0) { - await this.persistCounts() - } + // CRITICAL FIX: Persist counts on EVERY change for cloud storage adapters + // This ensures counts survive container restarts (GCS, S3, etc.) + // For memory/file storage, this is fast; for cloud storage, it's essential + await this.persistCounts() }) } @@ -958,9 +958,8 @@ export abstract class BaseStorageAdapter implements StorageAdapter { const mutex = getGlobalMutex() await mutex.runExclusive(`count-entity-${type}`, async () => { this.decrementEntityCount(type) - if (this.totalNounCount % 10 === 0) { - await this.persistCounts() - } + // CRITICAL FIX: Persist counts on EVERY change for cloud storage adapters + await this.persistCounts() }) } @@ -979,10 +978,8 @@ export abstract class BaseStorageAdapter implements StorageAdapter { timestamp: Date.now() }) - // Persist counts immediately for consistency - if (this.totalVerbCount % 10 === 0) { - await this.persistCounts() - } + // CRITICAL FIX: Persist counts on EVERY change for cloud storage adapters + await this.persistCounts() }) } @@ -1008,10 +1005,8 @@ export abstract class BaseStorageAdapter implements StorageAdapter { timestamp: Date.now() }) - // Persist counts immediately for consistency - if (this.totalVerbCount % 10 === 0) { - await this.persistCounts() - } + // CRITICAL FIX: Persist counts on EVERY change for cloud storage adapters + await this.persistCounts() }) } diff --git a/src/storage/adapters/gcsStorage.ts b/src/storage/adapters/gcsStorage.ts index 78028fa2..331deb7f 100644 --- a/src/storage/adapters/gcsStorage.ts +++ b/src/storage/adapters/gcsStorage.ts @@ -1435,9 +1435,9 @@ export class GcsStorage extends BaseStorage { * Initialize counts from storage */ protected async initializeCounts(): Promise { - try { - const key = `${this.systemPrefix}counts.json` + const key = `${this.systemPrefix}counts.json` + try { const file = this.bucket!.file(key) const [contents] = await file.download() @@ -1448,14 +1448,20 @@ export class GcsStorage extends BaseStorage { this.entityCounts = new Map(Object.entries(counts.entityCounts || {})) as Map this.verbCounts = new Map(Object.entries(counts.verbCounts || {})) as Map - prodLog.info(`๐Ÿ“Š Loaded counts: ${this.totalNounCount} nouns, ${this.totalVerbCount} verbs`) + prodLog.info(`๐Ÿ“Š Loaded counts from storage: ${this.totalNounCount} nouns, ${this.totalVerbCount} verbs`) } catch (error: any) { if (error.code === 404) { - // No counts file yet - initialize from scan - prodLog.info('๐Ÿ“Š No counts file found - initializing from storage scan...') + // No counts file yet - initialize from scan (first-time setup or counts not persisted) + prodLog.info('๐Ÿ“Š No counts file found - this is normal for first init or if <10 entities were added') await this.initializeCountsFromScan() } else { - this.logger.error('Error loading counts:', error) + // CRITICAL FIX: Don't silently fail on network/permission errors + this.logger.error('โŒ CRITICAL: Failed to load counts from GCS:', error) + prodLog.error(`โŒ Error loading ${key}: ${error.message}`) + + // Try to recover by scanning the bucket + prodLog.warn('โš ๏ธ Attempting recovery by scanning GCS bucket...') + await this.initializeCountsFromScan() } } } @@ -1465,6 +1471,8 @@ export class GcsStorage extends BaseStorage { */ private async initializeCountsFromScan(): Promise { try { + prodLog.info('๐Ÿ“Š Scanning GCS bucket to initialize counts...') + // Count nouns const [nounFiles] = await this.bucket!.getFiles({ prefix: this.nounPrefix }) this.totalNounCount = nounFiles?.filter((f: any) => f.name?.endsWith('.json')).length || 0 @@ -1476,9 +1484,11 @@ export class GcsStorage extends BaseStorage { // Save initial counts await this.persistCounts() - prodLog.info(`โœ… Initialized counts: ${this.totalNounCount} nouns, ${this.totalVerbCount} verbs`) + prodLog.info(`โœ… Initialized counts from scan: ${this.totalNounCount} nouns, ${this.totalVerbCount} verbs`) } catch (error) { - this.logger.error('Error initializing counts from scan:', error) + // CRITICAL FIX: Don't silently fail - this prevents data loss scenarios + this.logger.error('โŒ CRITICAL: Failed to initialize counts from GCS bucket scan:', error) + throw new Error(`Failed to initialize GCS storage counts: ${error}. This prevents container restarts from working correctly.`) } }