fix: resolve critical count persistence bugs affecting container restarts

Fixes two critical production bugs in GCS storage adapter:

1. brain.find({ where: {...} }) returned empty array after restart
   - Root cause: Counts only persisted every 10 operations
   - If <10 entities added before restart, counts were lost
   - After restart: totalNounCount = 0, causing empty results

2. brain.init() returned 0 entities after container restart
   - Same root cause as bug #1
   - Counts file never written for small datasets
   - getStats() returned 0 despite data in GCS bucket

Changes:
- baseStorageAdapter.ts: Persist counts on EVERY operation (not every 10)
  - incrementEntityCountSafe(): Now persists immediately
  - decrementEntityCountSafe(): Now persists immediately
  - incrementVerbCount(): Now persists immediately
  - decrementVerbCount(): Now persists immediately

- gcsStorage.ts: Better error handling for count initialization
  - initializeCounts(): Fail loudly on network/permission errors
  - initializeCountsFromScan(): Throw on scan failures instead of silent fail
  - Added recovery logic with bucket scan fallback

Impact: Critical for serverless/containerized deployments (Cloud Run, Fargate, Lambda)
where containers restart frequently. The basic write→restart→read scenario now works.
This commit is contained in:
David Snelling 2025-10-09 17:12:06 -07:00
parent 2ec7536333
commit 39525aff26
2 changed files with 28 additions and 23 deletions

View file

@ -923,10 +923,10 @@ export abstract class BaseStorageAdapter implements StorageAdapter {
const mutex = getGlobalMutex()
await mutex.runExclusive(`count-entity-${type}`, async () => {
this.incrementEntityCount(type)
// Persist counts periodically
if (this.totalNounCount % 10 === 0) {
await this.persistCounts()
}
// CRITICAL FIX: Persist counts on EVERY change for cloud storage adapters
// This ensures counts survive container restarts (GCS, S3, etc.)
// For memory/file storage, this is fast; for cloud storage, it's essential
await this.persistCounts()
})
}
@ -958,9 +958,8 @@ export abstract class BaseStorageAdapter implements StorageAdapter {
const mutex = getGlobalMutex()
await mutex.runExclusive(`count-entity-${type}`, async () => {
this.decrementEntityCount(type)
if (this.totalNounCount % 10 === 0) {
await this.persistCounts()
}
// CRITICAL FIX: Persist counts on EVERY change for cloud storage adapters
await this.persistCounts()
})
}
@ -979,10 +978,8 @@ export abstract class BaseStorageAdapter implements StorageAdapter {
timestamp: Date.now()
})
// Persist counts immediately for consistency
if (this.totalVerbCount % 10 === 0) {
await this.persistCounts()
}
// CRITICAL FIX: Persist counts on EVERY change for cloud storage adapters
await this.persistCounts()
})
}
@ -1008,10 +1005,8 @@ export abstract class BaseStorageAdapter implements StorageAdapter {
timestamp: Date.now()
})
// Persist counts immediately for consistency
if (this.totalVerbCount % 10 === 0) {
await this.persistCounts()
}
// CRITICAL FIX: Persist counts on EVERY change for cloud storage adapters
await this.persistCounts()
})
}

View file

@ -1435,9 +1435,9 @@ export class GcsStorage extends BaseStorage {
* Initialize counts from storage
*/
protected async initializeCounts(): Promise<void> {
try {
const key = `${this.systemPrefix}counts.json`
const key = `${this.systemPrefix}counts.json`
try {
const file = this.bucket!.file(key)
const [contents] = await file.download()
@ -1448,14 +1448,20 @@ export class GcsStorage extends BaseStorage {
this.entityCounts = new Map(Object.entries(counts.entityCounts || {})) as Map<string, number>
this.verbCounts = new Map(Object.entries(counts.verbCounts || {})) as Map<string, number>
prodLog.info(`📊 Loaded counts: ${this.totalNounCount} nouns, ${this.totalVerbCount} verbs`)
prodLog.info(`📊 Loaded counts from storage: ${this.totalNounCount} nouns, ${this.totalVerbCount} verbs`)
} catch (error: any) {
if (error.code === 404) {
// No counts file yet - initialize from scan
prodLog.info('📊 No counts file found - initializing from storage scan...')
// No counts file yet - initialize from scan (first-time setup or counts not persisted)
prodLog.info('📊 No counts file found - this is normal for first init or if <10 entities were added')
await this.initializeCountsFromScan()
} else {
this.logger.error('Error loading counts:', error)
// CRITICAL FIX: Don't silently fail on network/permission errors
this.logger.error('❌ CRITICAL: Failed to load counts from GCS:', error)
prodLog.error(`❌ Error loading ${key}: ${error.message}`)
// Try to recover by scanning the bucket
prodLog.warn('⚠️ Attempting recovery by scanning GCS bucket...')
await this.initializeCountsFromScan()
}
}
}
@ -1465,6 +1471,8 @@ export class GcsStorage extends BaseStorage {
*/
private async initializeCountsFromScan(): Promise<void> {
try {
prodLog.info('📊 Scanning GCS bucket to initialize counts...')
// Count nouns
const [nounFiles] = await this.bucket!.getFiles({ prefix: this.nounPrefix })
this.totalNounCount = nounFiles?.filter((f: any) => f.name?.endsWith('.json')).length || 0
@ -1476,9 +1484,11 @@ export class GcsStorage extends BaseStorage {
// Save initial counts
await this.persistCounts()
prodLog.info(`✅ Initialized counts: ${this.totalNounCount} nouns, ${this.totalVerbCount} verbs`)
prodLog.info(`✅ Initialized counts from scan: ${this.totalNounCount} nouns, ${this.totalVerbCount} verbs`)
} catch (error) {
this.logger.error('Error initializing counts from scan:', error)
// CRITICAL FIX: Don't silently fail - this prevents data loss scenarios
this.logger.error('❌ CRITICAL: Failed to initialize counts from GCS bucket scan:', error)
throw new Error(`Failed to initialize GCS storage counts: ${error}. This prevents container restarts from working correctly.`)
}
}