fix: resolve critical count persistence bugs affecting container restarts
Fixes two critical production bugs in GCS storage adapter:
1. brain.find({ where: {...} }) returned empty array after restart
- Root cause: Counts only persisted every 10 operations
- If <10 entities added before restart, counts were lost
- After restart: totalNounCount = 0, causing empty results
2. brain.init() returned 0 entities after container restart
- Same root cause as bug #1
- Counts file never written for small datasets
- getStats() returned 0 despite data in GCS bucket
Changes:
- baseStorageAdapter.ts: Persist counts on EVERY operation (not every 10)
- incrementEntityCountSafe(): Now persists immediately
- decrementEntityCountSafe(): Now persists immediately
- incrementVerbCount(): Now persists immediately
- decrementVerbCount(): Now persists immediately
- gcsStorage.ts: Better error handling for count initialization
- initializeCounts(): Fail loudly on network/permission errors
- initializeCountsFromScan(): Throw on scan failures instead of silent fail
- Added recovery logic with bucket scan fallback
Impact: Critical for serverless/containerized deployments (Cloud Run, Fargate, Lambda)
where containers restart frequently. The basic write→restart→read scenario now works.
This commit is contained in:
parent
2ec7536333
commit
39525aff26
2 changed files with 28 additions and 23 deletions
|
|
@ -923,10 +923,10 @@ export abstract class BaseStorageAdapter implements StorageAdapter {
|
|||
const mutex = getGlobalMutex()
|
||||
await mutex.runExclusive(`count-entity-${type}`, async () => {
|
||||
this.incrementEntityCount(type)
|
||||
// Persist counts periodically
|
||||
if (this.totalNounCount % 10 === 0) {
|
||||
await this.persistCounts()
|
||||
}
|
||||
// CRITICAL FIX: Persist counts on EVERY change for cloud storage adapters
|
||||
// This ensures counts survive container restarts (GCS, S3, etc.)
|
||||
// For memory/file storage, this is fast; for cloud storage, it's essential
|
||||
await this.persistCounts()
|
||||
})
|
||||
}
|
||||
|
||||
|
|
@ -958,9 +958,8 @@ export abstract class BaseStorageAdapter implements StorageAdapter {
|
|||
const mutex = getGlobalMutex()
|
||||
await mutex.runExclusive(`count-entity-${type}`, async () => {
|
||||
this.decrementEntityCount(type)
|
||||
if (this.totalNounCount % 10 === 0) {
|
||||
await this.persistCounts()
|
||||
}
|
||||
// CRITICAL FIX: Persist counts on EVERY change for cloud storage adapters
|
||||
await this.persistCounts()
|
||||
})
|
||||
}
|
||||
|
||||
|
|
@ -979,10 +978,8 @@ export abstract class BaseStorageAdapter implements StorageAdapter {
|
|||
timestamp: Date.now()
|
||||
})
|
||||
|
||||
// Persist counts immediately for consistency
|
||||
if (this.totalVerbCount % 10 === 0) {
|
||||
await this.persistCounts()
|
||||
}
|
||||
// CRITICAL FIX: Persist counts on EVERY change for cloud storage adapters
|
||||
await this.persistCounts()
|
||||
})
|
||||
}
|
||||
|
||||
|
|
@ -1008,10 +1005,8 @@ export abstract class BaseStorageAdapter implements StorageAdapter {
|
|||
timestamp: Date.now()
|
||||
})
|
||||
|
||||
// Persist counts immediately for consistency
|
||||
if (this.totalVerbCount % 10 === 0) {
|
||||
await this.persistCounts()
|
||||
}
|
||||
// CRITICAL FIX: Persist counts on EVERY change for cloud storage adapters
|
||||
await this.persistCounts()
|
||||
})
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -1435,9 +1435,9 @@ export class GcsStorage extends BaseStorage {
|
|||
* Initialize counts from storage
|
||||
*/
|
||||
protected async initializeCounts(): Promise<void> {
|
||||
try {
|
||||
const key = `${this.systemPrefix}counts.json`
|
||||
const key = `${this.systemPrefix}counts.json`
|
||||
|
||||
try {
|
||||
const file = this.bucket!.file(key)
|
||||
const [contents] = await file.download()
|
||||
|
||||
|
|
@ -1448,14 +1448,20 @@ export class GcsStorage extends BaseStorage {
|
|||
this.entityCounts = new Map(Object.entries(counts.entityCounts || {})) as Map<string, number>
|
||||
this.verbCounts = new Map(Object.entries(counts.verbCounts || {})) as Map<string, number>
|
||||
|
||||
prodLog.info(`📊 Loaded counts: ${this.totalNounCount} nouns, ${this.totalVerbCount} verbs`)
|
||||
prodLog.info(`📊 Loaded counts from storage: ${this.totalNounCount} nouns, ${this.totalVerbCount} verbs`)
|
||||
} catch (error: any) {
|
||||
if (error.code === 404) {
|
||||
// No counts file yet - initialize from scan
|
||||
prodLog.info('📊 No counts file found - initializing from storage scan...')
|
||||
// No counts file yet - initialize from scan (first-time setup or counts not persisted)
|
||||
prodLog.info('📊 No counts file found - this is normal for first init or if <10 entities were added')
|
||||
await this.initializeCountsFromScan()
|
||||
} else {
|
||||
this.logger.error('Error loading counts:', error)
|
||||
// CRITICAL FIX: Don't silently fail on network/permission errors
|
||||
this.logger.error('❌ CRITICAL: Failed to load counts from GCS:', error)
|
||||
prodLog.error(`❌ Error loading ${key}: ${error.message}`)
|
||||
|
||||
// Try to recover by scanning the bucket
|
||||
prodLog.warn('⚠️ Attempting recovery by scanning GCS bucket...')
|
||||
await this.initializeCountsFromScan()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -1465,6 +1471,8 @@ export class GcsStorage extends BaseStorage {
|
|||
*/
|
||||
private async initializeCountsFromScan(): Promise<void> {
|
||||
try {
|
||||
prodLog.info('📊 Scanning GCS bucket to initialize counts...')
|
||||
|
||||
// Count nouns
|
||||
const [nounFiles] = await this.bucket!.getFiles({ prefix: this.nounPrefix })
|
||||
this.totalNounCount = nounFiles?.filter((f: any) => f.name?.endsWith('.json')).length || 0
|
||||
|
|
@ -1476,9 +1484,11 @@ export class GcsStorage extends BaseStorage {
|
|||
// Save initial counts
|
||||
await this.persistCounts()
|
||||
|
||||
prodLog.info(`✅ Initialized counts: ${this.totalNounCount} nouns, ${this.totalVerbCount} verbs`)
|
||||
prodLog.info(`✅ Initialized counts from scan: ${this.totalNounCount} nouns, ${this.totalVerbCount} verbs`)
|
||||
} catch (error) {
|
||||
this.logger.error('Error initializing counts from scan:', error)
|
||||
// CRITICAL FIX: Don't silently fail - this prevents data loss scenarios
|
||||
this.logger.error('❌ CRITICAL: Failed to initialize counts from GCS bucket scan:', error)
|
||||
throw new Error(`Failed to initialize GCS storage counts: ${error}. This prevents container restarts from working correctly.`)
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue