fix: return minimal stats with counts when statistics don't exist

Previously, when no statistics file existed (e.g., first container restart
after adding entities), getStatisticsData() returned null, causing HNSW
rebuild to see entityCount=0 even though entities existed.

This fix ensures all storage adapters return minimal StatisticsData with
totalNodes/totalEdges populated from in-memory counts when statistics files
don't exist yet.

Changes:
- gcsStorage.ts: Return minimal stats instead of null on 404 errors
- memoryStorage.ts: Return minimal stats when statistics is null
- opfsStorage.ts: Return minimal stats on read errors and null checks
- s3CompatibleStorage.ts: Return minimal stats when no statistics found
- fileSystemStorage.ts: Use in-memory counts in mergeStatistics() null case

This completes the GCS persistence fix started in v3.37.3 and ensures
container restarts work correctly in all cloud environments.

Fixes: GCS container restart hang during HNSW index rebuild
This commit is contained in:
David Snelling 2025-10-11 09:05:16 -07:00
parent e9718b0145
commit 968e7daeab
5 changed files with 88 additions and 10 deletions

View file

@ -1641,13 +1641,17 @@ export class FileSystemStorage extends BaseStorage {
): StatisticsData {
// Handle null cases
if (!storageStats && !localStats) {
// CRITICAL FIX (v3.37.4): Statistics files don't exist yet (first init)
// Return minimal stats with counts instead of zeros
// This prevents HNSW from seeing entityCount=0 during index rebuild
return {
nounCount: {},
verbCount: {},
metadataCount: {},
hnswIndexSize: 0,
totalNodes: 0,
totalEdges: 0,
totalNodes: this.totalNounCount,
totalEdges: this.totalVerbCount,
totalMetadata: 0,
lastUpdated: new Date().toISOString()
}
}

View file

@ -1543,8 +1543,20 @@ export class GcsStorage extends BaseStorage {
}
} catch (error: any) {
if (error.code === 404) {
this.logger.trace('Statistics not found (creating new)')
return null
// CRITICAL FIX (v3.37.4): Statistics file doesn't exist yet (first restart)
// Return minimal stats with counts instead of null
// This prevents HNSW from seeing entityCount=0 during index rebuild
this.logger.trace('Statistics file not found - returning minimal stats with counts')
return {
nounCount: {},
verbCount: {},
metadataCount: {},
hnswIndexSize: 0,
totalNodes: this.totalNounCount,
totalEdges: this.totalVerbCount,
totalMetadata: 0,
lastUpdated: new Date().toISOString()
}
}
this.logger.error('Failed to get statistics:', error)

View file

@ -682,7 +682,19 @@ export class MemoryStorage extends BaseStorage {
*/
protected async getStatisticsData(): Promise<StatisticsData | null> {
if (!this.statistics) {
return null
// CRITICAL FIX (v3.37.4): Statistics don't exist yet (first init)
// Return minimal stats with counts instead of null
// This prevents HNSW from seeing entityCount=0 during index rebuild
return {
nounCount: {},
verbCount: {},
metadataCount: {},
hnswIndexSize: 0,
totalNodes: this.totalNounCount,
totalEdges: this.totalVerbCount,
totalMetadata: 0,
lastUpdated: new Date().toISOString()
}
}
// Return a deep copy to avoid reference issues

View file

@ -1438,14 +1438,38 @@ export class OPFSStorage extends BaseStorage {
}
}
} catch (error) {
// If the legacy file doesn't exist either, return null
return null
// CRITICAL FIX (v3.37.4): No statistics files exist (first init)
// Return minimal stats with counts instead of null
// This prevents HNSW from seeing entityCount=0 during index rebuild
return {
nounCount: {},
verbCount: {},
metadataCount: {},
hnswIndexSize: 0,
totalNodes: this.totalNounCount,
totalEdges: this.totalVerbCount,
totalMetadata: 0,
lastUpdated: new Date().toISOString()
}
}
}
}
// If we get here and statistics is null, return default statistics
return this.statistics ? this.statistics : null
// If we get here and statistics is null, return minimal stats with counts
if (!this.statistics) {
return {
nounCount: {},
verbCount: {},
metadataCount: {},
hnswIndexSize: 0,
totalNodes: this.totalNounCount,
totalEdges: this.totalVerbCount,
totalMetadata: 0,
lastUpdated: new Date().toISOString()
}
}
return this.statistics
} catch (error) {
console.error('Failed to get statistics data:', error)
throw new Error(`Failed to get statistics data: ${error}`)

View file

@ -2803,6 +2803,20 @@ export class S3CompatibleStorage extends BaseStorage {
}
}
// If we get here and statistics is null, return minimal stats with counts
if (!statistics) {
return {
nounCount: {},
verbCount: {},
metadataCount: {},
hnswIndexSize: 0,
totalNodes: this.totalNounCount,
totalEdges: this.totalVerbCount,
totalMetadata: 0,
lastUpdated: new Date().toISOString()
}
}
// Successfully loaded statistics from storage
return statistics
@ -2817,7 +2831,19 @@ export class S3CompatibleStorage extends BaseStorage {
totalEdges: this.totalVerbCount
}
}
return null
// CRITICAL FIX (v3.37.4): Statistics file doesn't exist yet (first restart)
// Return minimal stats with counts instead of null
// This prevents HNSW from seeing entityCount=0 during index rebuild
return {
nounCount: {},
verbCount: {},
metadataCount: {},
hnswIndexSize: 0,
totalNodes: this.totalNounCount,
totalEdges: this.totalVerbCount,
totalMetadata: 0,
lastUpdated: new Date().toISOString()
}
}
}