From 968e7daeabf8b205ff11322994c283caa6f26ef5 Mon Sep 17 00:00:00 2001 From: David Snelling Date: Sat, 11 Oct 2025 09:05:16 -0700 Subject: [PATCH] fix: return minimal stats with counts when statistics don't exist Previously, when no statistics file existed (e.g., first container restart after adding entities), getStatisticsData() returned null, causing HNSW rebuild to see entityCount=0 even though entities existed. This fix ensures all storage adapters return minimal StatisticsData with totalNodes/totalEdges populated from in-memory counts when statistics files don't exist yet. Changes: - gcsStorage.ts: Return minimal stats instead of null on 404 errors - memoryStorage.ts: Return minimal stats when statistics is null - opfsStorage.ts: Return minimal stats on read errors and null checks - s3CompatibleStorage.ts: Return minimal stats when no statistics found - fileSystemStorage.ts: Use in-memory counts in mergeStatistics() null case This completes the GCS persistence fix started in v3.37.3 and ensures container restarts work correctly in all cloud environments. Fixes: GCS container restart hang during HNSW index rebuild --- src/storage/adapters/fileSystemStorage.ts | 8 ++++-- src/storage/adapters/gcsStorage.ts | 16 +++++++++-- src/storage/adapters/memoryStorage.ts | 14 ++++++++- src/storage/adapters/opfsStorage.ts | 32 ++++++++++++++++++--- src/storage/adapters/s3CompatibleStorage.ts | 28 +++++++++++++++++- 5 files changed, 88 insertions(+), 10 deletions(-) diff --git a/src/storage/adapters/fileSystemStorage.ts b/src/storage/adapters/fileSystemStorage.ts index 6529447b..ccf3c622 100644 --- a/src/storage/adapters/fileSystemStorage.ts +++ b/src/storage/adapters/fileSystemStorage.ts @@ -1641,13 +1641,17 @@ export class FileSystemStorage extends BaseStorage { ): StatisticsData { // Handle null cases if (!storageStats && !localStats) { + // CRITICAL FIX (v3.37.4): Statistics files don't exist yet (first init) + // Return minimal stats with counts instead of zeros + // This prevents HNSW from seeing entityCount=0 during index rebuild return { nounCount: {}, verbCount: {}, metadataCount: {}, hnswIndexSize: 0, - totalNodes: 0, - totalEdges: 0, + totalNodes: this.totalNounCount, + totalEdges: this.totalVerbCount, + totalMetadata: 0, lastUpdated: new Date().toISOString() } } diff --git a/src/storage/adapters/gcsStorage.ts b/src/storage/adapters/gcsStorage.ts index 8b0942d9..f8a27122 100644 --- a/src/storage/adapters/gcsStorage.ts +++ b/src/storage/adapters/gcsStorage.ts @@ -1543,8 +1543,20 @@ export class GcsStorage extends BaseStorage { } } catch (error: any) { if (error.code === 404) { - this.logger.trace('Statistics not found (creating new)') - return null + // CRITICAL FIX (v3.37.4): Statistics file doesn't exist yet (first restart) + // Return minimal stats with counts instead of null + // This prevents HNSW from seeing entityCount=0 during index rebuild + this.logger.trace('Statistics file not found - returning minimal stats with counts') + return { + nounCount: {}, + verbCount: {}, + metadataCount: {}, + hnswIndexSize: 0, + totalNodes: this.totalNounCount, + totalEdges: this.totalVerbCount, + totalMetadata: 0, + lastUpdated: new Date().toISOString() + } } this.logger.error('Failed to get statistics:', error) diff --git a/src/storage/adapters/memoryStorage.ts b/src/storage/adapters/memoryStorage.ts index 8625c831..4dee05ee 100644 --- a/src/storage/adapters/memoryStorage.ts +++ b/src/storage/adapters/memoryStorage.ts @@ -682,7 +682,19 @@ export class MemoryStorage extends BaseStorage { */ protected async getStatisticsData(): Promise { if (!this.statistics) { - return null + // CRITICAL FIX (v3.37.4): Statistics don't exist yet (first init) + // Return minimal stats with counts instead of null + // This prevents HNSW from seeing entityCount=0 during index rebuild + return { + nounCount: {}, + verbCount: {}, + metadataCount: {}, + hnswIndexSize: 0, + totalNodes: this.totalNounCount, + totalEdges: this.totalVerbCount, + totalMetadata: 0, + lastUpdated: new Date().toISOString() + } } // Return a deep copy to avoid reference issues diff --git a/src/storage/adapters/opfsStorage.ts b/src/storage/adapters/opfsStorage.ts index 72a449f6..ab1fd865 100644 --- a/src/storage/adapters/opfsStorage.ts +++ b/src/storage/adapters/opfsStorage.ts @@ -1438,14 +1438,38 @@ export class OPFSStorage extends BaseStorage { } } } catch (error) { - // If the legacy file doesn't exist either, return null - return null + // CRITICAL FIX (v3.37.4): No statistics files exist (first init) + // Return minimal stats with counts instead of null + // This prevents HNSW from seeing entityCount=0 during index rebuild + return { + nounCount: {}, + verbCount: {}, + metadataCount: {}, + hnswIndexSize: 0, + totalNodes: this.totalNounCount, + totalEdges: this.totalVerbCount, + totalMetadata: 0, + lastUpdated: new Date().toISOString() + } } } } - // If we get here and statistics is null, return default statistics - return this.statistics ? this.statistics : null + // If we get here and statistics is null, return minimal stats with counts + if (!this.statistics) { + return { + nounCount: {}, + verbCount: {}, + metadataCount: {}, + hnswIndexSize: 0, + totalNodes: this.totalNounCount, + totalEdges: this.totalVerbCount, + totalMetadata: 0, + lastUpdated: new Date().toISOString() + } + } + + return this.statistics } catch (error) { console.error('Failed to get statistics data:', error) throw new Error(`Failed to get statistics data: ${error}`) diff --git a/src/storage/adapters/s3CompatibleStorage.ts b/src/storage/adapters/s3CompatibleStorage.ts index b33c0ffe..661545e6 100644 --- a/src/storage/adapters/s3CompatibleStorage.ts +++ b/src/storage/adapters/s3CompatibleStorage.ts @@ -2803,6 +2803,20 @@ export class S3CompatibleStorage extends BaseStorage { } } + // If we get here and statistics is null, return minimal stats with counts + if (!statistics) { + return { + nounCount: {}, + verbCount: {}, + metadataCount: {}, + hnswIndexSize: 0, + totalNodes: this.totalNounCount, + totalEdges: this.totalVerbCount, + totalMetadata: 0, + lastUpdated: new Date().toISOString() + } + } + // Successfully loaded statistics from storage return statistics @@ -2817,7 +2831,19 @@ export class S3CompatibleStorage extends BaseStorage { totalEdges: this.totalVerbCount } } - return null + // CRITICAL FIX (v3.37.4): Statistics file doesn't exist yet (first restart) + // Return minimal stats with counts instead of null + // This prevents HNSW from seeing entityCount=0 during index rebuild + return { + nounCount: {}, + verbCount: {}, + metadataCount: {}, + hnswIndexSize: 0, + totalNodes: this.totalNounCount, + totalEdges: this.totalVerbCount, + totalMetadata: 0, + lastUpdated: new Date().toISOString() + } } }