From a21a84545ab1d041f46e04d5190a456ea3802852 Mon Sep 17 00:00:00 2001 From: David Snelling Date: Fri, 10 Oct 2025 17:48:40 -0700 Subject: [PATCH] fix: populate totalNodes/totalEdges in ALL storage adapters for HNSW rebuild MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Critical fix for v3.37.1 GCS persistence bug where HNSW rebuild reported "Preloading 0 vectors" and hung indefinitely during container restarts. Root Cause: HNSW rebuild (hnswIndex.ts:835) calls storage.getStatistics() and expects stats.totalNodes to determine entity count for adaptive caching. When totalNodes was undefined, entityCount evaluated to 0, causing HNSW to report "0 vectors" and hang waiting for entities that it couldn't find. Fixes Applied: - GcsStorage: Populate totalNodes/totalEdges from this.totalNounCount/totalVerbCount - MemoryStorage: Populate totalNodes/totalEdges from in-memory counts - OPFSStorage: Populate totalNodes/totalEdges in ALL return paths (cache, current day, yesterday, legacy) - S3CompatibleStorage: Populate totalNodes/totalEdges in ALL return paths (cache, fresh stats, error fallback) Impact: - ✅ GCS container restarts now work correctly - ✅ HNSW rebuild correctly detects entity count - ✅ Adaptive caching strategy works as designed - ✅ All storage adapters now consistent Files exist in GCS, counts load correctly, but HNSW couldn't see them because getStatisticsData() didn't populate the totalNodes field that HNSW depends on. Closes: BRAINY_V3_37_1_INDEX_REBUILD_BUG.md Related: v3.37.0 (metadata reconstruction), v3.37.1 (getNoun_internal fix) --- src/storage/adapters/gcsStorage.ts | 10 ++++++- src/storage/adapters/memoryStorage.ts | 10 +++++-- src/storage/adapters/opfsStorage.ts | 16 +++++++++++ src/storage/adapters/s3CompatibleStorage.ts | 32 +++++++++++++++++---- 4 files changed, 58 insertions(+), 10 deletions(-) diff --git a/src/storage/adapters/gcsStorage.ts b/src/storage/adapters/gcsStorage.ts index 51b6a4ae..8b0942d9 100644 --- a/src/storage/adapters/gcsStorage.ts +++ b/src/storage/adapters/gcsStorage.ts @@ -1532,7 +1532,15 @@ export class GcsStorage extends BaseStorage { const statistics = JSON.parse(contents.toString()) this.logger.trace('Statistics retrieved successfully') - return statistics + + // CRITICAL FIX: Populate totalNodes and totalEdges from in-memory counts + // HNSW rebuild depends on these fields to determine entity count + return { + ...statistics, + totalNodes: this.totalNounCount, + totalEdges: this.totalVerbCount, + lastUpdated: new Date().toISOString() + } } catch (error: any) { if (error.code === 404) { this.logger.trace('Statistics not found (creating new)') diff --git a/src/storage/adapters/memoryStorage.ts b/src/storage/adapters/memoryStorage.ts index 7d2edadf..8625c831 100644 --- a/src/storage/adapters/memoryStorage.ts +++ b/src/storage/adapters/memoryStorage.ts @@ -691,6 +691,10 @@ export class MemoryStorage extends BaseStorage { verbCount: {...this.statistics.verbCount}, metadataCount: {...this.statistics.metadataCount}, hnswIndexSize: this.statistics.hnswIndexSize, + // CRITICAL FIX: Populate totalNodes and totalEdges from in-memory counts + // HNSW rebuild depends on these fields to determine entity count + totalNodes: this.totalNounCount, + totalEdges: this.totalVerbCount, lastUpdated: this.statistics.lastUpdated, // Include serviceActivity if present ...(this.statistics.serviceActivity && { @@ -703,11 +707,11 @@ export class MemoryStorage extends BaseStorage { services: this.statistics.services.map(s => ({...s})) }), // Include distributedConfig if present - ...(this.statistics.distributedConfig && { - distributedConfig: JSON.parse(JSON.stringify(this.statistics.distributedConfig)) + ...(this.statistics.distributedConfig && { + distributedConfig: JSON.parse(JSON.stringify(this.statistics.distributedConfig)) }) } - + // Since this is in-memory, there's no need for fallback mechanisms // to check multiple storage locations } diff --git a/src/storage/adapters/opfsStorage.ts b/src/storage/adapters/opfsStorage.ts index 9d982413..72a449f6 100644 --- a/src/storage/adapters/opfsStorage.ts +++ b/src/storage/adapters/opfsStorage.ts @@ -1346,6 +1346,10 @@ export class OPFSStorage extends BaseStorage { verbCount: { ...this.statistics.verbCount }, metadataCount: { ...this.statistics.metadataCount }, hnswIndexSize: this.statistics.hnswIndexSize, + // CRITICAL FIX: Populate totalNodes and totalEdges from in-memory counts + // HNSW rebuild depends on these fields to determine entity count + totalNodes: this.totalNounCount, + totalEdges: this.totalVerbCount, lastUpdated: this.statistics.lastUpdated } } @@ -1374,6 +1378,10 @@ export class OPFSStorage extends BaseStorage { verbCount: { ...this.statistics.verbCount }, metadataCount: { ...this.statistics.metadataCount }, hnswIndexSize: this.statistics.hnswIndexSize, + // CRITICAL FIX: Populate totalNodes and totalEdges from in-memory counts + // HNSW rebuild depends on these fields to determine entity count + totalNodes: this.totalNounCount, + totalEdges: this.totalVerbCount, lastUpdated: this.statistics.lastUpdated } } @@ -1397,6 +1405,10 @@ export class OPFSStorage extends BaseStorage { verbCount: { ...this.statistics.verbCount }, metadataCount: { ...this.statistics.metadataCount }, hnswIndexSize: this.statistics.hnswIndexSize, + // CRITICAL FIX: Populate totalNodes and totalEdges from in-memory counts + // HNSW rebuild depends on these fields to determine entity count + totalNodes: this.totalNounCount, + totalEdges: this.totalVerbCount, lastUpdated: this.statistics.lastUpdated } } @@ -1418,6 +1430,10 @@ export class OPFSStorage extends BaseStorage { verbCount: { ...this.statistics.verbCount }, metadataCount: { ...this.statistics.metadataCount }, hnswIndexSize: this.statistics.hnswIndexSize, + // CRITICAL FIX: Populate totalNodes and totalEdges from in-memory counts + // HNSW rebuild depends on these fields to determine entity count + totalNodes: this.totalNounCount, + totalEdges: this.totalVerbCount, lastUpdated: this.statistics.lastUpdated } } diff --git a/src/storage/adapters/s3CompatibleStorage.ts b/src/storage/adapters/s3CompatibleStorage.ts index eaaf629f..b33c0ffe 100644 --- a/src/storage/adapters/s3CompatibleStorage.ts +++ b/src/storage/adapters/s3CompatibleStorage.ts @@ -2742,13 +2742,17 @@ export class S3CompatibleStorage extends BaseStorage { verbCount: { ...this.statisticsCache.verbCount }, metadataCount: { ...this.statisticsCache.metadataCount }, hnswIndexSize: this.statisticsCache.hnswIndexSize, + // CRITICAL FIX: Populate totalNodes and totalEdges from in-memory counts + // HNSW rebuild depends on these fields to determine entity count + totalNodes: this.totalNounCount, + totalEdges: this.totalVerbCount, lastUpdated: this.statisticsCache.lastUpdated } } try { // Fetching fresh statistics from storage - + // Import the GetObjectCommand only when needed const { GetObjectCommand } = await import('@aws-sdk/client-s3') @@ -2760,15 +2764,15 @@ export class S3CompatibleStorage extends BaseStorage { ...(this.shouldTryYesterday() ? [this.getStatisticsKeyForDate(this.getYesterday())] : []) // Legacy fallback removed - /index folder is auto-cleaned on initialization ] - + let statistics: StatisticsData | null = null - + // Try each key with a timeout to prevent hanging for (const key of keys) { try { statistics = await Promise.race([ this.tryGetStatisticsFromKey(key), - new Promise((_, reject) => + new Promise((_, reject) => setTimeout(() => reject(new Error('Timeout')), 2000) // 2 second timeout per key ) ]) @@ -2789,15 +2793,31 @@ export class S3CompatibleStorage extends BaseStorage { hnswIndexSize: statistics.hnswIndexSize, lastUpdated: statistics.lastUpdated } + + // CRITICAL FIX: Add totalNodes and totalEdges from in-memory counts + // HNSW rebuild depends on these fields to determine entity count + return { + ...statistics, + totalNodes: this.totalNounCount, + totalEdges: this.totalVerbCount + } } // Successfully loaded statistics from storage - + return statistics } catch (error: any) { this.logger.warn('Error getting statistics data, returning cached or null:', error) // Return cached data if available, even if stale, rather than throwing - return this.statisticsCache || null + // CRITICAL FIX: Add totalNodes and totalEdges when returning cached data + if (this.statisticsCache) { + return { + ...this.statisticsCache, + totalNodes: this.totalNounCount, + totalEdges: this.totalVerbCount + } + } + return null } }