diff --git a/src/storage/adapters/gcsStorage.ts b/src/storage/adapters/gcsStorage.ts index 51b6a4ae..8b0942d9 100644 --- a/src/storage/adapters/gcsStorage.ts +++ b/src/storage/adapters/gcsStorage.ts @@ -1532,7 +1532,15 @@ export class GcsStorage extends BaseStorage { const statistics = JSON.parse(contents.toString()) this.logger.trace('Statistics retrieved successfully') - return statistics + + // CRITICAL FIX: Populate totalNodes and totalEdges from in-memory counts + // HNSW rebuild depends on these fields to determine entity count + return { + ...statistics, + totalNodes: this.totalNounCount, + totalEdges: this.totalVerbCount, + lastUpdated: new Date().toISOString() + } } catch (error: any) { if (error.code === 404) { this.logger.trace('Statistics not found (creating new)') diff --git a/src/storage/adapters/memoryStorage.ts b/src/storage/adapters/memoryStorage.ts index 7d2edadf..8625c831 100644 --- a/src/storage/adapters/memoryStorage.ts +++ b/src/storage/adapters/memoryStorage.ts @@ -691,6 +691,10 @@ export class MemoryStorage extends BaseStorage { verbCount: {...this.statistics.verbCount}, metadataCount: {...this.statistics.metadataCount}, hnswIndexSize: this.statistics.hnswIndexSize, + // CRITICAL FIX: Populate totalNodes and totalEdges from in-memory counts + // HNSW rebuild depends on these fields to determine entity count + totalNodes: this.totalNounCount, + totalEdges: this.totalVerbCount, lastUpdated: this.statistics.lastUpdated, // Include serviceActivity if present ...(this.statistics.serviceActivity && { @@ -703,11 +707,11 @@ export class MemoryStorage extends BaseStorage { services: this.statistics.services.map(s => ({...s})) }), // Include distributedConfig if present - ...(this.statistics.distributedConfig && { - distributedConfig: JSON.parse(JSON.stringify(this.statistics.distributedConfig)) + ...(this.statistics.distributedConfig && { + distributedConfig: JSON.parse(JSON.stringify(this.statistics.distributedConfig)) }) } - + // Since this is in-memory, there's no need for fallback mechanisms // to check multiple storage locations } diff --git a/src/storage/adapters/opfsStorage.ts b/src/storage/adapters/opfsStorage.ts index 9d982413..72a449f6 100644 --- a/src/storage/adapters/opfsStorage.ts +++ b/src/storage/adapters/opfsStorage.ts @@ -1346,6 +1346,10 @@ export class OPFSStorage extends BaseStorage { verbCount: { ...this.statistics.verbCount }, metadataCount: { ...this.statistics.metadataCount }, hnswIndexSize: this.statistics.hnswIndexSize, + // CRITICAL FIX: Populate totalNodes and totalEdges from in-memory counts + // HNSW rebuild depends on these fields to determine entity count + totalNodes: this.totalNounCount, + totalEdges: this.totalVerbCount, lastUpdated: this.statistics.lastUpdated } } @@ -1374,6 +1378,10 @@ export class OPFSStorage extends BaseStorage { verbCount: { ...this.statistics.verbCount }, metadataCount: { ...this.statistics.metadataCount }, hnswIndexSize: this.statistics.hnswIndexSize, + // CRITICAL FIX: Populate totalNodes and totalEdges from in-memory counts + // HNSW rebuild depends on these fields to determine entity count + totalNodes: this.totalNounCount, + totalEdges: this.totalVerbCount, lastUpdated: this.statistics.lastUpdated } } @@ -1397,6 +1405,10 @@ export class OPFSStorage extends BaseStorage { verbCount: { ...this.statistics.verbCount }, metadataCount: { ...this.statistics.metadataCount }, hnswIndexSize: this.statistics.hnswIndexSize, + // CRITICAL FIX: Populate totalNodes and totalEdges from in-memory counts + // HNSW rebuild depends on these fields to determine entity count + totalNodes: this.totalNounCount, + totalEdges: this.totalVerbCount, lastUpdated: this.statistics.lastUpdated } } @@ -1418,6 +1430,10 @@ export class OPFSStorage extends BaseStorage { verbCount: { ...this.statistics.verbCount }, metadataCount: { ...this.statistics.metadataCount }, hnswIndexSize: this.statistics.hnswIndexSize, + // CRITICAL FIX: Populate totalNodes and totalEdges from in-memory counts + // HNSW rebuild depends on these fields to determine entity count + totalNodes: this.totalNounCount, + totalEdges: this.totalVerbCount, lastUpdated: this.statistics.lastUpdated } } diff --git a/src/storage/adapters/s3CompatibleStorage.ts b/src/storage/adapters/s3CompatibleStorage.ts index eaaf629f..b33c0ffe 100644 --- a/src/storage/adapters/s3CompatibleStorage.ts +++ b/src/storage/adapters/s3CompatibleStorage.ts @@ -2742,13 +2742,17 @@ export class S3CompatibleStorage extends BaseStorage { verbCount: { ...this.statisticsCache.verbCount }, metadataCount: { ...this.statisticsCache.metadataCount }, hnswIndexSize: this.statisticsCache.hnswIndexSize, + // CRITICAL FIX: Populate totalNodes and totalEdges from in-memory counts + // HNSW rebuild depends on these fields to determine entity count + totalNodes: this.totalNounCount, + totalEdges: this.totalVerbCount, lastUpdated: this.statisticsCache.lastUpdated } } try { // Fetching fresh statistics from storage - + // Import the GetObjectCommand only when needed const { GetObjectCommand } = await import('@aws-sdk/client-s3') @@ -2760,15 +2764,15 @@ export class S3CompatibleStorage extends BaseStorage { ...(this.shouldTryYesterday() ? [this.getStatisticsKeyForDate(this.getYesterday())] : []) // Legacy fallback removed - /index folder is auto-cleaned on initialization ] - + let statistics: StatisticsData | null = null - + // Try each key with a timeout to prevent hanging for (const key of keys) { try { statistics = await Promise.race([ this.tryGetStatisticsFromKey(key), - new Promise((_, reject) => + new Promise((_, reject) => setTimeout(() => reject(new Error('Timeout')), 2000) // 2 second timeout per key ) ]) @@ -2789,15 +2793,31 @@ export class S3CompatibleStorage extends BaseStorage { hnswIndexSize: statistics.hnswIndexSize, lastUpdated: statistics.lastUpdated } + + // CRITICAL FIX: Add totalNodes and totalEdges from in-memory counts + // HNSW rebuild depends on these fields to determine entity count + return { + ...statistics, + totalNodes: this.totalNounCount, + totalEdges: this.totalVerbCount + } } // Successfully loaded statistics from storage - + return statistics } catch (error: any) { this.logger.warn('Error getting statistics data, returning cached or null:', error) // Return cached data if available, even if stale, rather than throwing - return this.statisticsCache || null + // CRITICAL FIX: Add totalNodes and totalEdges when returning cached data + if (this.statisticsCache) { + return { + ...this.statisticsCache, + totalNodes: this.totalNounCount, + totalEdges: this.totalVerbCount + } + } + return null } }