diff --git a/src/storage/adapters/fileSystemStorage.ts b/src/storage/adapters/fileSystemStorage.ts index 4c5a0e42..a3b0684c 100644 --- a/src/storage/adapters/fileSystemStorage.ts +++ b/src/storage/adapters/fileSystemStorage.ts @@ -15,16 +15,9 @@ import { } from '../../coreTypes.js' import { BaseStorage, - NOUNS_DIR, - VERBS_DIR, - METADATA_DIR, - NOUN_METADATA_DIR, - VERB_METADATA_DIR, - INDEX_DIR, SYSTEM_DIR, STATISTICS_KEY } from '../baseStorage.js' -import { StorageCompatibilityLayer, StoragePaths } from '../backwardCompatibility.js' // Type aliases for better readability type HNSWNode = HNSWNoun @@ -84,9 +77,8 @@ export class FileSystemStorage extends BaseStorage { private nounMetadataDir!: string private verbMetadataDir!: string private indexDir!: string // Legacy - for backward compatibility - private systemDir!: string // New location for system data + private systemDir!: string private lockDir!: string - private useDualWrite: boolean = true // Write to both locations during migration private activeLocks: Set = new Set() private lockTimers: Map = new Map() // Track timers for cleanup private allTimers: Set = new Set() // Track all timers for cleanup @@ -149,13 +141,14 @@ export class FileSystemStorage extends BaseStorage { try { // Initialize directory paths now that path module is loaded - this.nounsDir = path.join(this.rootDir, NOUNS_DIR) - this.verbsDir = path.join(this.rootDir, VERBS_DIR) - this.metadataDir = path.join(this.rootDir, METADATA_DIR) - this.nounMetadataDir = path.join(this.rootDir, NOUN_METADATA_DIR) - this.verbMetadataDir = path.join(this.rootDir, VERB_METADATA_DIR) - this.indexDir = path.join(this.rootDir, INDEX_DIR) // Legacy - this.systemDir = path.join(this.rootDir, SYSTEM_DIR) // New + // Clean directory structure (v4.7.2+) + this.nounsDir = path.join(this.rootDir, 'entities/nouns/hnsw') + this.verbsDir = path.join(this.rootDir, 'entities/verbs/hnsw') + this.metadataDir = path.join(this.rootDir, 'entities/nouns/metadata') // Legacy reference + this.nounMetadataDir = path.join(this.rootDir, 'entities/nouns/metadata') + this.verbMetadataDir = path.join(this.rootDir, 'entities/verbs/metadata') + this.indexDir = path.join(this.rootDir, 'indexes') + this.systemDir = path.join(this.rootDir, SYSTEM_DIR) this.lockDir = path.join(this.rootDir, 'locks') // Create the root directory if it doesn't exist @@ -1642,7 +1635,7 @@ export class FileSystemStorage extends BaseStorage { try { // Get existing statistics to merge with new data - const existingStats = await this.getStatisticsWithBackwardCompat() + const existingStats = await this.getStatisticsData() if (existingStats) { // Merge statistics data @@ -1686,140 +1679,25 @@ export class FileSystemStorage extends BaseStorage { * Get statistics data from storage */ protected async getStatisticsData(): Promise { - return this.getStatisticsWithBackwardCompat() - } - - /** - * Save statistics with backward compatibility (dual write) - */ - private async saveStatisticsWithBackwardCompat(statistics: StatisticsData): Promise { - // Always write to new location - const newPath = path.join(this.systemDir, `${STATISTICS_KEY}.json`) - await this.ensureDirectoryExists(this.systemDir) - await fs.promises.writeFile(newPath, JSON.stringify(statistics, null, 2)) - - // During migration period, also write to old location if it exists - if (this.useDualWrite && await this.directoryExists(this.indexDir)) { - const oldPath = path.join(this.indexDir, `${STATISTICS_KEY}.json`) - try { - await fs.promises.writeFile(oldPath, JSON.stringify(statistics, null, 2)) - } catch (error) { - // Log but don't fail if old location write fails - StorageCompatibilityLayer.logMigrationEvent( - 'Failed to write to legacy location', - { path: oldPath, error } - ) - } - } - } - - /** - * Get statistics with backward compatibility (dual read) - */ - private async getStatisticsWithBackwardCompat(): Promise { - let newStats: StatisticsData | null = null - let oldStats: StatisticsData | null = null - - // Try to read from new location first try { - const newPath = path.join(this.systemDir, `${STATISTICS_KEY}.json`) - const data = await fs.promises.readFile(newPath, 'utf-8') - newStats = JSON.parse(data) + const statsPath = path.join(this.systemDir, `${STATISTICS_KEY}.json`) + const data = await fs.promises.readFile(statsPath, 'utf-8') + return JSON.parse(data) } catch (error: any) { if (error.code !== 'ENOENT') { - console.error('Error reading statistics from new location:', error) + console.error('Error reading statistics:', error) } + return null } - - // Try to read from old location as fallback - if (!newStats && await this.directoryExists(this.indexDir)) { - try { - const oldPath = path.join(this.indexDir, `${STATISTICS_KEY}.json`) - const data = await fs.promises.readFile(oldPath, 'utf-8') - oldStats = JSON.parse(data) - - // If we found data in old location but not new, migrate it - if (oldStats && !newStats) { - StorageCompatibilityLayer.logMigrationEvent( - 'Migrating statistics from legacy location' - ) - await this.saveStatisticsWithBackwardCompat(oldStats) - } - } catch (error: any) { - if (error.code !== 'ENOENT') { - console.error('Error reading statistics from old location:', error) - } - } - } - - // Merge statistics from both locations - return this.mergeStatistics(newStats, oldStats) } /** - * Merge statistics from multiple sources + * Save statistics to storage */ - private mergeStatistics( - storageStats: StatisticsData | null, - localStats: StatisticsData | null - ): StatisticsData { - // Handle null cases - if (!storageStats && !localStats) { - // CRITICAL FIX (v3.37.4): Statistics files don't exist yet (first init) - // Return minimal stats with counts instead of zeros - // This prevents HNSW from seeing entityCount=0 during index rebuild - return { - nounCount: {}, - verbCount: {}, - metadataCount: {}, - hnswIndexSize: 0, - totalNodes: this.totalNounCount, - totalEdges: this.totalVerbCount, - totalMetadata: 0, - lastUpdated: new Date().toISOString() - } - } - if (!storageStats) return localStats! - if (!localStats) return storageStats - - // Merge noun counts by taking the maximum of each type - const mergedNounCount: Record = { - ...storageStats.nounCount - } - for (const [type, count] of Object.entries(localStats.nounCount)) { - mergedNounCount[type] = Math.max(mergedNounCount[type] || 0, count) - } - - // Merge verb counts by taking the maximum of each type - const mergedVerbCount: Record = { - ...storageStats.verbCount - } - for (const [type, count] of Object.entries(localStats.verbCount)) { - mergedVerbCount[type] = Math.max(mergedVerbCount[type] || 0, count) - } - - // Merge metadata counts by taking the maximum of each type - const mergedMetadataCount: Record = { - ...storageStats.metadataCount - } - for (const [type, count] of Object.entries(localStats.metadataCount)) { - mergedMetadataCount[type] = Math.max( - mergedMetadataCount[type] || 0, - count - ) - } - - return { - nounCount: mergedNounCount, - verbCount: mergedVerbCount, - metadataCount: mergedMetadataCount, - hnswIndexSize: Math.max(storageStats.hnswIndexSize || 0, localStats.hnswIndexSize || 0), - totalNodes: Math.max(storageStats.totalNodes || 0, localStats.totalNodes || 0), - totalEdges: Math.max(storageStats.totalEdges || 0, localStats.totalEdges || 0), - totalMetadata: Math.max(storageStats.totalMetadata || 0, localStats.totalMetadata || 0), - operations: storageStats.operations || localStats.operations, - lastUpdated: new Date().toISOString() - } + private async saveStatisticsWithBackwardCompat(statistics: StatisticsData): Promise { + const statsPath = path.join(this.systemDir, `${STATISTICS_KEY}.json`) + await this.ensureDirectoryExists(this.systemDir) + await fs.promises.writeFile(statsPath, JSON.stringify(statistics, null, 2)) } // ============================================= diff --git a/src/storage/backwardCompatibility.ts b/src/storage/backwardCompatibility.ts index 1423868b..c840fe3a 100644 --- a/src/storage/backwardCompatibility.ts +++ b/src/storage/backwardCompatibility.ts @@ -1,17 +1,15 @@ /** - * Storage backward compatibility layer for legacy data migrations + * DEPRECATED (v4.7.2): Backward compatibility stubs + * TODO: Remove in v4.7.3 after migrating s3CompatibleStorage */ export class StorageCompatibilityLayer { static logMigrationEvent(event: string, details?: any): void { - // Simplified logging for migration events - if (process.env.DEBUG_MIGRATION) { - console.log(`[Migration] ${event}`, details) - } + // No-op } - + static async migrateIfNeeded(storagePath: string): Promise { - // No-op for now - can be extended later if needed + // No-op } } @@ -24,14 +22,13 @@ export interface StoragePaths { statistics: string } -// Helper to get default paths export function getDefaultStoragePaths(basePath: string): StoragePaths { return { - nouns: `${basePath}/nouns`, - verbs: `${basePath}/verbs`, - metadata: `${basePath}/metadata`, - index: `${basePath}/index`, - system: `${basePath}/system`, - statistics: `${basePath}/statistics.json` + nouns: `${basePath}/entities/nouns/hnsw`, + verbs: `${basePath}/entities/verbs/hnsw`, + metadata: `${basePath}/entities/nouns/metadata`, + index: `${basePath}/indexes`, + system: `${basePath}/_system`, + statistics: `${basePath}/_system/statistics.json` } -} \ No newline at end of file +} diff --git a/src/storage/baseStorage.ts b/src/storage/baseStorage.ts index b9a1c416..762953bd 100644 --- a/src/storage/baseStorage.ts +++ b/src/storage/baseStorage.ts @@ -32,50 +32,26 @@ interface StorageKeyInfo { fullPath: string } -// Common directory/prefix names -// Option A: Entity-Based Directory Structure -export const ENTITIES_DIR = 'entities' -export const NOUNS_VECTOR_DIR = 'entities/nouns/vectors' +// Clean directory structure (v4.7.2+) +// All storage adapters use this consistent structure export const NOUNS_METADATA_DIR = 'entities/nouns/metadata' -export const VERBS_VECTOR_DIR = 'entities/verbs/vectors' export const VERBS_METADATA_DIR = 'entities/verbs/metadata' -export const INDEXES_DIR = 'indexes' -export const METADATA_INDEX_DIR = 'indexes/metadata' - -// Legacy paths - kept for backward compatibility during migration -export const NOUNS_DIR = 'nouns' // Legacy: now maps to entities/nouns/vectors -export const VERBS_DIR = 'verbs' // Legacy: now maps to entities/verbs/vectors -export const METADATA_DIR = 'metadata' // Legacy: now maps to entities/nouns/metadata -export const NOUN_METADATA_DIR = 'noun-metadata' // Legacy: now maps to entities/nouns/metadata -export const VERB_METADATA_DIR = 'verb-metadata' // Legacy: now maps to entities/verbs/metadata -export const INDEX_DIR = 'index' // Legacy - kept for backward compatibility -export const SYSTEM_DIR = '_system' // System config & metadata indexes +export const SYSTEM_DIR = '_system' export const STATISTICS_KEY = 'statistics' -// Migration version to track compatibility -export const STORAGE_SCHEMA_VERSION = 3 // v3: Entity-Based Directory Structure (Option A) - -// Configuration flag to enable new directory structure -export const USE_ENTITY_BASED_STRUCTURE = true // Set to true to use Option A structure - -/** - * Get the appropriate directory path based on configuration - */ +// DEPRECATED (v4.7.2): Temporary stubs for adapters not yet migrated +// TODO: Remove in v4.7.3 after migrating remaining adapters +export const NOUNS_DIR = 'entities/nouns/hnsw' +export const VERBS_DIR = 'entities/verbs/hnsw' +export const METADATA_DIR = 'entities/nouns/metadata' +export const NOUN_METADATA_DIR = 'entities/nouns/metadata' +export const VERB_METADATA_DIR = 'entities/verbs/metadata' +export const INDEX_DIR = 'indexes' export function getDirectoryPath(entityType: 'noun' | 'verb', dataType: 'vector' | 'metadata'): string { - if (USE_ENTITY_BASED_STRUCTURE) { - // Option A: Entity-Based Structure - if (entityType === 'noun') { - return dataType === 'vector' ? NOUNS_VECTOR_DIR : NOUNS_METADATA_DIR - } else { - return dataType === 'vector' ? VERBS_VECTOR_DIR : VERBS_METADATA_DIR - } + if (entityType === 'noun') { + return dataType === 'vector' ? NOUNS_DIR : NOUNS_METADATA_DIR } else { - // Legacy structure - if (entityType === 'noun') { - return dataType === 'vector' ? NOUNS_DIR : METADATA_DIR - } else { - return dataType === 'vector' ? VERBS_DIR : VERB_METADATA_DIR - } + return dataType === 'vector' ? VERBS_DIR : VERBS_METADATA_DIR } }