feat: migrate system metadata from 'index' to '_system' directory with backward compatibility

BREAKING CHANGE: System metadata location changed from 'index/' to '_system/' directory

- Rename INDEX_DIR to SYSTEM_DIR following database conventions
- Implement dual-read/write strategy for zero-downtime migration
- Add automatic migration from old to new location on first access
- Support mixed service versions sharing S3/cloud storage
- Add 30-day grace period for gradual rollout (configurable)
- Store distributed config alongside statistics in _system folder
- Add comprehensive migration guide and documentation

Migration features:
- Read from both locations (new first, fallback to old)
- Write to both during migration period
- Automatic data migration when found only in old location
- Services can update independently without coordination
- Full backward compatibility for production deployments

The change improves clarity ('_system' better represents system metadata than 'index')
and follows standard database conventions (MongoDB's _system, PostgreSQL's pg_*).

src/storage/adapters/fileSystemStorage.ts

@@ -12,8 +12,10 @@ import {
   NOUN_METADATA_DIR,
   VERB_METADATA_DIR,
   INDEX_DIR,
+  SYSTEM_DIR,
   STATISTICS_KEY
 } from '../baseStorage.js'
+import { StorageCompatibilityLayer, StoragePaths } from '../backwardCompatibility.js'
 
 // Type aliases for better readability
 type HNSWNode = HNSWNoun
@@ -57,8 +59,10 @@ export class FileSystemStorage extends BaseStorage {
   private metadataDir!: string
   private nounMetadataDir!: string
   private verbMetadataDir!: string
-  private indexDir!: string
+  private indexDir!: string  // Legacy - for backward compatibility
+  private systemDir!: string  // New location for system data
   private lockDir!: string
+  private useDualWrite: boolean = true  // Write to both locations during migration
   private activeLocks: Set<string> = new Set()
 
   /**
@@ -104,7 +108,8 @@ export class FileSystemStorage extends BaseStorage {
       this.metadataDir = path.join(this.rootDir, METADATA_DIR)
       this.nounMetadataDir = path.join(this.rootDir, NOUN_METADATA_DIR)
       this.verbMetadataDir = path.join(this.rootDir, VERB_METADATA_DIR)
-      this.indexDir = path.join(this.rootDir, INDEX_DIR)
+      this.indexDir = path.join(this.rootDir, INDEX_DIR)  // Legacy
+      this.systemDir = path.join(this.rootDir, SYSTEM_DIR)  // New
       this.lockDir = path.join(this.rootDir, 'locks')
 
       // Create the root directory if it doesn't exist
@@ -125,8 +130,12 @@ export class FileSystemStorage extends BaseStorage {
       // Create the verb metadata directory if it doesn't exist
       await this.ensureDirectoryExists(this.verbMetadataDir)
 
-      // Create the index directory if it doesn't exist
-      await this.ensureDirectoryExists(this.indexDir)
+      // Create both directories for backward compatibility
+      await this.ensureDirectoryExists(this.systemDir)
+      // Only create legacy directory if it exists (don't create new legacy dirs)
+      if (await this.directoryExists(this.indexDir)) {
+        await this.ensureDirectoryExists(this.indexDir)
+      }
 
       // Create the locks directory if it doesn't exist
       await this.ensureDirectoryExists(this.lockDir)
@@ -138,6 +147,18 @@ export class FileSystemStorage extends BaseStorage {
     }
   }
 
+  /**
+   * Check if a directory exists
+   */
+  private async directoryExists(dirPath: string): Promise<boolean> {
+    try {
+      const stats = await fs.promises.stat(dirPath)
+      return stats.isDirectory()
+    } catch (error) {
+      return false
+    }
+  }
+
   /**
    * Ensure a directory exists, creating it if necessary
    */
@@ -576,8 +597,11 @@ export class FileSystemStorage extends BaseStorage {
     // Remove all files in the verb metadata directory
     await removeDirectoryContents(this.verbMetadataDir)
 
-    // Remove all files in the index directory
-    await removeDirectoryContents(this.indexDir)
+    // Remove all files in both system directories
+    await removeDirectoryContents(this.systemDir)
+    if (await this.directoryExists(this.indexDir)) {
+      await removeDirectoryContents(this.indexDir)
+    }
     
     // Clear the statistics cache
     this.statisticsCache = null
@@ -976,9 +1000,7 @@ export class FileSystemStorage extends BaseStorage {
 
     try {
       // Get existing statistics to merge with new data
-      const existingStats = (await this.getMetadata(
-        STATISTICS_KEY
-      )) as StatisticsData | null
+      const existingStats = await this.getStatisticsWithBackwardCompat()
 
       if (existingStats) {
         // Merge statistics data
@@ -1002,14 +1024,14 @@ export class FileSystemStorage extends BaseStorage {
           // Always update lastUpdated to current time
           lastUpdated: new Date().toISOString()
         }
-        await this.saveMetadata(STATISTICS_KEY, mergedStats)
+        await this.saveStatisticsWithBackwardCompat(mergedStats)
       } else {
         // No existing statistics, save new ones
         const newStats: StatisticsData = {
           ...statistics,
           lastUpdated: new Date().toISOString()
         }
-        await this.saveMetadata(STATISTICS_KEY, newStats)
+        await this.saveStatisticsWithBackwardCompat(newStats)
       }
     } finally {
       if (lockAcquired) {
@@ -1022,6 +1044,73 @@ export class FileSystemStorage extends BaseStorage {
    * Get statistics data from storage
    */
   protected async getStatisticsData(): Promise<StatisticsData | null> {
-    return this.getMetadata(STATISTICS_KEY)
+    return this.getStatisticsWithBackwardCompat()
+  }
+
+  /**
+   * Save statistics with backward compatibility (dual write)
+   */
+  private async saveStatisticsWithBackwardCompat(statistics: StatisticsData): Promise<void> {
+    // Always write to new location
+    const newPath = path.join(this.systemDir, `${STATISTICS_KEY}.json`)
+    await this.ensureDirectoryExists(this.systemDir)
+    await fs.promises.writeFile(newPath, JSON.stringify(statistics, null, 2))
+    
+    // During migration period, also write to old location if it exists
+    if (this.useDualWrite && await this.directoryExists(this.indexDir)) {
+      const oldPath = path.join(this.indexDir, `${STATISTICS_KEY}.json`)
+      try {
+        await fs.promises.writeFile(oldPath, JSON.stringify(statistics, null, 2))
+      } catch (error) {
+        // Log but don't fail if old location write fails
+        StorageCompatibilityLayer.logMigrationEvent(
+          'Failed to write to legacy location',
+          { path: oldPath, error }
+        )
+      }
+    }
+  }
+
+  /**
+   * Get statistics with backward compatibility (dual read)
+   */
+  private async getStatisticsWithBackwardCompat(): Promise<StatisticsData | null> {
+    let newStats: StatisticsData | null = null
+    let oldStats: StatisticsData | null = null
+    
+    // Try to read from new location first
+    try {
+      const newPath = path.join(this.systemDir, `${STATISTICS_KEY}.json`)
+      const data = await fs.promises.readFile(newPath, 'utf-8')
+      newStats = JSON.parse(data)
+    } catch (error: any) {
+      if (error.code !== 'ENOENT') {
+        console.error('Error reading statistics from new location:', error)
+      }
+    }
+    
+    // Try to read from old location as fallback
+    if (!newStats && await this.directoryExists(this.indexDir)) {
+      try {
+        const oldPath = path.join(this.indexDir, `${STATISTICS_KEY}.json`)
+        const data = await fs.promises.readFile(oldPath, 'utf-8')
+        oldStats = JSON.parse(data)
+        
+        // If we found data in old location but not new, migrate it
+        if (oldStats && !newStats) {
+          StorageCompatibilityLayer.logMigrationEvent(
+            'Migrating statistics from legacy location'
+          )
+          await this.saveStatisticsWithBackwardCompat(oldStats)
+        }
+      } catch (error: any) {
+        if (error.code !== 'ENOENT') {
+          console.error('Error reading statistics from old location:', error)
+        }
+      }
+    }
+    
+    // Merge statistics from both locations
+    return StorageCompatibilityLayer.mergeStatistics(newStats, oldStats)
   }
 }

src/storage/adapters/memoryStorage.ts

@@ -621,7 +621,11 @@ export class MemoryStorage extends BaseStorage {
       verbCount: {...statistics.verbCount},
       metadataCount: {...statistics.metadataCount},
       hnswIndexSize: statistics.hnswIndexSize,
-      lastUpdated: statistics.lastUpdated
+      lastUpdated: statistics.lastUpdated,
+      // Include distributedConfig if present
+      ...(statistics.distributedConfig && { 
+        distributedConfig: JSON.parse(JSON.stringify(statistics.distributedConfig)) 
+      })
     }
     
     // Since this is in-memory, there's no need for time-based partitioning
@@ -643,7 +647,11 @@ export class MemoryStorage extends BaseStorage {
       verbCount: {...this.statistics.verbCount},
       metadataCount: {...this.statistics.metadataCount},
       hnswIndexSize: this.statistics.hnswIndexSize,
-      lastUpdated: this.statistics.lastUpdated
+      lastUpdated: this.statistics.lastUpdated,
+      // Include distributedConfig if present
+      ...(this.statistics.distributedConfig && { 
+        distributedConfig: JSON.parse(JSON.stringify(this.statistics.distributedConfig)) 
+      })
     }
     
     // Since this is in-memory, there's no need for fallback mechanisms

src/storage/adapters/s3CompatibleStorage.ts

@@ -11,8 +11,10 @@ import {
   VERBS_DIR,
   METADATA_DIR,
   INDEX_DIR,
+  SYSTEM_DIR,
   STATISTICS_KEY
 } from '../baseStorage.js'
+import { StorageCompatibilityLayer, StoragePaths } from '../backwardCompatibility.js'
 import {
   StorageOperationExecutors,
   OperationConfig
@@ -79,7 +81,9 @@ export class S3CompatibleStorage extends BaseStorage {
   private nounPrefix: string
   private verbPrefix: string
   private metadataPrefix: string
-  private indexPrefix: string
+  private indexPrefix: string  // Legacy - for backward compatibility
+  private systemPrefix: string  // New location for system data
+  private useDualWrite: boolean = true  // Write to both locations during migration
 
   // Statistics caching for better performance
   protected statisticsCache: StatisticsData | null = null
@@ -142,7 +146,8 @@ export class S3CompatibleStorage extends BaseStorage {
     this.nounPrefix = `${NOUNS_DIR}/`
     this.verbPrefix = `${VERBS_DIR}/`
     this.metadataPrefix = `${METADATA_DIR}/`
-    this.indexPrefix = `${INDEX_DIR}/`
+    this.indexPrefix = `${INDEX_DIR}/`  // Legacy
+    this.systemPrefix = `${SYSTEM_DIR}/`  // New
     
     // Initialize cache managers
     this.nounCacheManager = new CacheManager<HNSWNode>(options.cacheConfig)
@@ -1921,18 +1926,29 @@ export class S3CompatibleStorage extends BaseStorage {
       // Update local cache with merged data
       this.statisticsCache = mergedStats
 
-      // Also update the legacy key for backward compatibility, but less frequently
-      // Only update it once every 10 flushes (approximately)
-      if (Math.random() < 0.1) {
-        const legacyKey = this.getLegacyStatisticsKey()
-        await this.s3Client!.send(
-          new PutObjectCommand({
-            Bucket: this.bucketName,
-            Key: legacyKey,
-            Body: body,
-            ContentType: 'application/json'
-          })
-        )
+      // During migration period, also update the legacy location
+      // for backward compatibility with older services
+      if (this.useDualWrite) {
+        try {
+          const legacyKey = this.getLegacyStatisticsKey()
+          await this.s3Client!.send(
+            new PutObjectCommand({
+              Bucket: this.bucketName,
+              Key: legacyKey,
+              Body: body,
+              ContentType: 'application/json',
+              Metadata: {
+                'migration-note': 'dual-write-for-compatibility',
+                'schema-version': '2'
+              }
+            })
+          )
+        } catch (error) {
+          StorageCompatibilityLayer.logMigrationEvent(
+            'Failed to write statistics to legacy S3 location',
+            { error }
+          )
+        }
       }
     } catch (error) {
       this.logger.error('Failed to flush statistics data:', error)