feat: migrate system metadata from 'index' to '_system' directory with backward compatibility

BREAKING CHANGE: System metadata location changed from 'index/' to '_system/' directory

- Rename INDEX_DIR to SYSTEM_DIR following database conventions
- Implement dual-read/write strategy for zero-downtime migration
- Add automatic migration from old to new location on first access
- Support mixed service versions sharing S3/cloud storage
- Add 30-day grace period for gradual rollout (configurable)
- Store distributed config alongside statistics in _system folder
- Add comprehensive migration guide and documentation

Migration features:
- Read from both locations (new first, fallback to old)
- Write to both during migration period
- Automatic data migration when found only in old location
- Services can update independently without coordination
- Full backward compatibility for production deployments

The change improves clarity ('_system' better represents system metadata than 'index')
and follows standard database conventions (MongoDB's _system, PostgreSQL's pg_*).

src/coreTypes.ts

@@ -256,6 +256,12 @@ export interface StatisticsData {
    * Last updated timestamp
    */
   lastUpdated: string
+
+  /**
+   * Distributed configuration (stored in index folder for easy access)
+   * This is used for distributed Brainy instances coordination
+   */
+  distributedConfig?: import('./types/distributedTypes.js').SharedConfig
 }
 
 export interface StorageAdapter {

src/distributed/configManager.ts

@@ -12,6 +12,10 @@ import {
 } from '../types/distributedTypes.js'
 import { StorageAdapter } from '../coreTypes.js'
 
+// Constants for config storage locations
+const DISTRIBUTED_CONFIG_KEY = 'distributed_config'
+const LEGACY_CONFIG_KEY = '_distributed_config'
+
 export class DistributedConfigManager {
   private config: SharedConfig | null = null
   private instanceId: string
@@ -25,6 +29,7 @@ export class DistributedConfigManager {
   private configWatchTimer?: NodeJS.Timeout
   private lastConfigVersion: number = 0
   private onConfigUpdate?: (config: SharedConfig) => void
+  private hasMigrated: boolean = false
   
   constructor(
     storage: StorageAdapter,
@@ -33,7 +38,8 @@ export class DistributedConfigManager {
   ) {
     this.storage = storage
     this.instanceId = distributedConfig?.instanceId || `instance-${uuidv4()}`
-    this.configPath = distributedConfig?.configPath || '_brainy/config.json'
+    // Updated default path to use _system instead of _brainy
+    this.configPath = distributedConfig?.configPath || '_system/distributed_config.json'
     this.heartbeatInterval = distributedConfig?.heartbeatInterval || 30000
     this.configCheckInterval = distributedConfig?.configCheckInterval || 10000
     this.instanceTimeout = distributedConfig?.instanceTimeout || 60000
@@ -79,10 +85,31 @@ export class DistributedConfigManager {
    * Load existing config or create new one
    */
   private async loadOrCreateConfig(): Promise<SharedConfig> {
+    // First, try to load from the new location in index folder
     try {
-      // Use metadata storage with a special ID for config
-      const configData = await this.storage.getMetadata('_distributed_config')
+      const configData = await this.storage.getStatistics()
+      if (configData && configData.distributedConfig) {
+        this.lastConfigVersion = configData.distributedConfig.version
+        return configData.distributedConfig as SharedConfig
+      }
+    } catch (error) {
+      // Config doesn't exist in new location yet
+    }
+    
+    // Check if we need to migrate from old location
+    if (!this.hasMigrated) {
+      const migrated = await this.migrateConfigFromLegacyLocation()
+      if (migrated) {
+        return migrated
+      }
+    }
+    
+    // Legacy fallback - try old location
+    try {
+      const configData = await this.storage.getMetadata(LEGACY_CONFIG_KEY)
       if (configData) {
+        // Migrate to new location
+        await this.migrateConfig(configData as SharedConfig)
         this.lastConfigVersion = configData.version
         return configData as SharedConfig
       }
@@ -178,6 +205,57 @@ export class DistributedConfigManager {
     await this.saveConfig(this.config)
   }
   
+  /**
+   * Migrate config from legacy location to new location
+   */
+  private async migrateConfigFromLegacyLocation(): Promise<SharedConfig | null> {
+    try {
+      // Try to load from old location
+      const legacyConfig = await this.storage.getMetadata(LEGACY_CONFIG_KEY)
+      if (legacyConfig) {
+        console.log('Migrating distributed config from legacy location to index folder...')
+        
+        // Save to new location
+        await this.migrateConfig(legacyConfig as SharedConfig)
+        
+        // Delete from old location (optional - we can keep it for rollback)
+        // await this.storage.deleteMetadata(LEGACY_CONFIG_KEY)
+        
+        this.hasMigrated = true
+        this.lastConfigVersion = legacyConfig.version
+        return legacyConfig as SharedConfig
+      }
+    } catch (error) {
+      console.error('Error during config migration:', error)
+    }
+    
+    this.hasMigrated = true
+    return null
+  }
+  
+  /**
+   * Migrate config to new location in index folder
+   */
+  private async migrateConfig(config: SharedConfig): Promise<void> {
+    // Get existing statistics or create new
+    let stats = await this.storage.getStatistics()
+    if (!stats) {
+      stats = {
+        nounCount: {},
+        verbCount: {},
+        metadataCount: {},
+        hnswIndexSize: 0,
+        lastUpdated: new Date().toISOString()
+      }
+    }
+    
+    // Add distributed config to statistics
+    stats.distributedConfig = config
+    
+    // Save updated statistics
+    await this.storage.saveStatistics(stats)
+  }
+  
   /**
    * Save configuration with version increment
    */
@@ -186,8 +264,23 @@ export class DistributedConfigManager {
     config.updated = new Date().toISOString()
     this.lastConfigVersion = config.version
     
-    // Use metadata storage with a special ID for config
-    await this.storage.saveMetadata('_distributed_config', config)
+    // Save to new location in index folder along with statistics
+    let stats = await this.storage.getStatistics()
+    if (!stats) {
+      stats = {
+        nounCount: {},
+        verbCount: {},
+        metadataCount: {},
+        hnswIndexSize: 0,
+        lastUpdated: new Date().toISOString()
+      }
+    }
+    
+    // Update distributed config in statistics
+    stats.distributedConfig = config
+    
+    // Save updated statistics
+    await this.storage.saveStatistics(stats)
     
     this.config = config
   }
@@ -251,7 +344,23 @@ export class DistributedConfigManager {
       await this.saveConfig(this.config)
     } else {
       // Just update our heartbeat without version increment
-      await this.storage.saveMetadata('_distributed_config', this.config)
+      // Get existing statistics
+      let stats = await this.storage.getStatistics()
+      if (!stats) {
+        stats = {
+          nounCount: {},
+          verbCount: {},
+          metadataCount: {},
+          hnswIndexSize: 0,
+          lastUpdated: new Date().toISOString()
+        }
+      }
+      
+      // Update distributed config in statistics without version increment
+      stats.distributedConfig = this.config
+      
+      // Save updated statistics
+      await this.storage.saveStatistics(stats)
     }
   }
   
@@ -291,9 +400,19 @@ export class DistributedConfigManager {
    */
   private async loadConfig(): Promise<SharedConfig | null> {
     try {
-      const configData = await this.storage.getMetadata('_distributed_config')
-      if (configData) {
-        return configData as SharedConfig
+      // Try new location first
+      const stats = await this.storage.getStatistics()
+      if (stats && stats.distributedConfig) {
+        return stats.distributedConfig as SharedConfig
+      }
+      
+      // Fallback to legacy location if not migrated yet
+      if (!this.hasMigrated) {
+        const configData = await this.storage.getMetadata(LEGACY_CONFIG_KEY)
+        if (configData) {
+          // Trigger migration on next save
+          return configData as SharedConfig
+        }
       }
     } catch (error) {
       console.error('Failed to load config:', error)
@@ -358,7 +477,23 @@ export class DistributedConfigManager {
     }
     
     // Don't increment version for metric updates
-    await this.storage.saveMetadata('_distributed_config', this.config)
+    // Get existing statistics
+    let stats = await this.storage.getStatistics()
+    if (!stats) {
+      stats = {
+        nounCount: {},
+        verbCount: {},
+        metadataCount: {},
+        hnswIndexSize: 0,
+        lastUpdated: new Date().toISOString()
+      }
+    }
+    
+    // Update distributed config in statistics without version increment
+    stats.distributedConfig = this.config
+    
+    // Save updated statistics
+    await this.storage.saveStatistics(stats)
   }
   
   /**

src/storage/adapters/fileSystemStorage.ts

@@ -12,8 +12,10 @@ import {
   NOUN_METADATA_DIR,
   VERB_METADATA_DIR,
   INDEX_DIR,
+  SYSTEM_DIR,
   STATISTICS_KEY
 } from '../baseStorage.js'
+import { StorageCompatibilityLayer, StoragePaths } from '../backwardCompatibility.js'
 
 // Type aliases for better readability
 type HNSWNode = HNSWNoun
@@ -57,8 +59,10 @@ export class FileSystemStorage extends BaseStorage {
   private metadataDir!: string
   private nounMetadataDir!: string
   private verbMetadataDir!: string
-  private indexDir!: string
+  private indexDir!: string  // Legacy - for backward compatibility
+  private systemDir!: string  // New location for system data
   private lockDir!: string
+  private useDualWrite: boolean = true  // Write to both locations during migration
   private activeLocks: Set<string> = new Set()
 
   /**
@@ -104,7 +108,8 @@ export class FileSystemStorage extends BaseStorage {
       this.metadataDir = path.join(this.rootDir, METADATA_DIR)
       this.nounMetadataDir = path.join(this.rootDir, NOUN_METADATA_DIR)
       this.verbMetadataDir = path.join(this.rootDir, VERB_METADATA_DIR)
-      this.indexDir = path.join(this.rootDir, INDEX_DIR)
+      this.indexDir = path.join(this.rootDir, INDEX_DIR)  // Legacy
+      this.systemDir = path.join(this.rootDir, SYSTEM_DIR)  // New
       this.lockDir = path.join(this.rootDir, 'locks')
 
       // Create the root directory if it doesn't exist
@@ -125,8 +130,12 @@ export class FileSystemStorage extends BaseStorage {
       // Create the verb metadata directory if it doesn't exist
       await this.ensureDirectoryExists(this.verbMetadataDir)
 
-      // Create the index directory if it doesn't exist
-      await this.ensureDirectoryExists(this.indexDir)
+      // Create both directories for backward compatibility
+      await this.ensureDirectoryExists(this.systemDir)
+      // Only create legacy directory if it exists (don't create new legacy dirs)
+      if (await this.directoryExists(this.indexDir)) {
+        await this.ensureDirectoryExists(this.indexDir)
+      }
 
       // Create the locks directory if it doesn't exist
       await this.ensureDirectoryExists(this.lockDir)
@@ -138,6 +147,18 @@ export class FileSystemStorage extends BaseStorage {
     }
   }
 
+  /**
+   * Check if a directory exists
+   */
+  private async directoryExists(dirPath: string): Promise<boolean> {
+    try {
+      const stats = await fs.promises.stat(dirPath)
+      return stats.isDirectory()
+    } catch (error) {
+      return false
+    }
+  }
+
   /**
    * Ensure a directory exists, creating it if necessary
    */
@@ -576,8 +597,11 @@ export class FileSystemStorage extends BaseStorage {
     // Remove all files in the verb metadata directory
     await removeDirectoryContents(this.verbMetadataDir)
 
-    // Remove all files in the index directory
-    await removeDirectoryContents(this.indexDir)
+    // Remove all files in both system directories
+    await removeDirectoryContents(this.systemDir)
+    if (await this.directoryExists(this.indexDir)) {
+      await removeDirectoryContents(this.indexDir)
+    }
     
     // Clear the statistics cache
     this.statisticsCache = null
@@ -976,9 +1000,7 @@ export class FileSystemStorage extends BaseStorage {
 
     try {
       // Get existing statistics to merge with new data
-      const existingStats = (await this.getMetadata(
-        STATISTICS_KEY
-      )) as StatisticsData | null
+      const existingStats = await this.getStatisticsWithBackwardCompat()
 
       if (existingStats) {
         // Merge statistics data
@@ -1002,14 +1024,14 @@ export class FileSystemStorage extends BaseStorage {
           // Always update lastUpdated to current time
           lastUpdated: new Date().toISOString()
         }
-        await this.saveMetadata(STATISTICS_KEY, mergedStats)
+        await this.saveStatisticsWithBackwardCompat(mergedStats)
       } else {
         // No existing statistics, save new ones
         const newStats: StatisticsData = {
           ...statistics,
           lastUpdated: new Date().toISOString()
         }
-        await this.saveMetadata(STATISTICS_KEY, newStats)
+        await this.saveStatisticsWithBackwardCompat(newStats)
       }
     } finally {
       if (lockAcquired) {
@@ -1022,6 +1044,73 @@ export class FileSystemStorage extends BaseStorage {
    * Get statistics data from storage
    */
   protected async getStatisticsData(): Promise<StatisticsData | null> {
-    return this.getMetadata(STATISTICS_KEY)
+    return this.getStatisticsWithBackwardCompat()
+  }
+
+  /**
+   * Save statistics with backward compatibility (dual write)
+   */
+  private async saveStatisticsWithBackwardCompat(statistics: StatisticsData): Promise<void> {
+    // Always write to new location
+    const newPath = path.join(this.systemDir, `${STATISTICS_KEY}.json`)
+    await this.ensureDirectoryExists(this.systemDir)
+    await fs.promises.writeFile(newPath, JSON.stringify(statistics, null, 2))
+    
+    // During migration period, also write to old location if it exists
+    if (this.useDualWrite && await this.directoryExists(this.indexDir)) {
+      const oldPath = path.join(this.indexDir, `${STATISTICS_KEY}.json`)
+      try {
+        await fs.promises.writeFile(oldPath, JSON.stringify(statistics, null, 2))
+      } catch (error) {
+        // Log but don't fail if old location write fails
+        StorageCompatibilityLayer.logMigrationEvent(
+          'Failed to write to legacy location',
+          { path: oldPath, error }
+        )
+      }
+    }
+  }
+
+  /**
+   * Get statistics with backward compatibility (dual read)
+   */
+  private async getStatisticsWithBackwardCompat(): Promise<StatisticsData | null> {
+    let newStats: StatisticsData | null = null
+    let oldStats: StatisticsData | null = null
+    
+    // Try to read from new location first
+    try {
+      const newPath = path.join(this.systemDir, `${STATISTICS_KEY}.json`)
+      const data = await fs.promises.readFile(newPath, 'utf-8')
+      newStats = JSON.parse(data)
+    } catch (error: any) {
+      if (error.code !== 'ENOENT') {
+        console.error('Error reading statistics from new location:', error)
+      }
+    }
+    
+    // Try to read from old location as fallback
+    if (!newStats && await this.directoryExists(this.indexDir)) {
+      try {
+        const oldPath = path.join(this.indexDir, `${STATISTICS_KEY}.json`)
+        const data = await fs.promises.readFile(oldPath, 'utf-8')
+        oldStats = JSON.parse(data)
+        
+        // If we found data in old location but not new, migrate it
+        if (oldStats && !newStats) {
+          StorageCompatibilityLayer.logMigrationEvent(
+            'Migrating statistics from legacy location'
+          )
+          await this.saveStatisticsWithBackwardCompat(oldStats)
+        }
+      } catch (error: any) {
+        if (error.code !== 'ENOENT') {
+          console.error('Error reading statistics from old location:', error)
+        }
+      }
+    }
+    
+    // Merge statistics from both locations
+    return StorageCompatibilityLayer.mergeStatistics(newStats, oldStats)
   }
 }

src/storage/adapters/memoryStorage.ts

@@ -621,7 +621,11 @@ export class MemoryStorage extends BaseStorage {
       verbCount: {...statistics.verbCount},
       metadataCount: {...statistics.metadataCount},
       hnswIndexSize: statistics.hnswIndexSize,
-      lastUpdated: statistics.lastUpdated
+      lastUpdated: statistics.lastUpdated,
+      // Include distributedConfig if present
+      ...(statistics.distributedConfig && { 
+        distributedConfig: JSON.parse(JSON.stringify(statistics.distributedConfig)) 
+      })
     }
     
     // Since this is in-memory, there's no need for time-based partitioning
@@ -643,7 +647,11 @@ export class MemoryStorage extends BaseStorage {
       verbCount: {...this.statistics.verbCount},
       metadataCount: {...this.statistics.metadataCount},
       hnswIndexSize: this.statistics.hnswIndexSize,
-      lastUpdated: this.statistics.lastUpdated
+      lastUpdated: this.statistics.lastUpdated,
+      // Include distributedConfig if present
+      ...(this.statistics.distributedConfig && { 
+        distributedConfig: JSON.parse(JSON.stringify(this.statistics.distributedConfig)) 
+      })
     }
     
     // Since this is in-memory, there's no need for fallback mechanisms

src/storage/adapters/s3CompatibleStorage.ts

@@ -11,8 +11,10 @@ import {
   VERBS_DIR,
   METADATA_DIR,
   INDEX_DIR,
+  SYSTEM_DIR,
   STATISTICS_KEY
 } from '../baseStorage.js'
+import { StorageCompatibilityLayer, StoragePaths } from '../backwardCompatibility.js'
 import {
   StorageOperationExecutors,
   OperationConfig
@@ -79,7 +81,9 @@ export class S3CompatibleStorage extends BaseStorage {
   private nounPrefix: string
   private verbPrefix: string
   private metadataPrefix: string
-  private indexPrefix: string
+  private indexPrefix: string  // Legacy - for backward compatibility
+  private systemPrefix: string  // New location for system data
+  private useDualWrite: boolean = true  // Write to both locations during migration
 
   // Statistics caching for better performance
   protected statisticsCache: StatisticsData | null = null
@@ -142,7 +146,8 @@ export class S3CompatibleStorage extends BaseStorage {
     this.nounPrefix = `${NOUNS_DIR}/`
     this.verbPrefix = `${VERBS_DIR}/`
     this.metadataPrefix = `${METADATA_DIR}/`
-    this.indexPrefix = `${INDEX_DIR}/`
+    this.indexPrefix = `${INDEX_DIR}/`  // Legacy
+    this.systemPrefix = `${SYSTEM_DIR}/`  // New
     
     // Initialize cache managers
     this.nounCacheManager = new CacheManager<HNSWNode>(options.cacheConfig)
@@ -1921,18 +1926,29 @@ export class S3CompatibleStorage extends BaseStorage {
       // Update local cache with merged data
       this.statisticsCache = mergedStats
 
-      // Also update the legacy key for backward compatibility, but less frequently
-      // Only update it once every 10 flushes (approximately)
-      if (Math.random() < 0.1) {
-        const legacyKey = this.getLegacyStatisticsKey()
-        await this.s3Client!.send(
-          new PutObjectCommand({
-            Bucket: this.bucketName,
-            Key: legacyKey,
-            Body: body,
-            ContentType: 'application/json'
-          })
-        )
+      // During migration period, also update the legacy location
+      // for backward compatibility with older services
+      if (this.useDualWrite) {
+        try {
+          const legacyKey = this.getLegacyStatisticsKey()
+          await this.s3Client!.send(
+            new PutObjectCommand({
+              Bucket: this.bucketName,
+              Key: legacyKey,
+              Body: body,
+              ContentType: 'application/json',
+              Metadata: {
+                'migration-note': 'dual-write-for-compatibility',
+                'schema-version': '2'
+              }
+            })
+          )
+        } catch (error) {
+          StorageCompatibilityLayer.logMigrationEvent(
+            'Failed to write statistics to legacy S3 location',
+            { error }
+          )
+        }
       }
     } catch (error) {
       this.logger.error('Failed to flush statistics data:', error)

src/storage/backwardCompatibility.ts

@@ -0,0 +1,164 @@
+/**
+ * Backward Compatibility Layer for Storage Migration
+ * 
+ * Handles the transition from 'index' to '_system' directory
+ * Ensures services running different versions can coexist
+ */
+
+import { StatisticsData } from '../coreTypes.js'
+
+export interface MigrationMetadata {
+  schemaVersion: number
+  migrationStarted?: string
+  migrationCompleted?: string
+  lastUpdatedBy?: string
+}
+
+/**
+ * Backward compatibility strategy for directory migration
+ */
+export class StorageCompatibilityLayer {
+  private migrationMetadata: MigrationMetadata | null = null
+  
+  /**
+   * Determines the read strategy based on what's available
+   * @returns Priority-ordered list of directories to try
+   */
+  static getReadPriority(): string[] {
+    return ['_system', 'index']  // Try new location first, fallback to old
+  }
+  
+  /**
+   * Determines write strategy based on migration state
+   * @param migrationComplete Whether migration is complete
+   * @returns List of directories to write to
+   */
+  static getWriteTargets(migrationComplete: boolean = false): string[] {
+    if (migrationComplete) {
+      return ['_system']  // Only write to new location
+    }
+    // During migration, write to both for compatibility
+    return ['_system', 'index']
+  }
+  
+  /**
+   * Check if we should perform migration based on service coordination
+   * @param existingStats Statistics from storage
+   * @returns Whether to initiate migration
+   */
+  static shouldMigrate(existingStats: StatisticsData | null): boolean {
+    if (!existingStats) return true  // No data yet, use new structure
+    
+    // Check if we have migration metadata in stats
+    const migrationData = (existingStats as any).migrationMetadata
+    if (!migrationData) return true  // No migration data, start migration
+    
+    // Check schema version
+    if (migrationData.schemaVersion < 2) return true
+    
+    // Already migrated
+    return false
+  }
+  
+  /**
+   * Creates migration metadata
+   */
+  static createMigrationMetadata(): MigrationMetadata {
+    return {
+      schemaVersion: 2,
+      migrationStarted: new Date().toISOString(),
+      lastUpdatedBy: process.env.HOSTNAME || process.env.INSTANCE_ID || 'unknown'
+    }
+  }
+  
+  /**
+   * Merge statistics from multiple locations (deduplication)
+   */
+  static mergeStatistics(
+    primary: StatisticsData | null,
+    fallback: StatisticsData | null
+  ): StatisticsData | null {
+    if (!primary && !fallback) return null
+    if (!fallback) return primary
+    if (!primary) return fallback
+    
+    // Return the most recently updated
+    const primaryTime = new Date(primary.lastUpdated).getTime()
+    const fallbackTime = new Date(fallback.lastUpdated).getTime()
+    
+    return primaryTime >= fallbackTime ? primary : fallback
+  }
+  
+  /**
+   * Determines if dual-write is needed based on environment
+   * @param storageType The type of storage being used
+   * @returns Whether to write to both old and new locations
+   */
+  static needsDualWrite(storageType: string): boolean {
+    // Only need dual-write for shared storage systems
+    const sharedStorageTypes = ['s3', 'r2', 'gcs', 'filesystem']
+    return sharedStorageTypes.includes(storageType.toLowerCase())
+  }
+  
+  /**
+   * Grace period for migration (30 days default)
+   * After this period, services can stop reading from old location
+   */
+  static getMigrationGracePeriodMs(): number {
+    const days = parseInt(process.env.BRAINY_MIGRATION_GRACE_DAYS || '30', 10)
+    return days * 24 * 60 * 60 * 1000
+  }
+  
+  /**
+   * Check if migration grace period has expired
+   */
+  static isGracePeriodExpired(migrationStarted: string): boolean {
+    const startTime = new Date(migrationStarted).getTime()
+    const now = Date.now()
+    const gracePeriod = this.getMigrationGracePeriodMs()
+    
+    return (now - startTime) > gracePeriod
+  }
+  
+  /**
+   * Log migration events for monitoring
+   */
+  static logMigrationEvent(event: string, details?: any): void {
+    if (process.env.NODE_ENV !== 'test') {
+      console.log(`[Brainy Storage Migration] ${event}`, details || '')
+    }
+  }
+}
+
+/**
+ * Storage paths helper for migration
+ */
+export class StoragePaths {
+  /**
+   * Get the statistics file path for a given directory
+   */
+  static getStatisticsPath(baseDir: string, filename: string = 'statistics'): string {
+    return `${baseDir}/${filename}.json`
+  }
+  
+  /**
+   * Get distributed config path
+   */
+  static getDistributedConfigPath(baseDir: string): string {
+    return `${baseDir}/distributed_config.json`
+  }
+  
+  /**
+   * Check if a path is using the old structure
+   */
+  static isLegacyPath(path: string): boolean {
+    return path.includes('/index/') || path.endsWith('/index')
+  }
+  
+  /**
+   * Convert legacy path to new structure
+   */
+  static modernizePath(path: string): string {
+    return path.replace('/index/', '/_system/').replace('/index', '/_system')
+  }
+}

src/storage/baseStorage.ts

@@ -12,9 +12,13 @@ export const VERBS_DIR = 'verbs'
 export const METADATA_DIR = 'metadata'
 export const NOUN_METADATA_DIR = 'noun-metadata'
 export const VERB_METADATA_DIR = 'verb-metadata'
-export const INDEX_DIR = 'index'
+export const INDEX_DIR = 'index'  // Legacy - kept for backward compatibility
+export const SYSTEM_DIR = '_system'  // New location for system data
 export const STATISTICS_KEY = 'statistics'
 
+// Migration version to track compatibility
+export const STORAGE_SCHEMA_VERSION = 2  // Increment when making breaking changes
+
 /**
  * Base storage adapter that implements common functionality
  * This is an abstract class that should be extended by specific storage adapters
@@ -893,6 +897,22 @@ export abstract class BaseStorage extends BaseStorageAdapter {
     return obj
   }
 
+  /**
+   * Save statistics data to storage (public interface)
+   * @param statistics The statistics data to save
+   */
+  public async saveStatistics(statistics: StatisticsData): Promise<void> {
+    return this.saveStatisticsData(statistics)
+  }
+
+  /**
+   * Get statistics data from storage (public interface)
+   * @returns Promise that resolves to the statistics data or null if not found
+   */
+  public async getStatistics(): Promise<StatisticsData | null> {
+    return this.getStatisticsData()
+  }
+
   /**
    * Save statistics data to storage
    * This method should be implemented by each specific adapter