/**
 * Multi-level Cache Manager
 * 
 * Implements a three-level caching strategy:
 * - Level 1: Hot cache (most accessed nodes) - RAM (automatically detecting and adjusting in each environment)
 * - Level 2: Warm cache (recent nodes) - OPFS, Filesystem or S3 depending on environment
 * - Level 3: Cold storage (all nodes) - OPFS, Filesystem or S3 depending on environment
 */

import { HNSWNoun, GraphVerb, HNSWVerb } from '../coreTypes.js'
import { BrainyError } from '../errors/brainyError.js'

// Type aliases for better readability
type HNSWNode = HNSWNoun
type Edge = GraphVerb

// Cache entry with metadata for LRU and TTL management
interface CacheEntry<T> {
  data: T
  lastAccessed: number
  accessCount: number
  expiresAt: number | null
}

// Cache statistics for monitoring and tuning
interface CacheStats {
  hits: number
  misses: number
  evictions: number
  size: number
  maxSize: number
  hotCacheSize: number
  warmCacheSize: number
  hotCacheHits: number
  hotCacheMisses: number
  warmCacheHits: number
  warmCacheMisses: number
}

// Storage type for warm and cold caches. Brainy 8.0 only ships filesystem +
// memory tiers, but the enum is retained for diagnostic/log output.
enum StorageType {
  MEMORY,
  FILESYSTEM
}

/**
 * Multi-level cache manager for efficient data access
 */
export class CacheManager<T extends HNSWNode | Edge | HNSWVerb> {
  // Hot cache (RAM)
  private hotCache = new Map<string, CacheEntry<T>>()
  
  // Cache statistics
  private stats: CacheStats = {
    hits: 0,
    misses: 0,
    evictions: 0,
    size: 0,
    maxSize: 0,
    hotCacheSize: 0,
    warmCacheSize: 0,
    hotCacheHits: 0,
    hotCacheMisses: 0,
    warmCacheHits: 0,
    warmCacheMisses: 0
  }
  
  // Storage configuration
  private warmStorageType: StorageType
  private coldStorageType: StorageType
  
  // Cache configuration
  private hotCacheMaxSize: number
  private hotCacheEvictionThreshold: number
  private warmCacheTTL: number
  private batchSize: number
  
  // Auto-tuning configuration
  private autoTune: boolean
  private lastAutoTuneTime: number = 0
  private autoTuneInterval: number = 5 * 60 * 1000 // 5 minutes
  private storageStatistics: any = null
  
  // Storage adapters for warm and cold caches
  private warmStorage: any
  private coldStorage: any
  
  // Store options for later reference
  private options: {
    hotCacheMaxSize?: number
    hotCacheEvictionThreshold?: number
    warmCacheTTL?: number
    batchSize?: number
    autoTune?: boolean
    warmStorage?: any
    coldStorage?: any
    readOnly?: boolean
    environmentConfig?: {
      node?: {
        hotCacheMaxSize?: number
        hotCacheEvictionThreshold?: number
        warmCacheTTL?: number
        batchSize?: number
      }
      [key: string]: {
        hotCacheMaxSize?: number
        hotCacheEvictionThreshold?: number
        warmCacheTTL?: number
        batchSize?: number
      } | undefined
    }
  }

  /**
   * Initialize the cache manager
   * @param options Configuration options
   */
  constructor(options: {
    hotCacheMaxSize?: number
    hotCacheEvictionThreshold?: number
    warmCacheTTL?: number
    batchSize?: number
    autoTune?: boolean
    warmStorage?: any
    coldStorage?: any
    readOnly?: boolean
    environmentConfig?: {
      node?: {
        hotCacheMaxSize?: number
        hotCacheEvictionThreshold?: number
        warmCacheTTL?: number
        batchSize?: number
      }
      [key: string]: {
        hotCacheMaxSize?: number
        hotCacheEvictionThreshold?: number
        warmCacheTTL?: number
        batchSize?: number
      } | undefined
    }
  } = {}) {
    // Store options for later reference
    this.options = options

    // Set storage types (Brainy 8.0 ships filesystem only)
    this.warmStorageType = this.detectWarmStorageType()
    this.coldStorageType = this.detectColdStorageType()

    // Initialize storage adapters
    this.warmStorage = options.warmStorage || this.initializeWarmStorage()
    this.coldStorage = options.coldStorage || this.initializeColdStorage()

    // Set auto-tuning flag
    this.autoTune = options.autoTune !== undefined ? options.autoTune : true

    // Brainy 8.0 only runs on Node-like runtimes, so only the `node` slot of
    // environmentConfig is honored.
    const envConfig = options.environmentConfig?.node

    // Set default values or use environment-specific values or global values
    this.hotCacheMaxSize = envConfig?.hotCacheMaxSize || options.hotCacheMaxSize || this.detectOptimalCacheSize()
    this.hotCacheEvictionThreshold = envConfig?.hotCacheEvictionThreshold || options.hotCacheEvictionThreshold || 0.8
    this.warmCacheTTL = envConfig?.warmCacheTTL || options.warmCacheTTL || 24 * 60 * 60 * 1000 // 24 hours
    this.batchSize = envConfig?.batchSize || options.batchSize || 10

    // If auto-tuning is enabled, perform initial tuning
    if (this.autoTune) {
      this.tuneParameters()
    }

    // Log configuration
    if (process.env.DEBUG) {
      console.log('Cache Manager initialized with configuration:', {
        environment: 'node',
        hotCacheMaxSize: this.hotCacheMaxSize,
        hotCacheEvictionThreshold: this.hotCacheEvictionThreshold,
        warmCacheTTL: this.warmCacheTTL,
        batchSize: this.batchSize,
        autoTune: this.autoTune,
        warmStorageType: StorageType[this.warmStorageType],
        coldStorageType: StorageType[this.coldStorageType]
      })
    }
  }


  /**
   * Detect the optimal cache size based on available memory and operating mode
   * 
   * Enhanced to better handle large datasets in S3 or other storage:
   * - Increases cache size for read-only mode
   * - Adjusts based on total dataset size when available
   * - Provides more aggressive caching for large datasets
   * - Optimizes memory usage based on environment
   */
  private detectOptimalCacheSize(): number {
    try {
      // Default to a conservative value
      const defaultSize = 1000

      // Get the total dataset size if available
      const totalItems = this.storageStatistics ?
        (this.storageStatistics.totalNodes || 0) + (this.storageStatistics.totalEdges || 0) : 0

      // Determine if we're dealing with a large dataset (>100K items)
      const isLargeDataset = totalItems > 100000

      // Check if we're in read-only mode (from parent Brainy instance)
      const isReadOnly = this.options?.readOnly || false

      try {
        // Synchronous path can't use dynamic imports, so we use conservative
        // assumed defaults. The async variant (detectAvailableMemory) reads
        // real values via `node:os` when possible.
        const estimatedFreeMemory = 4 * 1024 * 1024 * 1024   // Assume 4GB free

        // Estimate average entry size (in bytes)
        // This is a conservative estimate for complex objects with vectors
        const ESTIMATED_BYTES_PER_ENTRY = 1024 // 1KB per entry

        // Base memory percentage - 10% by default
        let memoryPercentage = 0.1

        // Adjust based on operating mode and dataset size
        if (isReadOnly) {
          // In read-only mode, we can use more memory for caching
          memoryPercentage = 0.25 // 25% of free memory

          // For large datasets in read-only mode, be even more aggressive
          if (isLargeDataset) {
            memoryPercentage = 0.4 // 40% of free memory
          }
        } else if (isLargeDataset) {
          // For large datasets in normal mode, increase slightly
          memoryPercentage = 0.15 // 15% of free memory
        }

        // Calculate optimal size based on adjusted percentage
        const optimalSize = Math.max(
          Math.floor(estimatedFreeMemory * memoryPercentage / ESTIMATED_BYTES_PER_ENTRY),
          1000
        )

        // If we know the total dataset size, cap at a reasonable percentage
        if (totalItems > 0) {
          // In read-only mode, we can cache a larger percentage
          const maxPercentage = isReadOnly ? 0.5 : 0.3
          const maxItems = Math.ceil(totalItems * maxPercentage)

          // Return the smaller of the two to avoid excessive memory usage
          return Math.min(optimalSize, maxItems)
        }

        return optimalSize
      } catch (error) {
        console.warn('Failed to detect optimal cache size:', error)
        return defaultSize
      }
    } catch (error) {
      console.warn('Error detecting optimal cache size:', error)
      return 1000 // Conservative default
    }
  }
  
  /**
   * Async version of detectOptimalCacheSize that uses dynamic imports
   * to access system information in Node.js environments
   * 
   * This method provides more accurate memory detection by using
   * the OS module's dynamic import in Node.js environments
   */
  private async detectOptimalCacheSizeAsync(): Promise<number> {
    try {
      // Default to a conservative value
      const defaultSize = 1000
      
      // Get the total dataset size if available
      const totalItems = this.storageStatistics ? 
        (this.storageStatistics.totalNodes || 0) + (this.storageStatistics.totalEdges || 0) : 0
      
      // Determine if we're dealing with a large dataset (>100K items)
      const isLargeDataset = totalItems > 100000
      
      // Check if we're in read-only mode (from parent Brainy instance)
      const isReadOnly = this.options?.readOnly || false
      
      // Get memory information based on environment
      const memoryInfo = await this.detectAvailableMemory()
      
      // If memory detection failed, use the synchronous method
      if (!memoryInfo) {
        return this.detectOptimalCacheSize()
      }
      
      // Estimate average entry size (in bytes)
      // This is a conservative estimate for complex objects with vectors
      const ESTIMATED_BYTES_PER_ENTRY = 1024 // 1KB per entry
      
      // Base memory percentage - 10% by default
      let memoryPercentage = 0.1
      
      // Adjust based on operating mode and dataset size
      if (isReadOnly) {
        // In read-only mode, we can use more memory for caching
        memoryPercentage = 0.25 // 25% of free memory
        
        // For large datasets in read-only mode, be even more aggressive
        if (isLargeDataset) {
          memoryPercentage = 0.4 // 40% of free memory
        }
      } else if (isLargeDataset) {
        // For large datasets in normal mode, increase slightly
        memoryPercentage = 0.15 // 15% of free memory
      }
      
      // Calculate optimal size based on adjusted percentage
      const optimalSize = Math.max(
        Math.floor(memoryInfo.freeMemory * memoryPercentage / ESTIMATED_BYTES_PER_ENTRY),
        1000
      )
      
      // If we know the total dataset size, cap at a reasonable percentage
      if (totalItems > 0) {
        // In read-only mode, we can cache a larger percentage
        const maxPercentage = isReadOnly ? 0.5 : 0.3
        const maxItems = Math.ceil(totalItems * maxPercentage)
        
        // Return the smaller of the two to avoid excessive memory usage
        return Math.min(optimalSize, maxItems)
      }
      
      return optimalSize
    } catch (error) {
      console.warn('Error detecting optimal cache size asynchronously:', error)
      return 1000 // Conservative default
    }
  }
  
  /**
   * Detect available memory using Node-like runtime primitives.
   *
   * Brainy 8.0 only runs on Node.js, Bun, and Deno, so this method reads real
   * values from `node:os` (`process.memoryUsage()` is not used here because we
   * want system-wide free memory, not just the V8 heap). If the import fails,
   * we fall back to conservative defaults that match the synchronous path.
   *
   * @returns An object with totalMemory and freeMemory in bytes, or null if detection fails
   */
  private async detectAvailableMemory(): Promise<{ totalMemory: number, freeMemory: number } | null> {
    try {
      try {
        // Use dynamic import for OS module
        const os = await import('node:os')

        // Get actual system memory information
        const totalMemory = os.totalmem()
        const freeMemory = os.freemem()

        return { totalMemory, freeMemory }
      } catch (error) {
        console.warn('Failed to detect memory via node:os:', error)
      }

      // If detection failed, use conservative defaults
      return {
        totalMemory: 8 * 1024 * 1024 * 1024,  // Assume 8GB total
        freeMemory: 4 * 1024 * 1024 * 1024    // Assume 4GB free
      }
    } catch (error) {
      console.warn('Memory detection failed:', error)
      return null
    }
  }
  
  /**
   * Tune cache parameters based on statistics and environment
   * This method is called periodically if auto-tuning is enabled
   * 
   * The auto-tuning process:
   * 1. Retrieves storage statistics if available
   * 2. Tunes each parameter based on statistics and environment
   * 3. Logs the tuned parameters if debug is enabled
   * 
   * Auto-tuning helps optimize cache performance by adapting to:
   * - The current environment (Node.js, browser, worker)
   * - Available system resources (memory, CPU)
   * - Usage patterns (read-heavy vs. write-heavy workloads)
   * - Cache efficiency (hit/miss ratios)
   */
  private async tuneParameters(): Promise<void> {
    // Skip if auto-tuning is disabled
    if (!this.autoTune) return
    
    // Check if it's time to tune parameters
    const now = Date.now()
    if (now - this.lastAutoTuneTime < this.autoTuneInterval) return
    
    // Update last tune time
    this.lastAutoTuneTime = now
    
    try {
      // Get storage statistics if available
      if (this.coldStorage && typeof this.coldStorage.getStatistics === 'function') {
        this.storageStatistics = await this.coldStorage.getStatistics()
      }
      
      // Get cache statistics for adaptive tuning
      const cacheStats = this.getStats()
      
      // Use the async version of tuneHotCacheSize which uses detectOptimalCacheSizeAsync
      await this.tuneHotCacheSize()
      
      // Tune eviction threshold based on hit/miss ratio
      this.tuneEvictionThreshold(cacheStats)
      
      // Tune warm cache TTL based on access patterns
      this.tuneWarmCacheTTL(cacheStats)
      
      // Tune batch size based on access patterns and storage type
      this.tuneBatchSize(cacheStats)
      
      // Log tuned parameters if debug is enabled
      if (process.env.DEBUG) {
        console.log('Cache parameters auto-tuned:', {
          hotCacheMaxSize: this.hotCacheMaxSize,
          hotCacheEvictionThreshold: this.hotCacheEvictionThreshold,
          warmCacheTTL: this.warmCacheTTL,
          batchSize: this.batchSize,
          cacheStats: {
            hotCacheSize: cacheStats.hotCacheSize,
            warmCacheSize: cacheStats.warmCacheSize,
            hotCacheHits: cacheStats.hotCacheHits,
            hotCacheMisses: cacheStats.hotCacheMisses,
            warmCacheHits: cacheStats.warmCacheHits,
            warmCacheMisses: cacheStats.warmCacheMisses
          }
        })
      }
    } catch (error) {
      console.warn('Error during cache parameter auto-tuning:', error)
    }
  }
  
  /**
   * Tune hot cache size based on statistics, environment, and operating mode
   * 
   * The hot cache size is tuned based on:
   * 1. Available memory in the current environment
   * 2. Total number of nodes and edges in the system
   * 3. Cache hit/miss ratio
   * 4. Operating mode (read-only vs. read-write)
   * 5. Storage type (S3, filesystem, memory)
   * 
   * Enhanced algorithm:
   * - Start with a size based on available memory and operating mode
   * - For large datasets in S3 or other remote storage, use more aggressive caching
   * - Adjust based on access patterns (read-heavy vs. write-heavy)
   * - For read-only mode, prioritize cache size over eviction speed
   * - Dynamically adjust based on hit/miss ratio and query patterns
   */
  private async tuneHotCacheSize(): Promise<void> {
    // Use the async version to get more accurate memory information
    let optimalSize = await this.detectOptimalCacheSizeAsync()

    const isReadOnly = this.options?.readOnly || false

    // Adjust by total entity count if we have storage statistics.
    if (this.storageStatistics) {
      const totalItems = (this.storageStatistics.totalNodes || 0) +
                         (this.storageStatistics.totalEdges || 0)

      if (totalItems > 0) {
        let percentageToCache = isReadOnly ? 0.3 : 0.2

        if (totalItems > 1_000_000) {
          percentageToCache = Math.min(percentageToCache, 0.15)
        } else if (totalItems > 100_000) {
          percentageToCache = Math.min(percentageToCache, 0.25)
        }

        const statisticsBasedSize = Math.ceil(totalItems * percentageToCache)
        optimalSize = Math.min(optimalSize, statisticsBasedSize)
      }
    }

    // Adjust by observed hit ratio.
    const totalAccesses = this.stats.hits + this.stats.misses
    if (totalAccesses > 100) {
      const hitRatio = this.stats.hits / totalAccesses

      if (hitRatio < 0.5) {
        const baseAdjustment = 0.5 - hitRatio
        const hitRatioFactor = isReadOnly ? 1 + baseAdjustment * 1.5 : 1 + baseAdjustment
        optimalSize = Math.ceil(optimalSize * hitRatioFactor)
      } else if (hitRatio > 0.9 && !isReadOnly) {
        optimalSize = Math.ceil(optimalSize * 0.9)
      }
    }

    // Read-heavy workloads warrant a slightly larger cache.
    if (this.storageStatistics?.operations) {
      const ops = this.storageStatistics.operations
      const totalOps = ops.total || 1
      const readOps = (ops.search || 0) + (ops.get || 0)

      if (totalOps > 100 && readOps / totalOps > 0.8) {
        optimalSize = Math.ceil(optimalSize * 1.2)
      }
    }

    const minSize = isReadOnly ? 2000 : 1000
    optimalSize = Math.max(optimalSize, minSize)
    
    // Update the hot cache max size
    this.hotCacheMaxSize = optimalSize
    this.stats.maxSize = optimalSize
  }
  
  /**
   * Tune eviction threshold based on statistics
   * 
   * The eviction threshold determines when items start being evicted from the hot cache.
   * It is tuned based on:
   * 1. Cache hit/miss ratio
   * 2. Operation patterns (read-heavy vs. write-heavy workloads)
   * 3. Memory pressure and available resources
   * 
   * Algorithm:
   * - Start with a default threshold of 0.8 (80% of max size)
   * - For high hit ratios, increase the threshold to keep more items in cache
   * - For low hit ratios, decrease the threshold to evict items more aggressively
   * - For read-heavy workloads, use a higher threshold
   * - For write-heavy workloads, use a lower threshold
   * - Under memory pressure, use a lower threshold to conserve resources
   * 
   * @param cacheStats Optional cache statistics for more adaptive tuning
   */
  private tuneEvictionThreshold(cacheStats?: CacheStats): void {
    // Default threshold
    let threshold = 0.8
    
    // Use provided cache stats or internal stats
    const stats = cacheStats || this.getStats()
    
    // Adjust based on hit/miss ratio if we have enough data
    const totalHotAccesses = stats.hotCacheHits + stats.hotCacheMisses
    if (totalHotAccesses > 100) {
      const hotHitRatio = stats.hotCacheHits / totalHotAccesses
      
      // If hit ratio is high, we can use a higher threshold
      // If hit ratio is low, we should use a lower threshold to evict more aggressively
      if (hotHitRatio > 0.8) {
        // High hit ratio, increase threshold (up to 0.9)
        threshold = Math.min(0.9, 0.8 + (hotHitRatio - 0.8) * 0.5)
      } else if (hotHitRatio < 0.5) {
        // Low hit ratio, decrease threshold (down to 0.6)
        threshold = Math.max(0.6, 0.8 - (0.5 - hotHitRatio) * 0.5)
      }
    }
    
    // If we have storage statistics with operation counts, adjust based on operation patterns
    if (this.storageStatistics && this.storageStatistics.operations) {
      const ops = this.storageStatistics.operations
      const totalOps = ops.total || 1
      
      // Calculate read/write ratio
      const readOps = ops.search || 0
      const writeOps = (ops.add || 0) + (ops.update || 0) + (ops.delete || 0)
      
      if (totalOps > 100) {
        const readRatio = readOps / totalOps
        const writeRatio = writeOps / totalOps
        
        // For read-heavy workloads, use higher threshold
        // For write-heavy workloads, use lower threshold
        if (readRatio > 0.8) {
          // Read-heavy, increase threshold slightly
          threshold = Math.min(0.9, threshold + 0.05)
        } else if (writeRatio > 0.5) {
          // Write-heavy, decrease threshold
          threshold = Math.max(0.6, threshold - 0.1)
        }
      }
    }
    
    // Check memory pressure - if hot cache is growing too fast relative to hits,
    // reduce the threshold to conserve memory
    if (stats.hotCacheSize > 0 && totalHotAccesses > 0) {
      const sizeToAccessRatio = stats.hotCacheSize / totalHotAccesses
      
      // If the ratio is high, it means we're caching a lot but not getting many hits
      if (sizeToAccessRatio > 10) {
        // Reduce threshold more aggressively under high memory pressure
        threshold = Math.max(0.5, threshold - 0.1)
      }
    }
    
    // If we're in read-only mode, we can be more aggressive with caching
    const isReadOnly = this.options?.readOnly || false
    if (isReadOnly) {
      threshold = Math.min(0.95, threshold + 0.05)
    }
    
    // Update the eviction threshold
    this.hotCacheEvictionThreshold = threshold
  }
  
  /**
   * Tune warm cache TTL based on statistics
   * 
   * The warm cache TTL determines how long items remain in the warm cache.
   * It is tuned based on:
   * 1. Update frequency from operation statistics
   * 2. Warm cache hit/miss ratio
   * 3. Access patterns and frequency
   * 4. Available storage resources
   * 
   * Algorithm:
   * - Start with a default TTL of 24 hours
   * - For frequently updated data, use a shorter TTL
   * - For rarely updated data, use a longer TTL
   * - For frequently accessed data, use a longer TTL
   * - For rarely accessed data, use a shorter TTL
   * - Under storage pressure, use a shorter TTL
   * 
   * @param cacheStats Optional cache statistics for more adaptive tuning
   */
  private tuneWarmCacheTTL(cacheStats?: CacheStats): void {
    // Default TTL (24 hours)
    let ttl = 24 * 60 * 60 * 1000
    
    // Use provided cache stats or internal stats
    const stats = cacheStats || this.getStats()
    
    // Adjust based on warm cache hit/miss ratio if we have enough data
    const totalWarmAccesses = stats.warmCacheHits + stats.warmCacheMisses
    if (totalWarmAccesses > 50) {
      const warmHitRatio = stats.warmCacheHits / totalWarmAccesses
      
      // If warm cache hit ratio is high, items in warm cache are useful
      // so we should keep them longer
      if (warmHitRatio > 0.7) {
        // High hit ratio, increase TTL (up to 36 hours)
        ttl = Math.min(36 * 60 * 60 * 1000, ttl * (1 + (warmHitRatio - 0.7)))
      } else if (warmHitRatio < 0.3) {
        // Low hit ratio, decrease TTL (down to 12 hours)
        ttl = Math.max(12 * 60 * 60 * 1000, ttl * (0.8 - (0.3 - warmHitRatio)))
      }
    }
    
    // If we have storage statistics with operation counts, adjust based on update frequency
    if (this.storageStatistics && this.storageStatistics.operations) {
      const ops = this.storageStatistics.operations
      const totalOps = ops.total || 1
      const updateOps = (ops.update || 0)
      
      if (totalOps > 100) {
        const updateRatio = updateOps / totalOps
        
        // For frequently updated data, use shorter TTL
        // For rarely updated data, use longer TTL
        if (updateRatio > 0.3) {
          // Frequently updated, decrease TTL (down to 6 hours)
          ttl = Math.max(6 * 60 * 60 * 1000, ttl * (1 - updateRatio * 0.5))
        } else if (updateRatio < 0.1) {
          // Rarely updated, increase TTL (up to 48 hours)
          ttl = Math.min(48 * 60 * 60 * 1000, ttl * (1.2 - updateRatio))
        }
      }
    }
    
    // Check warm cache size relative to hot cache size
    // If warm cache is much larger than hot cache, reduce TTL to prevent excessive storage use
    if (stats.warmCacheSize > 0 && stats.hotCacheSize > 0) {
      const warmToHotRatio = stats.warmCacheSize / stats.hotCacheSize
      
      if (warmToHotRatio > 5) {
        // Warm cache is much larger than hot cache, reduce TTL
        ttl = Math.max(6 * 60 * 60 * 1000, ttl * (0.9 - Math.min(0.3, (warmToHotRatio - 5) / 20)))
      }
    }
    
    // If we're in read-only mode, we can use a longer TTL
    const isReadOnly = this.options?.readOnly || false
    if (isReadOnly) {
      ttl = Math.min(72 * 60 * 60 * 1000, ttl * 1.5)
    }
    
    // Update the warm cache TTL
    this.warmCacheTTL = ttl
  }
  
  /**
   * Tune batch size based on environment, statistics, and operating mode
   * 
   * The batch size determines how many items are processed in a single batch
   * for operations like prefetching. It is tuned based on:
   * 1. Current environment (Node.js, browser, worker)
   * 2. Available memory
   * 3. Operation patterns
   * 4. Cache hit/miss ratio
   * 5. Operating mode (read-only vs. read-write)
   * 6. Storage type (S3, filesystem, memory)
   * 7. Dataset size
   * 8. Cache efficiency and access patterns
   * 
   * Enhanced algorithm:
   * - Start with a default based on the environment
   * - For large datasets in S3 or other remote storage, use larger batches
   * - For read-only mode, use larger batches to improve throughput
   * - Dynamically adjust based on network latency and throughput
   * - Balance between memory usage and performance
   * - Adapt to cache hit/miss patterns
   * 
   * @param cacheStats Optional cache statistics for more adaptive tuning
   */
  private tuneBatchSize(cacheStats?: CacheStats): void {
    let batchSize = 10
    const stats = cacheStats || this.getStats()
    const isReadOnly = this.options?.readOnly || false

    const totalItems = this.storageStatistics
      ? (this.storageStatistics.totalNodes || 0) + (this.storageStatistics.totalEdges || 0)
      : 0
    const isLargeDataset = totalItems > 100_000
    const isVeryLargeDataset = totalItems > 1_000_000

    // Brainy 8.0 ships Node-like runtimes only (Bun, Deno, Node). All paths
    // funnel into the same batch-size envelope.
    batchSize = isReadOnly ? 30 : 20
    if (isLargeDataset) batchSize = Math.min(100, batchSize * 1.5)
    if (isVeryLargeDataset) batchSize = Math.min(200, batchSize * 2)

    // Adjust by hit-ratio observations.
    const totalHotAccesses = stats.hotCacheHits + stats.hotCacheMisses
    const totalWarmAccesses = stats.warmCacheHits + stats.warmCacheMisses

    if (totalHotAccesses > 100) {
      const hotHitRatio = stats.hotCacheHits / totalHotAccesses
      if (hotHitRatio > 0.8) batchSize = Math.min(batchSize * 1.5, 150)
      else if (hotHitRatio < 0.4) batchSize = Math.max(5, batchSize * 0.8)
    }

    if (totalWarmAccesses > 50) {
      const warmHitRatio = stats.warmCacheHits / totalWarmAccesses
      if (warmHitRatio > 0.7) batchSize = Math.min(batchSize * 1.3, 120)
      else if (warmHitRatio < 0.3) batchSize = Math.max(5, batchSize * 0.9)
    }
    
    // If we have storage statistics with operation counts, adjust based on operation patterns
    if (this.storageStatistics && this.storageStatistics.operations) {
      const ops = this.storageStatistics.operations
      const totalOps = ops.total || 1
      const searchOps = (ops.search || 0)
      const getOps = (ops.get || 0)
      
      if (totalOps > 100) {
        const searchRatio = searchOps / totalOps
        const getRatio = getOps / totalOps

        if (searchRatio > 0.6) {
          batchSize = Math.min(100, Math.ceil(batchSize * 1.5))
        }
        if (getRatio > 0.6) {
          batchSize = Math.max(10, Math.ceil(batchSize * 0.9))
        }
      }
    }

    // Memory-pressure trim.
    if (stats.hotCacheSize > 0 && this.hotCacheMaxSize > 0) {
      const cacheUtilization = stats.hotCacheSize / this.hotCacheMaxSize
      if (cacheUtilization > 0.85) {
        batchSize = Math.max(5, Math.floor(batchSize * 0.8))
      }
    }

    // Combined hot+warm hit-ratio adjustment.
    const totalAccesses = stats.hotCacheHits + stats.hotCacheMisses + stats.warmCacheHits + stats.warmCacheMisses
    if (totalAccesses > 100) {
      const hitRatio = (stats.hotCacheHits + stats.warmCacheHits) / totalAccesses
      const increaseFactorForLowHitRatio = isReadOnly ? 1.5 : 1.2
      const decreaseFactorForHighHitRatio = isReadOnly ? 0.9 : 0.8

      if (hitRatio > 0.8 && !isVeryLargeDataset) {
        batchSize = Math.max(isReadOnly ? 10 : 5, Math.floor(batchSize * decreaseFactorForHighHitRatio))
      } else if (hitRatio < 0.5) {
        const maxBatchSize = isVeryLargeDataset ? 150 : 100
        batchSize = Math.min(maxBatchSize, Math.ceil(batchSize * increaseFactorForLowHitRatio))
      }
    }

    // Min/max envelope. 8.0 is Node-like only.
    const minBatchSize = isReadOnly ? 10 : 5
    batchSize = Math.max(minBatchSize, batchSize)
    batchSize = Math.min(150, batchSize)
    
    // Update the batch size with the adaptively tuned value
    this.batchSize = Math.round(batchSize)
  }
  
  /**
   * Resolve the warm-tier storage type. Brainy 8.0 ships filesystem +
   * memory only.
   */
  private detectWarmStorageType(): StorageType {
    return StorageType.FILESYSTEM
  }

  /**
   * Resolve the cold-tier storage type. Brainy 8.0 ships filesystem +
   * memory only.
   */
  private detectColdStorageType(): StorageType {
    return StorageType.FILESYSTEM
  }
  
  /**
   * Initialize warm storage adapter
   */
  private initializeWarmStorage(): any {
    // Implementation depends on the detected storage type
    // For now, return null as this will be provided by the storage adapter
    return null
  }
  
  /**
   * Initialize cold storage adapter
   */
  private initializeColdStorage(): any {
    // Implementation depends on the detected storage type
    // For now, return null as this will be provided by the storage adapter
    return null
  }
  
  /**
   * Get an item from cache, trying each level in order
   * @param id The item ID
   * @returns The cached item or null if not found
   */
  public async get(id: string): Promise<T | null> {
    // Check if it's time to tune parameters
    await this.checkAndTuneParameters()
    
    // Try hot cache first (fastest)
    const hotCacheEntry = this.hotCache.get(id)
    if (hotCacheEntry) {
      // Update access metadata
      hotCacheEntry.lastAccessed = Date.now()
      hotCacheEntry.accessCount++
      
      // Update stats
      this.stats.hits++
      
      return hotCacheEntry.data
    }
    
    // Try warm cache next
    try {
      const warmCacheItem = await this.getFromWarmCache(id)
      if (warmCacheItem) {
        // Promote to hot cache
        this.addToHotCache(id, warmCacheItem)
        
        // Update stats
        this.stats.hits++
        
        return warmCacheItem
      }
    } catch (error) {
      console.warn(`Error accessing warm cache for ${id}:`, error)
    }
    
    // Finally, try cold storage
    try {
      const coldStorageItem = await this.getFromColdStorage(id)
      if (coldStorageItem) {
        // Promote to hot and warm caches
        this.addToHotCache(id, coldStorageItem)
        await this.addToWarmCache(id, coldStorageItem)
        
        // Update stats
        this.stats.misses++
        
        return coldStorageItem
      }
    } catch (error) {
      console.warn(`Error accessing cold storage for ${id}:`, error)
    }
    
    // Item not found in any cache level
    this.stats.misses++
    return null
  }
  
  /**
   * Get an item from warm cache
   * @param id The item ID
   * @returns The cached item or null if not found
   */
  private async getFromWarmCache(id: string): Promise<T | null> {
    if (!this.warmStorage) return null
    
    try {
      return await this.warmStorage.get(id)
    } catch (error) {
      console.warn(`Error getting item ${id} from warm cache:`, error)
      return null
    }
  }
  
  /**
   * Get an item from cold storage
   * @param id The item ID
   * @returns The item or null if not found
   */
  private async getFromColdStorage(id: string): Promise<T | null> {
    if (!this.coldStorage) return null
    
    try {
      return await this.coldStorage.get(id)
    } catch (error) {
      console.warn(`Error getting item ${id} from cold storage:`, error)
      return null
    }
  }
  
  /**
   * Add an item to hot cache
   * @param id The item ID
   * @param item The item to cache
   */
  private addToHotCache(id: string, item: T): void {
    // Check if we need to evict items
    if (this.hotCache.size >= this.hotCacheMaxSize * this.hotCacheEvictionThreshold) {
      this.evictFromHotCache()
    }
    
    // Add to hot cache
    this.hotCache.set(id, {
      data: item,
      lastAccessed: Date.now(),
      accessCount: 1,
      expiresAt: null // Hot cache items don't expire
    })
    
    // Update stats
    this.stats.size = this.hotCache.size
  }
  
  /**
   * Add an item to warm cache
   * @param id The item ID
   * @param item The item to cache
   */
  private async addToWarmCache(id: string, item: T): Promise<void> {
    if (!this.warmStorage) return
    
    try {
      // Add to warm cache with TTL
      await this.warmStorage.set(id, item, {
        ttl: this.warmCacheTTL
      })
    } catch (error) {
      console.warn(`Error adding item ${id} to warm cache:`, error)
    }
  }
  
  /**
   * Evict items from hot cache based on LRU policy
   */
  private evictFromHotCache(): void {
    // Find the least recently used items
    const entries = Array.from(this.hotCache.entries())
    
    // Sort by last accessed time (oldest first)
    entries.sort((a, b) => a[1].lastAccessed - b[1].lastAccessed)
    
    // Remove the oldest 20% of items
    const itemsToRemove = Math.ceil(this.hotCache.size * 0.2)
    for (let i = 0; i < itemsToRemove && i < entries.length; i++) {
      this.hotCache.delete(entries[i][0])
      this.stats.evictions++
    }
    
    // Update stats
    this.stats.size = this.hotCache.size
    
    if (process.env.DEBUG) {
      console.log(`Evicted ${itemsToRemove} items from hot cache, new size: ${this.hotCache.size}`)
    }
  }
  
  /**
   * Set an item in all cache levels
   * @param id The item ID
   * @param item The item to cache
   */
  public async set(id: string, item: T): Promise<void> {
    // Add to hot cache
    this.addToHotCache(id, item)
    
    // Add to warm cache
    await this.addToWarmCache(id, item)
    
    // Add to cold storage
    if (this.coldStorage) {
      try {
        await this.coldStorage.set(id, item)
      } catch (error) {
        console.warn(`Error adding item ${id} to cold storage:`, error)
      }
    }
  }
  
  /**
   * Delete an item from all cache levels
   * @param id The item ID to delete
   */
  public async delete(id: string): Promise<void> {
    // Remove from hot cache
    this.hotCache.delete(id)
    
    // Remove from warm cache
    if (this.warmStorage) {
      try {
        await this.warmStorage.delete(id)
      } catch (error) {
        console.warn(`Error deleting item ${id} from warm cache:`, error)
      }
    }
    
    // Remove from cold storage
    if (this.coldStorage) {
      try {
        await this.coldStorage.delete(id)
      } catch (error) {
        console.warn(`Error deleting item ${id} from cold storage:`, error)
      }
    }
    
    // Update stats
    this.stats.size = this.hotCache.size
  }
  
  /**
   * Clear all cache levels
   */
  public async clear(): Promise<void> {
    // Clear hot cache
    this.hotCache.clear()
    
    // Clear warm cache
    if (this.warmStorage) {
      try {
        await this.warmStorage.clear()
      } catch (error) {
        console.warn('Error clearing warm cache:', error)
      }
    }
    
    // Clear cold storage
    if (this.coldStorage) {
      try {
        await this.coldStorage.clear()
      } catch (error) {
        console.warn('Error clearing cold storage:', error)
      }
    }
    
    // Reset stats
    this.stats = {
      hits: 0,
      misses: 0,
      evictions: 0,
      size: 0,
      maxSize: this.hotCacheMaxSize,
      hotCacheSize: 0,
      warmCacheSize: 0,
      hotCacheHits: 0,
      hotCacheMisses: 0,
      warmCacheHits: 0,
      warmCacheMisses: 0
    }
  }
  
  /**
   * Get cache statistics
   * @returns Cache statistics
   */
  public getStats(): CacheStats {
    return { ...this.stats }
  }
  
  /**
   * Prefetch items based on ID patterns or relationships
   * @param ids Array of IDs to prefetch
   */
  public async prefetch(ids: string[]): Promise<void> {
    // Check if it's time to tune parameters
    await this.checkAndTuneParameters()
    
    // Prefetch in batches to avoid overwhelming the system
    const batches: string[][] = []
    
    // Split into batches using the configurable batch size
    for (let i = 0; i < ids.length; i += this.batchSize) {
      const batch = ids.slice(i, i + this.batchSize)
      batches.push(batch)
    }
    
    // Process each batch
    for (const batch of batches) {
      await Promise.all(
        batch.map(async (id) => {
          // Skip if already in hot cache
          if (this.hotCache.has(id)) return
          
          try {
            // Try to get from any cache level
            await this.get(id)
          } catch (error) {
            // Ignore errors during prefetching
            if (process.env.DEBUG) {
              console.warn(`Error prefetching ${id}:`, error)
            }
          }
        })
      )
    }
  }
  
  /**
   * Check if it's time to tune parameters and do so if needed
   * This is called before operations that might benefit from tuned parameters
   * 
   * This method serves as a checkpoint for auto-tuning, ensuring that:
   * 1. Parameters are tuned periodically based on the auto-tune interval
   * 2. Tuning happens before critical operations that would benefit from optimized parameters
   * 3. Tuning doesn't happen too frequently, which could impact performance
   * 
   * By calling this method before get(), getMany(), and prefetch() operations,
   * we ensure that the cache parameters are optimized for the current workload
   * without adding unnecessary overhead to every operation.
   */
  private async checkAndTuneParameters(): Promise<void> {
    // Skip if auto-tuning is disabled
    if (!this.autoTune) return
    
    // Check if it's time to tune parameters
    const now = Date.now()
    if (now - this.lastAutoTuneTime >= this.autoTuneInterval) {
      await this.tuneParameters()
    }
  }
  
  /**
   * Get multiple items at once, optimizing for batch retrieval
   * @param ids Array of IDs to get
   * @returns Map of ID to item
   */
  public async getMany(ids: string[]): Promise<Map<string, T>> {
    // Check if it's time to tune parameters
    await this.checkAndTuneParameters()
    
    const result = new Map<string, T>()
    
    // First check hot cache for all IDs
    const missingIds: string[] = []
    for (const id of ids) {
      const hotCacheEntry = this.hotCache.get(id)
      if (hotCacheEntry) {
        // Update access metadata
        hotCacheEntry.lastAccessed = Date.now()
        hotCacheEntry.accessCount++
        
        // Add to result
        result.set(id, hotCacheEntry.data)
        
        // Update stats
        this.stats.hits++
      } else {
        missingIds.push(id)
      }
    }
    
    if (missingIds.length === 0) {
      return result
    }
    
    // Try to get missing items from warm cache
    if (this.warmStorage) {
      try {
        const warmCacheItems = await this.warmStorage.getMany(missingIds)
        for (const [id, item] of warmCacheItems.entries()) {
          if (item) {
            // Promote to hot cache
            this.addToHotCache(id, item)
            
            // Add to result
            result.set(id, item)
            
            // Update stats
            this.stats.hits++
            
            // Remove from missing IDs
            const index = missingIds.indexOf(id)
            if (index !== -1) {
              missingIds.splice(index, 1)
            }
          }
        }
      } catch (error) {
        console.warn('Error accessing warm cache for batch:', error)
      }
    }
    
    if (missingIds.length === 0) {
      return result
    }
    
    // Try to get remaining missing items from cold storage
    if (this.coldStorage) {
      try {
        const coldStorageItems = await this.coldStorage.getMany(missingIds)
        for (const [id, item] of coldStorageItems.entries()) {
          if (item) {
            // Promote to hot and warm caches
            this.addToHotCache(id, item)
            await this.addToWarmCache(id, item)
            
            // Add to result
            result.set(id, item)
            
            // Update stats
            this.stats.misses++
          }
        }
      } catch (error) {
        console.warn('Error accessing cold storage for batch:', error)
      }
    }
    
    return result
  }
  
  /**
   * Set the storage adapters for warm and cold caches
   * @param warmStorage Warm cache storage adapter
   * @param coldStorage Cold storage adapter
   */
  public setStorageAdapters(warmStorage: any, coldStorage: any): void {
    this.warmStorage = warmStorage
    this.coldStorage = coldStorage
  }
}