open-brainy/src/storage/adapters/memoryStorage.ts

/**
 * Memory Storage Adapter
 * In-memory storage adapter for environments where persistent storage is not available or needed
 */

import {
  GraphVerb,
  HNSWNoun,
  HNSWVerb,
  NounMetadata,
  VerbMetadata,
  HNSWNounWithMetadata,
  HNSWVerbWithMetadata,
  StatisticsData
} from '../../coreTypes.js'
import { BaseStorage, STATISTICS_KEY } from '../baseStorage.js'
import { PaginatedResult } from '../../types/paginationTypes.js'

// No type aliases needed - using the original types directly

/**
 * In-memory storage adapter
 * Uses Maps to store data in memory
 */
export class MemoryStorage extends BaseStorage {
  // Single map of noun ID to noun
  private nouns: Map<string, HNSWNoun> = new Map()
  private verbs: Map<string, HNSWVerb> = new Map()
  private statistics: StatisticsData | null = null

  // Unified object store for primitive operations (replaces metadata, nounMetadata, verbMetadata)
  private objectStore: Map<string, any> = new Map()

  // Backward compatibility aliases
  private get metadata(): Map<string, any> {
    return this.objectStore
  }
  private get nounMetadata(): Map<string, any> {
    return this.objectStore
  }
  private get verbMetadata(): Map<string, any> {
    return this.objectStore
  }

  constructor() {
    super()
  }

  /**
   * Initialize the storage adapter
   * Nothing to initialize for in-memory storage
   */
  public async init(): Promise<void> {
    this.isInitialized = true
  }

  /**
   * Save a noun to storage (v4.0.0: pure vector only, no metadata)
   */
  protected async saveNoun_internal(noun: HNSWNoun): Promise<void> {
    const isNew = !this.nouns.has(noun.id)

    // Create a deep copy to avoid reference issues
    // v4.0.0: Store ONLY vector data (no metadata field)
    // Metadata is saved separately via saveNounMetadata() by base class
    const nounCopy: HNSWNoun = {
      id: noun.id,
      vector: [...noun.vector],
      connections: new Map(),
      level: noun.level || 0
      // ✅ NO metadata field in v4.0.0
    }

    // Copy connections
    for (const [level, connections] of noun.connections.entries()) {
      nounCopy.connections.set(level, new Set(connections))
    }

    // Save the noun directly in the nouns map
    this.nouns.set(noun.id, nounCopy)

    // Note: Count tracking happens in saveNounMetadata since type info is in metadata now
  }

  /**
   * Get a noun from storage (v4.0.0: returns pure vector only)
   * Base class handles combining with metadata
   */
  protected async getNoun_internal(id: string): Promise<HNSWNoun | null> {
    // Get the noun directly from the nouns map
    const noun = this.nouns.get(id)

    // If not found, return null
    if (!noun) {
      return null
    }

    // Return a deep copy to avoid reference issues
    // v4.0.0: Return ONLY vector data (no metadata field)
    const nounCopy: HNSWNoun = {
      id: noun.id,
      vector: [...noun.vector],
      connections: new Map(),
      level: noun.level || 0
      // ✅ NO metadata field in v4.0.0
    }

    // Copy connections
    for (const [level, connections] of noun.connections.entries()) {
      nounCopy.connections.set(level, new Set(connections))
    }

    return nounCopy
  }

  /**
   * Get nouns with pagination and filtering
   * v4.0.0: Returns HNSWNounWithMetadata[] (includes metadata field)
   * @param options Pagination and filtering options
   * @returns Promise that resolves to a paginated result of nouns with metadata
   */
  public async getNouns(options: {
    pagination?: {
      offset?: number
      limit?: number
      cursor?: string
    }
    filter?: {
      nounType?: string | string[]
      service?: string | string[]
      metadata?: Record<string, any>
    }
  } = {}): Promise<{ items: HNSWNounWithMetadata[]; totalCount?: number; hasMore: boolean; nextCursor?: string }> {
    const pagination = options.pagination || {}
    const filter = options.filter || {}
    
    // Default values
    const offset = pagination.offset || 0
    const limit = pagination.limit || 100
    
    // Convert string types to arrays for consistent handling
    const nounTypes = filter.nounType 
      ? Array.isArray(filter.nounType) ? filter.nounType : [filter.nounType]
      : undefined
    
    const services = filter.service
      ? Array.isArray(filter.service) ? filter.service : [filter.service]
      : undefined
    
    // First, collect all noun IDs that match the filter criteria
    const matchingIds: string[] = []
    
    // Iterate through all nouns to find matches
    // v4.0.0: Load metadata from separate storage (no embedded metadata field)
    for (const [nounId, noun] of this.nouns.entries()) {
      // Get metadata from separate storage
      const metadata = await this.getNounMetadata(nounId)

      // Skip if no metadata (shouldn't happen in v4.0.0 but be defensive)
      if (!metadata) {
        continue
      }

      // Filter by noun type if specified
      if (nounTypes && metadata.noun && !nounTypes.includes(metadata.noun)) {
        continue
      }

      // Filter by service if specified
      if (services && metadata.service && !services.includes(metadata.service)) {
        continue
      }

      // Filter by metadata fields if specified
      if (filter.metadata) {
        let metadataMatch = true
        for (const [key, value] of Object.entries(filter.metadata)) {
          if (metadata[key] !== value) {
            metadataMatch = false
            break
          }
        }
        if (!metadataMatch) continue
      }

      // If we got here, the noun matches all filters
      matchingIds.push(nounId)
    }
    
    // Calculate pagination
    const totalCount = matchingIds.length
    const paginatedIds = matchingIds.slice(offset, offset + limit)
    const hasMore = offset + limit < totalCount
    
    // Create cursor for next page if there are more results
    const nextCursor = hasMore ? `${offset + limit}` : undefined
    
    // Fetch the actual nouns for the current page
    // v4.0.0: Return HNSWNounWithMetadata (includes metadata field)
    const items: HNSWNounWithMetadata[] = []
    for (const id of paginatedIds) {
      const noun = this.nouns.get(id)
      if (!noun) continue

      // Get metadata from separate storage
      const metadata = await this.getNounMetadata(id)
      if (!metadata) continue // Skip if no metadata

      // v4.0.0: Create HNSWNounWithMetadata with metadata field
      const nounWithMetadata: HNSWNounWithMetadata = {
        id: noun.id,
        vector: [...noun.vector],
        connections: new Map(),
        level: noun.level || 0,
        metadata: metadata // Include metadata field
      }

      // Copy connections
      for (const [level, connections] of noun.connections.entries()) {
        nounWithMetadata.connections.set(level, new Set(connections))
      }

      items.push(nounWithMetadata)
    }
    
    return {
      items,
      totalCount,
      hasMore,
      nextCursor
    }
  }

  /**
   * Get nouns with pagination - simplified interface for compatibility
   * v4.0.0: Returns HNSWNounWithMetadata[] (includes metadata field)
   */
  public async getNounsWithPagination(options: {
    limit?: number
    cursor?: string
    filter?: any
  } = {}): Promise<{
    items: HNSWNounWithMetadata[]
    totalCount: number
    hasMore: boolean
    nextCursor?: string
  }> {
    // Convert to the getNouns format
    const result = await this.getNouns({
      pagination: {
        offset: options.cursor ? parseInt(options.cursor) : 0,
        limit: options.limit || 100
      },
      filter: options.filter
    })

    return {
      items: result.items,
      totalCount: result.totalCount || 0,
      hasMore: result.hasMore,
      nextCursor: result.nextCursor
    }
  }

  /**
   * Get nouns by noun type
   * @param nounType The noun type to filter by
   * @returns Promise that resolves to an array of nouns of the specified noun type
   * @deprecated Use getNouns() with filter.nounType instead
   */
  protected async getNounsByNounType_internal(nounType: string): Promise<HNSWNoun[]> {
    const result = await this.getNouns({
      filter: {
        nounType
      }
    })
    return result.items
  }

  /**
   * Delete a noun from storage (v4.0.0)
   */
  protected async deleteNoun_internal(id: string): Promise<void> {
    // v4.0.0: Get type from separate metadata storage
    const metadata = await this.getNounMetadata(id)
    if (metadata) {
      const type = metadata.noun || 'default'
      this.decrementEntityCount(type)
    }
    this.nouns.delete(id)
  }

  /**
   * Save a verb to storage (v4.0.0: pure vector + core fields, no metadata)
   */
  protected async saveVerb_internal(verb: HNSWVerb): Promise<void> {
    const isNew = !this.verbs.has(verb.id)

    // Create a deep copy to avoid reference issues
    // v4.0.0: Include core relational fields but NO metadata field
    const verbCopy: HNSWVerb = {
      id: verb.id,
      vector: [...verb.vector],
      connections: new Map(),

      // CORE RELATIONAL DATA (part of HNSWVerb in v4.0.0)
      verb: verb.verb,
      sourceId: verb.sourceId,
      targetId: verb.targetId
      // ✅ NO metadata field in v4.0.0
    }

    // Copy connections
    for (const [level, connections] of verb.connections.entries()) {
      verbCopy.connections.set(level, new Set(connections))
    }

    // Save the verb directly in the verbs map
    this.verbs.set(verb.id, verbCopy)

    // Note: Count tracking happens in saveVerbMetadata since metadata is separate
  }

  /**
   * Get a verb from storage (v4.0.0: returns pure vector + core fields)
   * Base class handles combining with metadata
   */
  protected async getVerb_internal(id: string): Promise<HNSWVerb | null> {
    // Get the verb directly from the verbs map
    const verb = this.verbs.get(id)

    // If not found, return null
    if (!verb) {
      return null
    }

    // Return a deep copy of the HNSWVerb
    // v4.0.0: Include core relational fields but NO metadata field
    const verbCopy: HNSWVerb = {
      id: verb.id,
      vector: [...verb.vector],
      connections: new Map(),

      // CORE RELATIONAL DATA (part of HNSWVerb in v4.0.0)
      verb: verb.verb,
      sourceId: verb.sourceId,
      targetId: verb.targetId
      // ✅ NO metadata field in v4.0.0
    }

    // Copy connections
    for (const [level, connections] of verb.connections.entries()) {
      verbCopy.connections.set(level, new Set(connections))
    }

    return verbCopy
  }

  /**
   * Get verbs with pagination and filtering
   * v4.0.0: Returns HNSWVerbWithMetadata[] (includes metadata field)
   * @param options Pagination and filtering options
   * @returns Promise that resolves to a paginated result of verbs with metadata
   */
  public async getVerbs(options: {
    pagination?: {
      offset?: number
      limit?: number
      cursor?: string
    }
    filter?: {
      verbType?: string | string[]
      sourceId?: string | string[]
      targetId?: string | string[]
      service?: string | string[]
      metadata?: Record<string, any>
    }
  } = {}): Promise<{ items: HNSWVerbWithMetadata[]; totalCount?: number; hasMore: boolean; nextCursor?: string }> {
    const pagination = options.pagination || {}
    const filter = options.filter || {}
    
    // Default values
    const offset = pagination.offset || 0
    const limit = pagination.limit || 100
    
    // Convert string types to arrays for consistent handling
    const verbTypes = filter.verbType 
      ? Array.isArray(filter.verbType) ? filter.verbType : [filter.verbType]
      : undefined
    
    const sourceIds = filter.sourceId
      ? Array.isArray(filter.sourceId) ? filter.sourceId : [filter.sourceId]
      : undefined
    
    const targetIds = filter.targetId
      ? Array.isArray(filter.targetId) ? filter.targetId : [filter.targetId]
      : undefined
    
    const services = filter.service
      ? Array.isArray(filter.service) ? filter.service : [filter.service]
      : undefined
    
    // First, collect all verb IDs that match the filter criteria
    const matchingIds: string[] = []
    
    // Iterate through all verbs to find matches
    // v4.0.0: Core fields (verb, sourceId, targetId) are in HNSWVerb, not metadata
    for (const [verbId, hnswVerb] of this.verbs.entries()) {
      // Get the metadata for service/data filtering
      const metadata = await this.getVerbMetadata(verbId)

      // Filter by verb type if specified
      // v4.0.0: verb type is in HNSWVerb.verb
      if (verbTypes && !verbTypes.includes(hnswVerb.verb || '')) {
        continue
      }

      // Filter by source ID if specified
      // v4.0.0: sourceId is in HNSWVerb.sourceId
      if (sourceIds && !sourceIds.includes(hnswVerb.sourceId || '')) {
        continue
      }

      // Filter by target ID if specified
      // v4.0.0: targetId is in HNSWVerb.targetId
      if (targetIds && !targetIds.includes(hnswVerb.targetId || '')) {
        continue
      }

      // Filter by metadata fields if specified
      if (filter.metadata && metadata) {
        let metadataMatch = true
        for (const [key, value] of Object.entries(filter.metadata)) {
          const metadataValue = (metadata as any)[key]
          if (metadataValue !== value) {
            metadataMatch = false
            break
          }
        }
        if (!metadataMatch) continue
      }

      // Filter by service if specified
      if (services && metadata && metadata.service && !services.includes(metadata.service)) {
        continue
      }

      // If we got here, the verb matches all filters
      matchingIds.push(verbId)
    }
    
    // Calculate pagination
    const totalCount = matchingIds.length
    const paginatedIds = matchingIds.slice(offset, offset + limit)
    const hasMore = offset + limit < totalCount
    
    // Create cursor for next page if there are more results
    const nextCursor = hasMore ? `${offset + limit}` : undefined
    
    // Fetch the actual verbs for the current page
    // v4.0.0: Return HNSWVerbWithMetadata (includes metadata field)
    const items: HNSWVerbWithMetadata[] = []
    for (const id of paginatedIds) {
      const hnswVerb = this.verbs.get(id)
      if (!hnswVerb) continue

      // Get metadata from separate storage
      const metadata = await this.getVerbMetadata(id)
      if (!metadata) continue // Skip if no metadata

      // v4.0.0: Create HNSWVerbWithMetadata with metadata field
      const verbWithMetadata: HNSWVerbWithMetadata = {
        id: hnswVerb.id,
        vector: [...hnswVerb.vector],
        connections: new Map(),

        // Core relational fields (part of HNSWVerb)
        verb: hnswVerb.verb,
        sourceId: hnswVerb.sourceId,
        targetId: hnswVerb.targetId,

        // Metadata field
        metadata: metadata
      }

      // Copy connections
      for (const [level, connections] of hnswVerb.connections.entries()) {
        verbWithMetadata.connections.set(level, new Set(connections))
      }

      items.push(verbWithMetadata)
    }
    
    return {
      items,
      totalCount,
      hasMore,
      nextCursor
    }
  }

  /**
   * Get verbs by source
   * @deprecated Use getVerbs() with filter.sourceId instead
   */
  protected async getVerbsBySource_internal(sourceId: string): Promise<HNSWVerbWithMetadata[]> {
    const result = await this.getVerbs({
      filter: {
        sourceId
      }
    })
    return result.items
  }

  /**
   * Get verbs by target
   * @deprecated Use getVerbs() with filter.targetId instead
   */
  protected async getVerbsByTarget_internal(targetId: string): Promise<HNSWVerbWithMetadata[]> {
    const result = await this.getVerbs({
      filter: {
        targetId
      }
    })
    return result.items
  }

  /**
   * Get verbs by type
   * @deprecated Use getVerbs() with filter.verbType instead
   */
  protected async getVerbsByType_internal(type: string): Promise<HNSWVerbWithMetadata[]> {
    const result = await this.getVerbs({
      filter: {
        verbType: type
      }
    })
    return result.items
  }

  /**
   * Delete a verb from storage
   */
  protected async deleteVerb_internal(id: string): Promise<void> {
    // Delete the HNSWVerb from the verbs map
    this.verbs.delete(id)

    // CRITICAL: Also delete verb metadata - this is what getVerbs() uses to find verbs
    // Without this, getVerbsBySource() will still find "deleted" verbs via their metadata
    const metadata = await this.getVerbMetadata(id)
    if (metadata) {
      const verbType = metadata.verb || metadata.type || 'default'
      this.decrementVerbCount(verbType as string)

      // Delete the metadata using the base storage method
      await this.deleteVerbMetadata(id)
    }
  }

  /**
   * Primitive operation: Write object to path
   * All metadata operations use this internally via base class routing
   */
  protected async writeObjectToPath(path: string, data: any): Promise<void> {
    // Store in unified object store using path as key
    this.objectStore.set(path, JSON.parse(JSON.stringify(data)))
  }

  /**
   * Primitive operation: Read object from path
   * All metadata operations use this internally via base class routing
   */
  protected async readObjectFromPath(path: string): Promise<any | null> {
    const data = this.objectStore.get(path)
    if (!data) {
      return null
    }
    return JSON.parse(JSON.stringify(data))
  }

  /**
   * Primitive operation: Delete object from path
   * All metadata operations use this internally via base class routing
   */
  protected async deleteObjectFromPath(path: string): Promise<void> {
    this.objectStore.delete(path)
  }

  /**
   * Primitive operation: List objects under path prefix
   * All metadata operations use this internally via base class routing
   */
  protected async listObjectsUnderPath(prefix: string): Promise<string[]> {
    const paths: string[] = []
    for (const key of this.objectStore.keys()) {
      if (key.startsWith(prefix)) {
        paths.push(key)
      }
    }
    return paths.sort()
  }

  /**
   * Get multiple metadata objects in batches (CRITICAL: Prevents socket exhaustion)
   * Memory storage implementation is simple since all data is already in memory
   */
  public async getMetadataBatch(ids: string[]): Promise<Map<string, any>> {
    const results = new Map<string, any>()

    // Memory storage can handle all IDs at once since it's in-memory
    for (const id of ids) {
      // CRITICAL: Use getNounMetadata() instead of deprecated getMetadata()
      // This ensures we fetch from the correct noun metadata store (2-file system)
      const metadata = await this.getNounMetadata(id)
      if (metadata) {
        results.set(id, metadata)
      }
    }

    return results
  }

  /**
   * Clear all data from storage
   */
  public async clear(): Promise<void> {
    this.nouns.clear()
    this.verbs.clear()
    this.objectStore.clear()
    this.statistics = null

    // Clear the statistics cache
    this.statisticsCache = null
    this.statisticsModified = false
  }

  /**
   * Get information about storage usage and capacity
   */
  public async getStorageStatus(): Promise<{
    type: string
    used: number
    quota: number | null
    details?: Record<string, any>
  }> {
    return {
      type: 'memory',
      used: 0, // In-memory storage doesn't have a meaningful size
      quota: null, // In-memory storage doesn't have a quota
      details: {
        nodeCount: this.nouns.size,
        edgeCount: this.verbs.size,
        metadataCount: this.objectStore.size
      }
    }
  }

  /**
   * Save statistics data to storage
   * @param statistics The statistics data to save
   */
  protected async saveStatisticsData(statistics: StatisticsData): Promise<void> {
    // For memory storage, we just need to store the statistics in memory
    // Create a deep copy to avoid reference issues
    this.statistics = {
      nounCount: {...statistics.nounCount},
      verbCount: {...statistics.verbCount},
      metadataCount: {...statistics.metadataCount},
      hnswIndexSize: statistics.hnswIndexSize,
      lastUpdated: statistics.lastUpdated,
      // Include serviceActivity if present
      ...(statistics.serviceActivity && {
        serviceActivity: Object.fromEntries(
          Object.entries(statistics.serviceActivity).map(([k, v]) => [k, {...v}])
        )
      }),
      // Include services if present
      ...(statistics.services && {
        services: statistics.services.map(s => ({...s}))
      }),
      // Include distributedConfig if present
      ...(statistics.distributedConfig && { 
        distributedConfig: JSON.parse(JSON.stringify(statistics.distributedConfig)) 
      })
    }
    
    // Since this is in-memory, there's no need for time-based partitioning
    // or legacy file handling
  }

  /**
   * Get statistics data from storage
   * @returns Promise that resolves to the statistics data or null if not found
   */
  protected async getStatisticsData(): Promise<StatisticsData | null> {
    if (!this.statistics) {
      // CRITICAL FIX (v3.37.4): Statistics don't exist yet (first init)
      // Return minimal stats with counts instead of null
      // This prevents HNSW from seeing entityCount=0 during index rebuild
      return {
        nounCount: {},
        verbCount: {},
        metadataCount: {},
        hnswIndexSize: 0,
        totalNodes: this.totalNounCount,
        totalEdges: this.totalVerbCount,
        totalMetadata: 0,
        lastUpdated: new Date().toISOString()
      }
    }

    // Return a deep copy to avoid reference issues
    return {
      nounCount: {...this.statistics.nounCount},
      verbCount: {...this.statistics.verbCount},
      metadataCount: {...this.statistics.metadataCount},
      hnswIndexSize: this.statistics.hnswIndexSize,
      // CRITICAL FIX: Populate totalNodes and totalEdges from in-memory counts
      // HNSW rebuild depends on these fields to determine entity count
      totalNodes: this.totalNounCount,
      totalEdges: this.totalVerbCount,
      lastUpdated: this.statistics.lastUpdated,
      // Include serviceActivity if present
      ...(this.statistics.serviceActivity && {
        serviceActivity: Object.fromEntries(
          Object.entries(this.statistics.serviceActivity).map(([k, v]) => [k, {...v}])
        )
      }),
      // Include services if present
      ...(this.statistics.services && {
        services: this.statistics.services.map(s => ({...s}))
      }),
      // Include distributedConfig if present
      ...(this.statistics.distributedConfig && {
        distributedConfig: JSON.parse(JSON.stringify(this.statistics.distributedConfig))
      })
    }

    // Since this is in-memory, there's no need for fallback mechanisms
    // to check multiple storage locations
  }

  /**
   * Initialize counts from in-memory storage - O(1) operation (v4.0.0)
   */
  protected async initializeCounts(): Promise<void> {
    // For memory storage, initialize counts from current in-memory state
    this.totalNounCount = this.nouns.size
    this.totalVerbCount = this.verbs.size

    // Initialize type-based counts by scanning metadata storage (v4.0.0)
    this.entityCounts.clear()
    this.verbCounts.clear()

    // Count nouns by loading metadata for each
    for (const [nounId, noun] of this.nouns.entries()) {
      const metadata = await this.getNounMetadata(nounId)
      if (metadata) {
        const type = metadata.noun || 'default'
        this.entityCounts.set(type, (this.entityCounts.get(type) || 0) + 1)
      }
    }

    // Count verbs by loading metadata for each
    for (const [verbId, verb] of this.verbs.entries()) {
      const metadata = await this.getVerbMetadata(verbId)
      if (metadata) {
        // VerbMetadata doesn't have verb type - that's in HNSWVerb now
        // Use the verb's type from the HNSWVerb itself
        const type = verb.verb || 'default'
        this.verbCounts.set(type, (this.verbCounts.get(type) || 0) + 1)
      }
    }
  }

  /**
   * Persist counts to storage - no-op for memory storage
   */
  protected async persistCounts(): Promise<void> {
    // No persistence needed for in-memory storage
    // Counts are always accurate from the live data structures
  }

  // =============================================
  // HNSW Index Persistence (v3.35.0+)
  // =============================================

  /**
   * Get vector for a noun
   */
  public async getNounVector(id: string): Promise<number[] | null> {
    const noun = this.nouns.get(id)
    return noun ? [...noun.vector] : null
  }

  /**
   * Save HNSW graph data for a noun
   */
  public async saveHNSWData(nounId: string, hnswData: {
    level: number
    connections: Record<string, string[]>
  }): Promise<void> {
    // For memory storage, HNSW data is already in the noun object
    // This method is a no-op since saveNoun already stores the full graph
    // But we store it separately for consistency with other adapters
    const path = `hnsw/${nounId}.json`
    await this.writeObjectToPath(path, hnswData)
  }

  /**
   * Get HNSW graph data for a noun
   */
  public async getHNSWData(nounId: string): Promise<{
    level: number
    connections: Record<string, string[]>
  } | null> {
    const path = `hnsw/${nounId}.json`
    const data = await this.readObjectFromPath(path)
    return data || null
  }

  /**
   * Save HNSW system data (entry point, max level)
   */
  public async saveHNSWSystem(systemData: {
    entryPointId: string | null
    maxLevel: number
  }): Promise<void> {
    const path = 'system/hnsw-system.json'
    await this.writeObjectToPath(path, systemData)
  }

  /**
   * Get HNSW system data
   */
  public async getHNSWSystem(): Promise<{
    entryPointId: string | null
    maxLevel: number
  } | null> {
    const path = 'system/hnsw-system.json'
    const data = await this.readObjectFromPath(path)
    return data || null
  }
}