brainy/src/utils/jsonProcessing.ts

/**
 * Utility functions for processing JSON documents for vectorization and search
 */

/**
 * Extracts text from a JSON object for vectorization
 * This function recursively processes the JSON object and extracts text from all fields
 * It can also prioritize specific fields if provided
 * 
 * @param jsonObject The JSON object to extract text from
 * @param options Configuration options for text extraction
 * @returns A string containing the extracted text
 */
export function extractTextFromJson(
  jsonObject: any,
  options: {
    priorityFields?: string[] // Fields to prioritize (will be repeated for emphasis)
    excludeFields?: string[] // Fields to exclude from extraction
    includeFieldNames?: boolean // Whether to include field names in the extracted text
    maxDepth?: number // Maximum depth to recurse into nested objects
    currentDepth?: number // Current recursion depth (internal use)
    fieldPath?: string[] // Current field path (internal use)
  } = {}
): string {
  // Set default options
  const {
    priorityFields = [],
    excludeFields = [],
    includeFieldNames = true,
    maxDepth = 5,
    currentDepth = 0,
    fieldPath = []
  } = options

  // If input is not an object or array, or we've reached max depth, return as string
  if (
    jsonObject === null ||
    jsonObject === undefined ||
    typeof jsonObject !== 'object' ||
    currentDepth >= maxDepth
  ) {
    return String(jsonObject || '')
  }

  const extractedText: string[] = []
  const priorityText: string[] = []

  // Process arrays
  if (Array.isArray(jsonObject)) {
    for (let i = 0; i < jsonObject.length; i++) {
      const value = jsonObject[i]
      const newPath = [...fieldPath, i.toString()]
      
      // Recursively extract text from array items
      const itemText = extractTextFromJson(value, {
        priorityFields,
        excludeFields,
        includeFieldNames,
        maxDepth,
        currentDepth: currentDepth + 1,
        fieldPath: newPath
      })
      
      if (itemText) {
        extractedText.push(itemText)
      }
    }
  } 
  // Process objects
  else {
    for (const [key, value] of Object.entries(jsonObject)) {
      // Skip excluded fields
      if (excludeFields.includes(key)) {
        continue
      }

      const newPath = [...fieldPath, key]
      const fullPath = newPath.join('.')
      
      // Check if this is a priority field
      const isPriority = priorityFields.some(field => {
        // Exact match
        if (field === key) return true
        // Path match
        if (field === fullPath) return true
        // Wildcard match (e.g., "user.*" matches "user.name", "user.email", etc.)
        if (field.endsWith('.*') && fullPath.startsWith(field.slice(0, -2))) return true
        return false
      })

      // Get the field value as text
      let fieldText: string
      
      if (typeof value === 'object' && value !== null) {
        // Recursively extract text from nested objects
        fieldText = extractTextFromJson(value, {
          priorityFields,
          excludeFields,
          includeFieldNames,
          maxDepth,
          currentDepth: currentDepth + 1,
          fieldPath: newPath
        })
      } else {
        fieldText = String(value || '')
      }

      // Add field name if requested
      if (includeFieldNames && fieldText) {
        fieldText = `${key}: ${fieldText}`
      }

      // Add to appropriate collection
      if (fieldText) {
        if (isPriority) {
          priorityText.push(fieldText)
        } else {
          extractedText.push(fieldText)
        }
      }
    }
  }

  // Combine priority text (repeated for emphasis) and regular text
  return [...priorityText, ...priorityText, ...extractedText].join(' ')
}

/**
 * Prepares a JSON document for vectorization
 * This function extracts text from the JSON document and formats it for optimal vectorization
 * 
 * @param jsonDocument The JSON document to prepare
 * @param options Configuration options for preparation
 * @returns A string ready for vectorization
 */
export function prepareJsonForVectorization(
  jsonDocument: any,
  options: {
    priorityFields?: string[]
    excludeFields?: string[]
    includeFieldNames?: boolean
    maxDepth?: number
  } = {}
): string {
  // If input is a string, try to parse it as JSON
  let document = jsonDocument
  if (typeof jsonDocument === 'string') {
    try {
      document = JSON.parse(jsonDocument)
    } catch (e) {
      // If parsing fails, treat it as a plain string
      return jsonDocument
    }
  }

  // If not an object after parsing, return as is
  if (typeof document !== 'object' || document === null) {
    return String(document || '')
  }

  // Extract text from the document
  return extractTextFromJson(document, options)
}

/**
 * Extracts text from a specific field in a JSON document
 * This is useful for searching within specific fields
 * 
 * @param jsonDocument The JSON document to extract from
 * @param fieldPath The path to the field (e.g., "user.name" or "addresses[0].city")
 * @returns The extracted text or empty string if field not found
 */
export function extractFieldFromJson(
  jsonDocument: any,
  fieldPath: string
): string {
  // If input is a string, try to parse it as JSON
  let document = jsonDocument
  if (typeof jsonDocument === 'string') {
    try {
      document = JSON.parse(jsonDocument)
    } catch (e) {
      // If parsing fails, return empty string
      return ''
    }
  }

  // If not an object after parsing, return empty string
  if (typeof document !== 'object' || document === null) {
    return ''
  }

  // Parse the field path
  const parts = fieldPath.split('.')
  let current = document

  // Navigate through the path
  for (const part of parts) {
    // Handle array indexing (e.g., "addresses[0]")
    const match = part.match(/^([^[]+)(?:\[(\d+)\])?$/)
    if (!match) {
      return ''
    }

    const [, key, indexStr] = match
    
    // Move to the next level
    current = current[key]
    
    // If we have an array index, access that element
    if (indexStr !== undefined && Array.isArray(current)) {
      const index = parseInt(indexStr, 10)
      current = current[index]
    }

    // If we've reached a null or undefined value, return empty string
    if (current === null || current === undefined) {
      return ''
    }
  }

  // Convert the final value to string
  return typeof current === 'object' 
    ? JSON.stringify(current) 
    : String(current)
}