brainy/src/utils/metadataFilter.ts

/**
 * Smart metadata filtering for vector search
 * Filters DURING search to ensure relevant results
 * Simple API that just works without configuration
 */

import { SearchResult, HNSWNoun, HNSWNounWithMetadata } from '../coreTypes.js'
import { BrainyError } from '../errors/brainyError.js'

/**
 * Brainy Field Operators (BFO) - Our own field query system
 * Designed for performance, clarity, and patent independence
 */
export interface BrainyFieldOperators {
  // Equality operators (canonical + long-form aliases)
  eq?: any
  equals?: any
  ne?: any
  notEquals?: any

  // Comparison operators (canonical + long-form aliases)
  greaterThan?: any
  gt?: any
  greaterThanOrEqual?: any
  gte?: any
  lessThan?: any
  lt?: any
  lessThanOrEqual?: any
  lte?: any
  between?: [any, any]
  
  // Array/Set operators
  oneOf?: any[]
  in?: any[] // documented alias for oneOf
  noneOf?: any[]
  contains?: any
  excludes?: any
  hasAll?: any[]
  length?: number
  
  // Existence operators
  exists?: boolean
  missing?: boolean
  
  // Pattern operators
  matches?: string | RegExp
  startsWith?: string
  endsWith?: string
  
  // Logical operators
  allOf?: MetadataFilter[]
  anyOf?: MetadataFilter[]
  not?: MetadataFilter
}

/**
 * Metadata filter definition
 */
export interface MetadataFilter {
  [key: string]: any | BrainyFieldOperators
}

/**
 * Options for metadata filtering
 */
export interface MetadataFilterOptions {
  metadata?: MetadataFilter
  scoring?: {
    vectorWeight?: number
    metadataWeight?: number
    metadataBoosts?: Record<string, number | ((value: any, query: any) => number)>
  }
}

/**
 * Value-level operators (the keys inside `{ field: { <op>: operand } }`) — the
 * complete documented set, kept in lockstep with the `matchesQuery` switch below
 * and the metadata-index path. `in` is the documented alias for `oneOf`.
 */
const VALUE_OPERATORS = new Set<string>([
  'equals', 'eq', 'notEquals', 'ne',
  'greaterThan', 'gt', 'greaterThanOrEqual', 'gte',
  'lessThan', 'lt', 'lessThanOrEqual', 'lte',
  'between', 'oneOf', 'in', 'noneOf',
  'contains', 'excludes', 'hasAll', 'length',
  'exists', 'missing', 'matches', 'startsWith', 'endsWith'
])

/** Filter-level logical operators (siblings of field names). */
const LOGICAL_OPERATORS = new Set<string>(['allOf', 'anyOf', 'not'])

/**
 * Validate a `where` filter's operators up front, throwing a typed
 * `BrainyError('INVALID_QUERY')` on the first unrecognized operator — so a typo
 * like `{ subtype: { notIn: [...] } }` fails LOUD instead of silently matching
 * nothing (the invisible-degrade class). Called by `find()` before either the
 * index path or the in-memory matcher runs, so the throw fires even when the
 * result set is empty (the index path would otherwise return `[]` without ever
 * invoking the matcher). Nested fields use dot notation (`{ 'a.b': v }`), so a
 * field's object value carries operators, never sub-field names.
 *
 * @param filter - the user-supplied `where` clause (validated raw, before any
 *   internal field injection like visibility or `type`→`noun`).
 * @throws BrainyError('INVALID_QUERY') naming the bad operator + the valid set.
 */
export function validateWhereFilter(filter: unknown): void {
  if (!filter || typeof filter !== 'object' || Array.isArray(filter)) return
  for (const [key, value] of Object.entries(filter as Record<string, unknown>)) {
    if (LOGICAL_OPERATORS.has(key)) {
      if (key === 'not') {
        validateWhereFilter(value)
      } else if (Array.isArray(value)) {
        for (const sub of value) validateWhereFilter(sub)
      }
      continue
    }
    // A field key. Its value is a scalar/array (equality) — nothing to validate —
    // or an object of value-operators, every key of which must be recognized.
    if (value && typeof value === 'object' && !Array.isArray(value)) {
      for (const op of Object.keys(value as Record<string, unknown>)) {
        if (!VALUE_OPERATORS.has(op)) {
          throw new BrainyError(
            `Unknown filter operator "${op}" on field "${key}". Valid operators: ` +
              `${[...VALUE_OPERATORS].sort().join(', ')}. For nested fields use dot ` +
              `notation, e.g. { '${key}.subfield': value }.`,
            'INVALID_QUERY'
          )
        }
      }
    }
  }
}

/**
 * Check if a value matches a query with operators
 */
function matchesQuery(value: any, query: any): boolean {
  // Direct equality check
  if (typeof query !== 'object' || query === null || Array.isArray(query)) {
    return value === query
  }

  // Check for Brainy Field Operators (BFO)
  for (const [op, operand] of Object.entries(query)) {
    switch (op) {
      // Equality operators
      case 'equals':
      case 'eq':
        if (value !== operand) return false
        break
      case 'notEquals':
      case 'ne':
        // Special handling: if value is undefined and operand is not undefined,
        // they are not equal (so the condition passes)
        // This ensures items without a 'deleted' field match 'deleted !== true'
        if (value === operand) return false
        // If value is undefined and operand is not, they're not equal (pass)
        // If both are undefined, they're equal (fail, handled above)
        break
        
      // Comparison operators
      case 'greaterThan':
      case 'gt':
        if (typeof value !== 'number' || typeof operand !== 'number' || !(value > operand)) return false
        break
      case 'gte':
      case 'greaterThanOrEqual':
        if (typeof value !== 'number' || typeof operand !== 'number' || !(value >= operand)) return false
        break
      case 'lessThan':
      case 'lt':
        if (typeof value !== 'number' || typeof operand !== 'number' || !(value < operand)) return false
        break
      case 'lte':
      case 'lessThanOrEqual':
        if (typeof value !== 'number' || typeof operand !== 'number' || !(value <= operand)) return false
        break
      case 'between':
        if (!Array.isArray(operand) || operand.length !== 2) return false
        if (typeof value !== 'number' || !(value >= operand[0] && value <= operand[1])) return false
        break
        
      // Array/Set operators
      case 'oneOf':
      case 'in': // documented alias for oneOf
        if (!Array.isArray(operand) || !operand.includes(value)) return false
        break
      case 'noneOf':
        if (!Array.isArray(operand) || operand.includes(value)) return false
        break
      case 'contains':
        if (!Array.isArray(value) || !value.includes(operand)) return false
        break
      case 'excludes':
        if (!Array.isArray(value) || value.includes(operand)) return false
        break
      case 'hasAll':
        if (!Array.isArray(value) || !Array.isArray(operand)) return false
        for (const item of operand) {
          if (!value.includes(item)) return false
        }
        break
      case 'length':
        if (!Array.isArray(value) || value.length !== operand) return false
        break
        
      // Existence operators
      case 'exists':
        if ((value !== undefined) !== operand) return false
        break
      case 'missing':
        if ((value === undefined) !== operand) return false
        break
        
      // Pattern operators
      case 'matches':
        const regex = typeof operand === 'string' ? new RegExp(operand) : operand as RegExp
        if (!(regex instanceof RegExp) || !regex.test(String(value))) return false
        break
      case 'startsWith':
        if (typeof value !== 'string' || !value.startsWith(String(operand))) return false
        break
      case 'endsWith':
        if (typeof value !== 'string' || !value.endsWith(String(operand))) return false
        break
        
      default:
        // Unknown operator. The old behavior treated any unknown key as a
        // nested-object field name (an UNDOCUMENTED fallback — dot notation
        // `{ 'a.b': v }` is the supported nested form), which silently swallowed
        // operator typos: `{ x: { notIn: [...] } }` matched nothing instead of
        // erroring. Fail loud. find() validates the whole where clause up front
        // (validateWhereFilter), so a typo throws even on an empty result set;
        // this is the belt-and-suspenders for any matcher caller that bypasses
        // that path.
        throw new BrainyError(
          `Unknown filter operator "${op}". Valid operators: ` +
            `${[...VALUE_OPERATORS].sort().join(', ')}. For nested fields use dot ` +
            `notation, e.g. { 'address.city': 'NYC' }.`,
          'INVALID_QUERY'
        )
    }
  }

  return true
}

/**
 * Get nested value from object using dot notation
 */
function getNestedValue(obj: any, path: string): any {
  const parts = path.split('.')
  let current = obj

  for (const part of parts) {
    if (current === null || current === undefined) {
      return undefined
    }
    current = current[part]
  }

  return current
}

/**
 * Check if metadata matches the filter
 */
export function matchesMetadataFilter(metadata: any, filter: MetadataFilter): boolean {
  if (!filter || Object.keys(filter).length === 0) {
    return true
  }

  for (const [key, query] of Object.entries(filter)) {
    // Handle logical operators
    if (key === 'allOf') {
      if (!Array.isArray(query)) return false
      for (const subFilter of query) {
        if (!matchesMetadataFilter(metadata, subFilter)) return false
      }
      continue
    }

    if (key === 'anyOf') {
      if (!Array.isArray(query)) return false
      let matched = false
      for (const subFilter of query) {
        if (matchesMetadataFilter(metadata, subFilter)) {
          matched = true
          break
        }
      }
      if (!matched) return false
      continue
    }

    if (key === 'not') {
      if (matchesMetadataFilter(metadata, query)) return false
      continue
    }

    // Handle field queries
    const value = getNestedValue(metadata, key)
    if (!matchesQuery(value, query)) {
      return false
    }
  }

  return true
}

/**
 * Calculate metadata boost score
 */
export function calculateMetadataScore(
  metadata: any,
  filter: MetadataFilter,
  scoring?: MetadataFilterOptions['scoring']
): number {
  if (!scoring || !scoring.metadataBoosts) {
    return 0
  }

  let score = 0

  for (const [field, boost] of Object.entries(scoring.metadataBoosts)) {
    const value = getNestedValue(metadata, field)
    
    if (typeof boost === 'function') {
      score += boost(value, filter)
    } else if (value !== undefined) {
      // Check if the field matches the filter
      const fieldFilter = filter[field]
      if (fieldFilter && matchesQuery(value, fieldFilter)) {
        score += boost
      }
    }
  }

  return score
}

/**
 * Apply compound scoring to search results
 */
export function applyCompoundScoring<T>(
  results: SearchResult<T>[],
  filter: MetadataFilter,
  scoring?: MetadataFilterOptions['scoring']
): SearchResult<T>[] {
  if (!scoring || (!scoring.vectorWeight && !scoring.metadataWeight)) {
    return results
  }

  const vectorWeight = scoring.vectorWeight ?? 1.0
  const metadataWeight = scoring.metadataWeight ?? 0.0

  return results.map(result => {
    const metadataScore = calculateMetadataScore(result.metadata, filter, scoring)
    const combinedScore = (result.score * vectorWeight) + (metadataScore * metadataWeight)
    
    return {
      ...result,
      score: combinedScore
    }
  }).sort((a, b) => b.score - a.score) // Re-sort by combined score
}

/**
 * Filter search results by metadata
 */
export function filterSearchResultsByMetadata<T>(
  results: SearchResult<T>[],
  filter: MetadataFilter
): SearchResult<T>[] {
  if (!filter || Object.keys(filter).length === 0) {
    return results
  }

  return results.filter(result => 
    matchesMetadataFilter(result.metadata, filter)
  )
}

/**
 * Filter nouns by metadata before search
 * Takes HNSWNounWithMetadata which includes metadata field
 */
export function filterNounsByMetadata(
  nouns: HNSWNounWithMetadata[],
  filter: MetadataFilter
): HNSWNounWithMetadata[] {
  if (!filter || Object.keys(filter).length === 0) {
    return nouns
  }

  return nouns.filter(noun =>
    matchesMetadataFilter(noun.metadata, filter)
  )
}

/**
 * Aggregate search results for faceted search
 */
export interface FacetConfig {
  field: string
  limit?: number
}

export interface FacetResult {
  [value: string]: number
}

export interface AggregationResult<T> {
  results: SearchResult<T>[]
  facets: Record<string, FacetResult>
}

export function aggregateSearchResults<T>(
  results: SearchResult<T>[],
  facets: Record<string, FacetConfig>
): AggregationResult<T> {
  const facetResults: Record<string, FacetResult> = {}

  for (const [facetName, config] of Object.entries(facets)) {
    const counts: Record<string, number> = {}

    for (const result of results) {
      const value = getNestedValue(result.metadata, config.field)
      
      if (value !== undefined) {
        const key = String(value)
        counts[key] = (counts[key] || 0) + 1
      }
    }

    // Sort by count and apply limit
    const sorted = Object.entries(counts)
      .sort((a, b) => b[1] - a[1])
      .slice(0, config.limit || 10)

    facetResults[facetName] = Object.fromEntries(sorted)
  }

  return {
    results,
    facets: facetResults
  }
}