brainy/tests/unit/fieldTypeInference.test.ts

/**
 * Field Type Inference Unit Tests
 *
 * Comprehensive tests for production-ready value-based type detection
 */

import { describe, it, expect, beforeEach, afterEach } from 'vitest'
import { FieldTypeInference, FieldType } from '../../src/utils/fieldTypeInference.js'
import { MemoryStorage } from '../../src/storage/adapters/memoryStorage.js'

describe('FieldTypeInference', () => {
  let storage: MemoryStorage
  let inference: FieldTypeInference

  beforeEach(async () => {
    storage = new MemoryStorage()
    await storage.init()
    inference = new FieldTypeInference(storage)
  })

  afterEach(async () => {
    await storage.clear()
  })

  // ============================================================================
  // Boolean Detection
  // ============================================================================

  describe('Boolean Detection', () => {
    it('should detect true/false values', async () => {
      const result = await inference.inferFieldType('active', [true, false, true])

      expect(result.inferredType).toBe(FieldType.BOOLEAN)
      expect(result.confidence).toBe(1.0)
      expect(result.sampleSize).toBe(3)
    })

    it('should detect "true"/"false" strings', async () => {
      const result = await inference.inferFieldType('flag', ['true', 'false', 'true'])

      expect(result.inferredType).toBe(FieldType.BOOLEAN)
      expect(result.confidence).toBe(1.0)
    })

    it('should detect 1/0 as boolean', async () => {
      const result = await inference.inferFieldType('enabled', ['1', '0', '1', '0'])

      expect(result.inferredType).toBe(FieldType.BOOLEAN)
      expect(result.confidence).toBe(1.0)
    })

    it('should detect yes/no as boolean', async () => {
      const result = await inference.inferFieldType('confirmed', ['yes', 'no', 'yes'])

      expect(result.inferredType).toBe(FieldType.BOOLEAN)
      expect(result.confidence).toBe(1.0)
    })
  })

  // ============================================================================
  // Integer Detection
  // ============================================================================

  describe('Integer Detection', () => {
    it('should detect integer numbers', async () => {
      const result = await inference.inferFieldType('count', [1, 2, 3, 42, 100])

      expect(result.inferredType).toBe(FieldType.INTEGER)
      expect(result.confidence).toBe(1.0)
    })

    it('should detect integer strings', async () => {
      const result = await inference.inferFieldType('age', ['25', '30', '45'])

      expect(result.inferredType).toBe(FieldType.INTEGER)
      expect(result.confidence).toBe(1.0)
    })

    it('should detect negative integers', async () => {
      const result = await inference.inferFieldType('balance', [-100, -50, 0, 50, 100])

      expect(result.inferredType).toBe(FieldType.INTEGER)
      expect(result.confidence).toBe(1.0)
    })
  })

  // ============================================================================
  // Unix Timestamp Detection (The Key Feature!)
  // ============================================================================

  describe('Unix Timestamp Detection', () => {
    it('should detect Unix timestamps in milliseconds', async () => {
      // January 16, 2025 timestamps
      const timestamps = [1705420800000, 1705420860000, 1705420920000]
      const result = await inference.inferFieldType('extractedAt', timestamps)

      expect(result.inferredType).toBe(FieldType.TIMESTAMP_MS)
      expect(result.confidence).toBe(0.95)
      expect(result.metadata?.precision).toBe('milliseconds')
      expect(result.metadata?.bucketSize).toBe(60000) // 1 minute buckets
      expect(result.metadata?.format).toBe('Unix timestamp')
    })

    it('should detect Unix timestamps in seconds', async () => {
      // January 16, 2025 timestamps (seconds)
      const timestamps = [1705420800, 1705420860, 1705420920]
      const result = await inference.inferFieldType('created', timestamps)

      expect(result.inferredType).toBe(FieldType.TIMESTAMP_S)
      expect(result.confidence).toBe(0.95)
      expect(result.metadata?.precision).toBe('seconds')
      expect(result.metadata?.bucketSize).toBe(60000)
    })

    it('should detect timestamps with field name irrelevant', async () => {
      // The field is called "randomFieldName" but VALUES are timestamps
      const timestamps = [1705420800000, 1705420860000]
      const result = await inference.inferFieldType('randomFieldName', timestamps)

      expect(result.inferredType).toBe(FieldType.TIMESTAMP_MS)
      expect(result.confidence).toBe(0.95)
    })

    it('should NOT detect non-timestamp integers as timestamps', async () => {
      // These are just regular integers, not in timestamp range
      const values = [1, 2, 3, 4, 5]
      const result = await inference.inferFieldType('count', values)

      expect(result.inferredType).toBe(FieldType.INTEGER)
      expect(result.confidence).toBe(1.0)
    })

    it('should handle mixed timestamp ranges (seconds and milliseconds)', async () => {
      // All milliseconds
      const values = [1705420800000, 1705420860000, 1705420920000]
      const result = await inference.inferFieldType('time', values)

      expect(result.inferredType).toBe(FieldType.TIMESTAMP_MS)
    })
  })

  // ============================================================================
  // Float Detection
  // ============================================================================

  describe('Float Detection', () => {
    it('should detect float numbers', async () => {
      const result = await inference.inferFieldType('price', [19.99, 29.99, 39.99])

      expect(result.inferredType).toBe(FieldType.FLOAT)
      expect(result.confidence).toBe(1.0)
    })

    it('should detect float strings', async () => {
      const result = await inference.inferFieldType('temperature', ['98.6', '100.2', '99.1'])

      expect(result.inferredType).toBe(FieldType.FLOAT)
      expect(result.confidence).toBe(1.0)
    })

    it('should detect scientific notation floats', async () => {
      // Use values that result in actual floats (not integers)
      const result = await inference.inferFieldType('distance', [1.23e-5, 4.56e-3, 7.89e1])

      expect(result.inferredType).toBe(FieldType.FLOAT)
    })
  })

  // ============================================================================
  // ISO 8601 Date/Datetime Detection
  // ============================================================================

  describe('ISO 8601 Detection', () => {
    it('should detect ISO 8601 dates (YYYY-MM-DD)', async () => {
      const dates = ['2025-01-16', '2025-01-17', '2025-01-18']
      const result = await inference.inferFieldType('date', dates)

      expect(result.inferredType).toBe(FieldType.DATE_ISO8601)
      expect(result.confidence).toBe(0.95)
      expect(result.metadata?.precision).toBe('date')
      expect(result.metadata?.bucketSize).toBe(86400000) // 1 day
    })

    it('should detect ISO 8601 datetimes with time', async () => {
      const datetimes = [
        '2025-01-16T10:30:00Z',
        '2025-01-16T10:31:00Z',
        '2025-01-16T10:32:00Z'
      ]
      const result = await inference.inferFieldType('createdAt', datetimes)

      expect(result.inferredType).toBe(FieldType.DATETIME_ISO8601)
      expect(result.confidence).toBe(0.95)
      expect(result.metadata?.precision).toBe('datetime')
      expect(result.metadata?.bucketSize).toBe(60000) // 1 minute
    })

    it('should detect ISO 8601 with timezone offset', async () => {
      const datetimes = [
        '2025-01-16T10:30:00+05:30',
        '2025-01-16T10:31:00+05:30'
      ]
      const result = await inference.inferFieldType('timestamp', datetimes)

      expect(result.inferredType).toBe(FieldType.DATETIME_ISO8601)
    })

    it('should detect ISO 8601 with milliseconds', async () => {
      const datetimes = [
        '2025-01-16T10:30:00.123Z',
        '2025-01-16T10:31:00.456Z'
      ]
      const result = await inference.inferFieldType('precise', datetimes)

      expect(result.inferredType).toBe(FieldType.DATETIME_ISO8601)
    })
  })

  // ============================================================================
  // UUID Detection
  // ============================================================================

  describe('UUID Detection', () => {
    it('should detect UUIDs', async () => {
      const uuids = [
        'f47ac10b-58cc-4372-a567-0e02b2c3d479',
        'e8a7c924-1234-5678-9abc-def012345678',
        '123e4567-e89b-12d3-a456-426614174000'
      ]
      const result = await inference.inferFieldType('id', uuids)

      expect(result.inferredType).toBe(FieldType.UUID)
      expect(result.confidence).toBe(1.0)
    })

    it('should NOT detect non-UUID strings as UUIDs', async () => {
      const values = ['not-a-uuid', 'random-string', 'abc123']
      const result = await inference.inferFieldType('text', values)

      expect(result.inferredType).not.toBe(FieldType.UUID)
      expect(result.inferredType).toBe(FieldType.STRING)
    })
  })

  // ============================================================================
  // Array/Object Detection
  // ============================================================================

  describe('Array and Object Detection', () => {
    it('should detect arrays', async () => {
      const arrays = [[1, 2, 3], [4, 5, 6], [7, 8, 9]]
      const result = await inference.inferFieldType('numbers', arrays)

      expect(result.inferredType).toBe(FieldType.ARRAY)
      expect(result.confidence).toBe(1.0)
    })

    it('should detect objects', async () => {
      const objects = [{ a: 1 }, { b: 2 }, { c: 3 }]
      const result = await inference.inferFieldType('data', objects)

      expect(result.inferredType).toBe(FieldType.OBJECT)
      expect(result.confidence).toBe(1.0)
    })
  })

  // ============================================================================
  // String Default
  // ============================================================================

  describe('String Detection', () => {
    it('should default to string for text values', async () => {
      const values = ['hello', 'world', 'foo', 'bar']
      const result = await inference.inferFieldType('name', values)

      expect(result.inferredType).toBe(FieldType.STRING)
      expect(result.confidence).toBe(0.8)
    })

    it('should handle empty strings', async () => {
      const values = ['', '', '']
      const result = await inference.inferFieldType('empty', values)

      expect(result.inferredType).toBe(FieldType.STRING)
    })

    it('should handle mixed content as string', async () => {
      const values = ['123abc', 'def456', 'mixed!@#']
      const result = await inference.inferFieldType('mixed', values)

      expect(result.inferredType).toBe(FieldType.STRING)
    })
  })

  // ============================================================================
  // Cache Functionality
  // ============================================================================

  describe('Cache Functionality', () => {
    it('should cache inferred types for fast lookups', async () => {
      const values = [1705420800000, 1705420860000]

      // First call: analyze values (cache miss)
      const result1 = await inference.inferFieldType('extractedAt', values)

      // Second call: cache hit (should return same result instantly)
      const result2 = await inference.inferFieldType('extractedAt', values)

      // Verify cache returns identical results
      expect(result1.inferredType).toBe(result2.inferredType)
      expect(result1.confidence).toBe(result2.confidence)
      expect(result1.sampleSize).toBe(result2.sampleSize)
    })

    it('should persist cache to storage', async () => {
      // Use 50+ samples to meet MIN_SAMPLE_SIZE_FOR_CONFIDENCE threshold
      const values = Array.from({ length: 60 }, (_, i) => 1705420800000 + i * 60000)
      await inference.inferFieldType('extractedAt', values)

      // Create new instance (should load from storage)
      const newInference = new FieldTypeInference(storage)
      const result = await newInference.inferFieldType('extractedAt', [])

      expect(result.inferredType).toBe(FieldType.TIMESTAMP_MS)
    })

    it('should clear cache', async () => {
      const values = [1, 2, 3]
      await inference.inferFieldType('count', values)

      await inference.clearCache('count')

      const stats = inference.getCacheStats()
      expect(stats.size).toBe(0)
    })
  })

  // ============================================================================
  // Progressive Refinement
  // ============================================================================

  describe('Progressive Refinement', () => {
    it('should refine type inference with more samples', async () => {
      // Start with few samples (low confidence)
      const fewSamples = [1, 2, 3]
      const result1 = await inference.inferFieldType('value', fewSamples)

      expect(result1.sampleSize).toBe(3)

      // Refine with more samples
      const moreSamples = Array.from({ length: 100 }, (_, i) => i + 1)
      await inference.refineTypeInference('value', moreSamples)

      // Should have updated sample size
      const result2 = await inference.inferFieldType('value', [])
      expect(result2.sampleSize).toBeGreaterThan(3)
    })
  })

  // ============================================================================
  // Edge Cases
  // ============================================================================

  describe('Edge Cases', () => {
    it('should handle empty value arrays', async () => {
      const result = await inference.inferFieldType('empty', [])

      expect(result.inferredType).toBe(FieldType.STRING)
      expect(result.confidence).toBe(0.5)
      expect(result.sampleSize).toBe(0)
    })

    it('should handle null/undefined values', async () => {
      const values = [null, undefined, null]
      const result = await inference.inferFieldType('nullable', values)

      expect(result.inferredType).toBe(FieldType.STRING)
      expect(result.confidence).toBe(0.5)
    })

    it('should handle very large sample sizes', async () => {
      const values = Array.from({ length: 10000 }, (_, i) => i)
      const result = await inference.inferFieldType('large', values)

      expect(result.inferredType).toBe(FieldType.INTEGER)
      expect(result.sampleSize).toBe(100) // Should sample only 100
    })
  })

  // ============================================================================
  // Utility Methods
  // ============================================================================

  describe('Utility Methods', () => {
    it('should identify temporal types', () => {
      expect(inference.isTemporal(FieldType.TIMESTAMP_MS)).toBe(true)
      expect(inference.isTemporal(FieldType.TIMESTAMP_S)).toBe(true)
      expect(inference.isTemporal(FieldType.DATE_ISO8601)).toBe(true)
      expect(inference.isTemporal(FieldType.DATETIME_ISO8601)).toBe(true)
      expect(inference.isTemporal(FieldType.STRING)).toBe(false)
      expect(inference.isTemporal(FieldType.INTEGER)).toBe(false)
    })

    it('should get bucket size for temporal types', async () => {
      const values = [1705420800000, 1705420860000]
      const result = await inference.inferFieldType('extractedAt', values)

      const bucketSize = inference.getBucketSize(result)
      expect(bucketSize).toBe(60000) // 1 minute
    })

    it('should get cache statistics', async () => {
      const values1 = [1705420800000, 1705420860000]
      const values2 = [1, 2, 3]
      const values3 = ['hello', 'world']

      await inference.inferFieldType('timestamp', values1)
      await inference.inferFieldType('count', values2)
      await inference.inferFieldType('name', values3)

      const stats = inference.getCacheStats()
      expect(stats.size).toBe(3)
      expect(stats.temporalFields).toBe(1)
      expect(stats.nonTemporalFields).toBe(2)
      expect(stats.fields).toContain('timestamp')
      expect(stats.fields).toContain('count')
      expect(stats.fields).toContain('name')
    })
  })

  // ============================================================================
  // Real-World Scenarios
  // ============================================================================

  describe('Real-World Scenarios', () => {
    it('should handle extractedAt field (the bug that started it all!)', async () => {
      // This is the exact scenario that caused 275k files!
      const extractedAtValues = [
        1705420800123,
        1705420800456,
        1705420800789,
        1705420801012
      ]

      const result = await inference.inferFieldType('extractedAt', extractedAtValues)

      expect(result.inferredType).toBe(FieldType.TIMESTAMP_MS)
      expect(result.confidence).toBe(0.95)
      expect(result.metadata?.bucketSize).toBe(60000)

      // With bucketing, all these should normalize to same bucket
      // This prevents file explosion!
    })

    it('should handle importedAt, uploadedAt, createdAt, etc.', async () => {
      const fields = ['importedAt', 'uploadedAt', 'createdAt', 'modifiedAt']

      for (const field of fields) {
        const values = [1705420800000, 1705420860000]
        const result = await inference.inferFieldType(field, values)

        expect(result.inferredType).toBe(FieldType.TIMESTAMP_MS)
        expect(result.metadata?.bucketSize).toBe(60000)
      }
    })

    it('should handle non-temporal fields with "at" suffix', async () => {
      // Not all fields ending in "at" are timestamps!
      const values = ['cat', 'bat', 'hat', 'rat']
      const result = await inference.inferFieldType('animal', values)

      expect(result.inferredType).toBe(FieldType.STRING)
      expect(result.inferredType).not.toBe(FieldType.TIMESTAMP_MS)
    })
  })
})