David Snelling
364360d447
The __words__ keyword index stores 50-5000 entries per entity (one per word), which inflated avg entries/entity well above the corruption threshold of 100. This caused: 1. validateConsistency() to falsely detect corruption on every startup, triggering unnecessary clearAllIndexData() + rebuild() cycles 2. getStats() to log false "Metadata index may be corrupted" warnings and report inflated totalEntries/totalIds stats Both methods now skip __words__ when counting, so stats and health checks reflect metadata fields only (noun, type, createdAt, etc.). Keyword search is unaffected since the __words__ field index itself is not modified.
547 lines
16 KiB
TypeScript
547 lines
16 KiB
TypeScript
/**
|
|
* Field Type Inference System
|
|
*
|
|
* Production-ready value-based type detection inspired by DuckDB, Arrow, and Snowflake.
|
|
*
|
|
* Replaces unreliable pattern matching with robust value analysis:
|
|
* - Samples actual data values (not field names)
|
|
* - Persistent caching for O(1) lookups at billion scale
|
|
* - Progressive refinement as more data arrives
|
|
* - Zero configuration required
|
|
*
|
|
* Performance:
|
|
* - Cache hit: 0.1-0.5ms (O(1))
|
|
* - Cache miss: 5-10ms (analyze 100 samples)
|
|
* - Accuracy: 95%+ (vs 70% with pattern matching)
|
|
* - Memory: ~500 bytes per field
|
|
*
|
|
* Architecture:
|
|
* 1. Check in-memory cache (hot path)
|
|
* 2. Check persistent storage (_system/)
|
|
* 3. Analyze values if cache miss
|
|
* 4. Store result for future queries
|
|
*/
|
|
|
|
import { StorageAdapter } from '../coreTypes.js'
|
|
import { prodLog } from './logger.js'
|
|
|
|
/**
|
|
* Field type enumeration
|
|
* Ordered from most to least specific (DuckDB-inspired)
|
|
*/
|
|
export enum FieldType {
|
|
// Temporal types (high priority - the whole point of this system!)
|
|
TIMESTAMP_MS = 'timestamp_ms', // Unix timestamp in milliseconds
|
|
TIMESTAMP_S = 'timestamp_s', // Unix timestamp in seconds
|
|
DATE_ISO8601 = 'date_iso8601', // ISO 8601 date string (YYYY-MM-DD)
|
|
DATETIME_ISO8601 = 'datetime_iso8601', // ISO 8601 datetime string
|
|
|
|
// Numeric types
|
|
BOOLEAN = 'boolean',
|
|
INTEGER = 'integer',
|
|
FLOAT = 'float',
|
|
|
|
// String types
|
|
UUID = 'uuid',
|
|
STRING = 'string',
|
|
|
|
// Complex types
|
|
ARRAY = 'array',
|
|
OBJECT = 'object'
|
|
}
|
|
|
|
/**
|
|
* Field type information with metadata
|
|
*/
|
|
export interface FieldTypeInfo {
|
|
field: string
|
|
inferredType: FieldType
|
|
confidence: number // 0-1 confidence score
|
|
sampleSize: number // Number of values analyzed
|
|
lastUpdated: number // Timestamp of last analysis
|
|
detectionMethod: 'value' // Always 'value' (no fallbacks!)
|
|
metadata?: {
|
|
format?: string // e.g., "Unix timestamp", "ISO 8601"
|
|
precision?: string // e.g., "milliseconds", "seconds"
|
|
bucketSize?: number // For temporal fields (60000 = 1 minute)
|
|
minValue?: number // Value range stats
|
|
maxValue?: number
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Field Type Inference System
|
|
*
|
|
* Infers data types by analyzing actual values, not field names.
|
|
* Maintains persistent cache for billion-scale performance.
|
|
*/
|
|
export class FieldTypeInference {
|
|
private storage: StorageAdapter
|
|
private typeCache: Map<string, FieldTypeInfo>
|
|
private readonly SAMPLE_SIZE = 100 // Analyze first 100 values
|
|
private readonly CACHE_STORAGE_PREFIX = '__field_type_cache__'
|
|
|
|
// Temporal detection constants
|
|
private readonly MIN_TIMESTAMP_S = 946684800 // 2000-01-01 in seconds
|
|
private readonly MAX_TIMESTAMP_S = 4102444800 // 2100-01-01 in seconds
|
|
private readonly MIN_TIMESTAMP_MS = this.MIN_TIMESTAMP_S * 1000
|
|
private readonly MAX_TIMESTAMP_MS = this.MAX_TIMESTAMP_S * 1000
|
|
|
|
// Cache freshness thresholds
|
|
private readonly CACHE_AGE_THRESHOLD = 24 * 60 * 60 * 1000 // 24 hours
|
|
private readonly MIN_SAMPLE_SIZE_FOR_CONFIDENCE = 50
|
|
|
|
constructor(storage: StorageAdapter) {
|
|
this.storage = storage
|
|
this.typeCache = new Map()
|
|
}
|
|
|
|
/**
|
|
* THE ONE FUNCTION: Infer field type from values
|
|
*
|
|
* Three-phase approach for billion-scale performance:
|
|
* 1. Check in-memory cache (O(1), <1ms)
|
|
* 2. Check persistent storage (O(1), ~1-2ms)
|
|
* 3. Analyze values (O(n), ~5-10ms for 100 samples)
|
|
*
|
|
* @param field Field name
|
|
* @param values Sample values to analyze (provide 1-100+ values)
|
|
* @returns Field type information with metadata
|
|
*/
|
|
async inferFieldType(field: string, values: any[]): Promise<FieldTypeInfo> {
|
|
// Phase 1: Check in-memory cache (hot path)
|
|
const cachedInMemory = this.typeCache.get(field)
|
|
if (cachedInMemory && this.isCacheFresh(cachedInMemory)) {
|
|
return cachedInMemory
|
|
}
|
|
|
|
// Phase 2: Check persistent storage
|
|
const cachedInStorage = await this.loadFromStorage(field)
|
|
if (cachedInStorage && this.isCacheFresh(cachedInStorage)) {
|
|
// Populate in-memory cache
|
|
this.typeCache.set(field, cachedInStorage)
|
|
return cachedInStorage
|
|
}
|
|
|
|
// Phase 3: Analyze values (cache miss)
|
|
const typeInfo = await this.analyzeValues(field, values)
|
|
|
|
// Store in both caches
|
|
await this.saveToCache(field, typeInfo)
|
|
|
|
return typeInfo
|
|
}
|
|
|
|
/**
|
|
* Analyze values to determine field type
|
|
*
|
|
* Uses DuckDB-inspired type detection order:
|
|
* BOOLEAN → INTEGER → FLOAT → DATE → TIMESTAMP → UUID → STRING
|
|
*
|
|
* No fallbacks - pure value-based detection
|
|
*/
|
|
private async analyzeValues(field: string, values: any[]): Promise<FieldTypeInfo> {
|
|
// Filter null/undefined values
|
|
const validValues = values.filter(v => v !== null && v !== undefined)
|
|
|
|
if (validValues.length === 0) {
|
|
return this.createTypeInfo(field, FieldType.STRING, 0.5, 0, 'No valid values to analyze')
|
|
}
|
|
|
|
const sampleSize = Math.min(validValues.length, this.SAMPLE_SIZE)
|
|
const samples = validValues.slice(0, sampleSize)
|
|
|
|
// Type detection in order from most to least specific
|
|
|
|
// 1. Boolean detection
|
|
if (this.looksLikeBoolean(samples)) {
|
|
return this.createTypeInfo(field, FieldType.BOOLEAN, 1.0, sampleSize, 'Boolean values detected')
|
|
}
|
|
|
|
// 2. Integer detection (includes Unix timestamp detection)
|
|
if (this.looksLikeInteger(samples)) {
|
|
// Check if it's a Unix timestamp
|
|
const timestampInfo = this.detectUnixTimestamp(samples)
|
|
if (timestampInfo) {
|
|
return this.createTypeInfo(
|
|
field,
|
|
timestampInfo.type,
|
|
0.95,
|
|
sampleSize,
|
|
timestampInfo.format,
|
|
{
|
|
precision: timestampInfo.precision,
|
|
bucketSize: 60000, // 1 minute buckets
|
|
minValue: timestampInfo.minValue,
|
|
maxValue: timestampInfo.maxValue
|
|
}
|
|
)
|
|
}
|
|
|
|
return this.createTypeInfo(field, FieldType.INTEGER, 1.0, sampleSize, 'Integer values detected')
|
|
}
|
|
|
|
// 3. Float detection
|
|
if (this.looksLikeFloat(samples)) {
|
|
return this.createTypeInfo(field, FieldType.FLOAT, 1.0, sampleSize, 'Float values detected')
|
|
}
|
|
|
|
// 4. ISO 8601 date/datetime detection
|
|
const iso8601Info = this.detectISO8601(samples)
|
|
if (iso8601Info) {
|
|
return this.createTypeInfo(
|
|
field,
|
|
iso8601Info.type,
|
|
0.95,
|
|
sampleSize,
|
|
'ISO 8601',
|
|
{
|
|
bucketSize: iso8601Info.bucketSize,
|
|
precision: iso8601Info.hasTime ? 'datetime' : 'date'
|
|
}
|
|
)
|
|
}
|
|
|
|
// 5. UUID detection
|
|
if (this.looksLikeUUID(samples)) {
|
|
return this.createTypeInfo(field, FieldType.UUID, 1.0, sampleSize, 'UUID values detected')
|
|
}
|
|
|
|
// 6. Array detection
|
|
if (samples.every(v => Array.isArray(v))) {
|
|
return this.createTypeInfo(field, FieldType.ARRAY, 1.0, sampleSize, 'Array values detected')
|
|
}
|
|
|
|
// 7. Object detection
|
|
if (samples.every(v => typeof v === 'object' && v !== null && !Array.isArray(v))) {
|
|
return this.createTypeInfo(field, FieldType.OBJECT, 1.0, sampleSize, 'Object values detected')
|
|
}
|
|
|
|
// 8. Default to string
|
|
return this.createTypeInfo(field, FieldType.STRING, 0.8, sampleSize, 'Default string type')
|
|
}
|
|
|
|
// ============================================================================
|
|
// Value Analysis Heuristics (DuckDB-inspired)
|
|
// ============================================================================
|
|
|
|
/**
|
|
* Check if values look like booleans
|
|
*/
|
|
private looksLikeBoolean(samples: any[]): boolean {
|
|
const validBooleans = new Set([
|
|
'true', 'false',
|
|
'1', '0',
|
|
'yes', 'no',
|
|
't', 'f',
|
|
'y', 'n'
|
|
])
|
|
|
|
return samples.every(v => {
|
|
if (typeof v === 'boolean') return true
|
|
const str = String(v).toLowerCase().trim()
|
|
return validBooleans.has(str)
|
|
})
|
|
}
|
|
|
|
/**
|
|
* Check if values look like integers
|
|
*/
|
|
private looksLikeInteger(samples: any[]): boolean {
|
|
return samples.every(v => {
|
|
if (typeof v === 'number' && Number.isInteger(v)) return true
|
|
if (typeof v === 'string') {
|
|
return /^-?\d+$/.test(v.trim())
|
|
}
|
|
return false
|
|
})
|
|
}
|
|
|
|
/**
|
|
* Check if values look like floats
|
|
*/
|
|
private looksLikeFloat(samples: any[]): boolean {
|
|
return samples.every(v => {
|
|
if (typeof v === 'number') return true
|
|
if (typeof v === 'string') {
|
|
return /^-?\d+\.?\d*$/.test(v.trim())
|
|
}
|
|
return false
|
|
})
|
|
}
|
|
|
|
/**
|
|
* Detect Unix timestamp (milliseconds or seconds)
|
|
*
|
|
* Unix timestamp range: 2000-01-01 to 2100-01-01
|
|
* - Seconds: 946,684,800 to 4,102,444,800
|
|
* - Milliseconds: 946,684,800,000 to 4,102,444,800,000
|
|
*/
|
|
private detectUnixTimestamp(samples: any[]): {
|
|
type: FieldType
|
|
format: string
|
|
precision: string
|
|
minValue: number
|
|
maxValue: number
|
|
} | null {
|
|
const numbers = samples.map(v => Number(v))
|
|
|
|
// All values must be valid numbers
|
|
if (numbers.some(n => isNaN(n))) return null
|
|
|
|
// Check if values fall in Unix timestamp range
|
|
const allInSecondsRange = numbers.every(
|
|
n => n >= this.MIN_TIMESTAMP_S && n <= this.MAX_TIMESTAMP_S
|
|
)
|
|
const allInMillisecondsRange = numbers.every(
|
|
n => n >= this.MIN_TIMESTAMP_MS && n <= this.MAX_TIMESTAMP_MS
|
|
)
|
|
|
|
if (!allInSecondsRange && !allInMillisecondsRange) return null
|
|
|
|
// Determine precision based on magnitude
|
|
const avgValue = numbers.reduce((sum, n) => sum + n, 0) / numbers.length
|
|
const isMilliseconds = avgValue > this.MAX_TIMESTAMP_S
|
|
|
|
const minValue = Math.min(...numbers)
|
|
const maxValue = Math.max(...numbers)
|
|
|
|
if (isMilliseconds) {
|
|
return {
|
|
type: FieldType.TIMESTAMP_MS,
|
|
format: 'Unix timestamp',
|
|
precision: 'milliseconds',
|
|
minValue,
|
|
maxValue
|
|
}
|
|
} else {
|
|
return {
|
|
type: FieldType.TIMESTAMP_S,
|
|
format: 'Unix timestamp',
|
|
precision: 'seconds',
|
|
minValue,
|
|
maxValue
|
|
}
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Detect ISO 8601 dates and datetimes
|
|
*
|
|
* Formats supported:
|
|
* - Date: YYYY-MM-DD
|
|
* - Datetime: YYYY-MM-DDTHH:MM:SS[.mmm][Z|±HH:MM]
|
|
*/
|
|
private detectISO8601(samples: any[]): {
|
|
type: FieldType
|
|
hasTime: boolean
|
|
bucketSize: number
|
|
} | null {
|
|
// ISO 8601 patterns
|
|
const datePattern = /^\d{4}-\d{2}-\d{2}$/
|
|
const datetimePattern = /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(\.\d+)?(Z|[+-]\d{2}:\d{2})?$/
|
|
|
|
let hasTime = false
|
|
const allMatch = samples.every(v => {
|
|
if (typeof v !== 'string') return false
|
|
const str = v.trim()
|
|
|
|
if (datetimePattern.test(str)) {
|
|
hasTime = true
|
|
return true
|
|
}
|
|
|
|
return datePattern.test(str)
|
|
})
|
|
|
|
if (!allMatch) return null
|
|
|
|
return {
|
|
type: hasTime ? FieldType.DATETIME_ISO8601 : FieldType.DATE_ISO8601,
|
|
hasTime,
|
|
bucketSize: hasTime ? 60000 : 86400000 // 1 minute for datetime, 1 day for date
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Check if values look like UUIDs
|
|
*/
|
|
private looksLikeUUID(samples: any[]): boolean {
|
|
const uuidPattern = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i
|
|
|
|
return samples.every(v => {
|
|
if (typeof v !== 'string') return false
|
|
return uuidPattern.test(v.trim())
|
|
})
|
|
}
|
|
|
|
// ============================================================================
|
|
// Cache Management
|
|
// ============================================================================
|
|
|
|
/**
|
|
* Load type info from persistent storage
|
|
*/
|
|
private async loadFromStorage(field: string): Promise<FieldTypeInfo | null> {
|
|
try {
|
|
const cacheKey = `${this.CACHE_STORAGE_PREFIX}${field}`
|
|
const data = await this.storage.getMetadata(cacheKey)
|
|
|
|
if (data) {
|
|
// Double cast for type boundary crossing
|
|
return data as unknown as FieldTypeInfo
|
|
}
|
|
} catch (error) {
|
|
prodLog.debug(`Failed to load field type cache for '${field}':`, error)
|
|
}
|
|
|
|
return null
|
|
}
|
|
|
|
/**
|
|
* Save type info to both in-memory and persistent cache
|
|
*/
|
|
private async saveToCache(field: string, typeInfo: FieldTypeInfo): Promise<void> {
|
|
// Save to in-memory cache
|
|
this.typeCache.set(field, typeInfo)
|
|
|
|
// Save to persistent storage (async, non-blocking)
|
|
// Add required 'noun' property for NounMetadata
|
|
const cacheKey = `${this.CACHE_STORAGE_PREFIX}${field}`
|
|
const metadataObj = {
|
|
noun: 'FieldTypeCache',
|
|
...typeInfo
|
|
}
|
|
await this.storage.saveMetadata(cacheKey, metadataObj as any).catch(error => {
|
|
prodLog.warn(`Failed to save field type cache for '${field}':`, error)
|
|
})
|
|
}
|
|
|
|
/**
|
|
* Check if cached type info is still fresh
|
|
*
|
|
* Cache is considered fresh if:
|
|
* - High confidence (>= 0.9)
|
|
* - Updated within last 24 hours
|
|
* - Analyzed at least 50 samples
|
|
*/
|
|
private isCacheFresh(typeInfo: FieldTypeInfo): boolean {
|
|
const age = Date.now() - typeInfo.lastUpdated
|
|
|
|
return (
|
|
typeInfo.confidence >= 0.9 &&
|
|
age < this.CACHE_AGE_THRESHOLD &&
|
|
typeInfo.sampleSize >= this.MIN_SAMPLE_SIZE_FOR_CONFIDENCE
|
|
)
|
|
}
|
|
|
|
/**
|
|
* Progressive refinement: Update type inference as more data arrives
|
|
*
|
|
* This is called when we have more samples and want to improve confidence.
|
|
* Only updates cache if confidence improves.
|
|
*/
|
|
async refineTypeInference(field: string, newValues: any[]): Promise<void> {
|
|
const current = await this.loadFromStorage(field)
|
|
if (!current) return
|
|
|
|
// Analyze with new samples
|
|
const refined = await this.analyzeValues(field, newValues)
|
|
|
|
// Only update if confidence improved or sample size increased significantly
|
|
if (
|
|
refined.confidence > current.confidence ||
|
|
refined.sampleSize > current.sampleSize * 2
|
|
) {
|
|
await this.saveToCache(field, refined)
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Check if a field type is temporal
|
|
*/
|
|
isTemporal(type: FieldType): boolean {
|
|
return [
|
|
FieldType.TIMESTAMP_MS,
|
|
FieldType.TIMESTAMP_S,
|
|
FieldType.DATE_ISO8601,
|
|
FieldType.DATETIME_ISO8601
|
|
].includes(type)
|
|
}
|
|
|
|
/**
|
|
* Get bucket size for a temporal field type
|
|
*/
|
|
getBucketSize(typeInfo: FieldTypeInfo): number {
|
|
if (!this.isTemporal(typeInfo.inferredType)) {
|
|
return 0
|
|
}
|
|
|
|
return typeInfo.metadata?.bucketSize || 60000 // Default: 1 minute
|
|
}
|
|
|
|
/**
|
|
* Clear cache for a field (useful for testing)
|
|
*/
|
|
async clearCache(field?: string): Promise<void> {
|
|
if (field) {
|
|
this.typeCache.delete(field)
|
|
const cacheKey = `${this.CACHE_STORAGE_PREFIX}${field}`
|
|
// null signals deletion to storage adapter
|
|
await this.storage.saveMetadata(cacheKey, null as any)
|
|
} else {
|
|
this.typeCache.clear()
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Get cache statistics for monitoring
|
|
*/
|
|
getCacheStats(): {
|
|
size: number
|
|
fields: string[]
|
|
temporalFields: number
|
|
nonTemporalFields: number
|
|
} {
|
|
const fields = Array.from(this.typeCache.keys())
|
|
const temporalFields = Array.from(this.typeCache.values()).filter(info =>
|
|
this.isTemporal(info.inferredType)
|
|
).length
|
|
|
|
return {
|
|
size: this.typeCache.size,
|
|
fields,
|
|
temporalFields,
|
|
nonTemporalFields: this.typeCache.size - temporalFields
|
|
}
|
|
}
|
|
|
|
// ============================================================================
|
|
// Helper Methods
|
|
// ============================================================================
|
|
|
|
/**
|
|
* Create a FieldTypeInfo object
|
|
*/
|
|
private createTypeInfo(
|
|
field: string,
|
|
type: FieldType,
|
|
confidence: number,
|
|
sampleSize: number,
|
|
format: string,
|
|
extraMetadata?: Record<string, any>
|
|
): FieldTypeInfo {
|
|
return {
|
|
field,
|
|
inferredType: type,
|
|
confidence,
|
|
sampleSize,
|
|
lastUpdated: Date.now(),
|
|
detectionMethod: 'value',
|
|
metadata: {
|
|
format,
|
|
...extraMetadata
|
|
}
|
|
}
|
|
}
|
|
}
|