refactor: migrate aggregation + neural field reads to resolveEntityField
Preventive cleanup of sites that previously used dual-lookup patterns (metadata[field] ?? entity[field]) or hardcoded timestamp if-chains to read fields off HNSWNounWithMetadata. These worked today but were fragile — the same failure mode that caused the orderBy sort bug would have recurred the next time someone reordered or dropped a lookup. - AggregationIndex.computeGroupKey + getNumericField now route all dimension and metric field lookups through resolveEntityField. - improvedNeuralAPI._getItemsByTimeWindow + _clusterItemsByField use resolveEntityField as the primary path, with item.data as a legacy producer fallback. The type/nounType legacy branch is preserved as-is since it handles storage format compatibility rather than shape contract drift. No behavior change for correct inputs. All aggregation and orderby regression tests pass unchanged.
This commit is contained in:
parent
9855f431f7
commit
108e2bcab4
2 changed files with 33 additions and 29 deletions
|
|
@ -13,7 +13,8 @@
|
||||||
* - Delta detection hashes definitions; only changed aggregates rebuild on restart
|
* - Delta detection hashes definitions; only changed aggregates rebuild on restart
|
||||||
*/
|
*/
|
||||||
|
|
||||||
import type { StorageAdapter } from '../coreTypes.js'
|
import type { StorageAdapter, HNSWNounWithMetadata } from '../coreTypes.js'
|
||||||
|
import { resolveEntityField } from '../coreTypes.js'
|
||||||
import type {
|
import type {
|
||||||
AggregateDefinition,
|
AggregateDefinition,
|
||||||
AggregateGroupState,
|
AggregateGroupState,
|
||||||
|
|
@ -96,22 +97,25 @@ function matchesSource(entity: Record<string, unknown>, source: AggregateDefinit
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Compute the group key for an entity given groupBy dimensions.
|
* Compute the group key for an entity given groupBy dimensions.
|
||||||
|
*
|
||||||
|
* Field lookups go through `resolveEntityField` to honor the
|
||||||
|
* HNSWNounWithMetadata shape contract (standard fields top-level,
|
||||||
|
* custom fields in metadata) in a single source of truth.
|
||||||
*/
|
*/
|
||||||
function computeGroupKey(
|
function computeGroupKey(
|
||||||
entity: Record<string, unknown>,
|
entity: Record<string, unknown>,
|
||||||
groupBy: GroupByDimension[]
|
groupBy: GroupByDimension[]
|
||||||
): Record<string, string | number> {
|
): Record<string, string | number> {
|
||||||
const key: Record<string, string | number> = {}
|
const key: Record<string, string | number> = {}
|
||||||
const metadata = (entity.metadata ?? {}) as Record<string, unknown>
|
const e = entity as unknown as HNSWNounWithMetadata
|
||||||
|
|
||||||
for (const dim of groupBy) {
|
for (const dim of groupBy) {
|
||||||
if (typeof dim === 'string') {
|
if (typeof dim === 'string') {
|
||||||
// Plain field lookup — check metadata first, then top-level
|
const val = resolveEntityField(e, dim)
|
||||||
const val = metadata[dim] ?? entity[dim]
|
|
||||||
key[dim] = val !== undefined && val !== null ? String(val) : '__null__'
|
key[dim] = val !== undefined && val !== null ? String(val) : '__null__'
|
||||||
} else {
|
} else {
|
||||||
// Time-windowed field
|
// Time-windowed field
|
||||||
const val = metadata[dim.field] ?? entity[dim.field]
|
const val = resolveEntityField(e, dim.field)
|
||||||
if (typeof val === 'number') {
|
if (typeof val === 'number') {
|
||||||
key[dim.field] = bucketTimestamp(val, dim.window)
|
key[dim.field] = bucketTimestamp(val, dim.window)
|
||||||
} else {
|
} else {
|
||||||
|
|
@ -124,11 +128,14 @@ function computeGroupKey(
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Get the numeric value of a field from entity metadata (for metric computation).
|
* Get the numeric value of a field from an entity (for metric computation).
|
||||||
|
*
|
||||||
|
* Routes through `resolveEntityField` so standard top-level numeric fields
|
||||||
|
* (weight, confidence, createdAt, updatedAt) and custom user numeric fields
|
||||||
|
* in metadata are both handled in one place.
|
||||||
*/
|
*/
|
||||||
function getNumericField(entity: Record<string, unknown>, field: string): number | undefined {
|
function getNumericField(entity: Record<string, unknown>, field: string): number | undefined {
|
||||||
const metadata = (entity.metadata ?? {}) as Record<string, unknown>
|
const val = resolveEntityField(entity as unknown as HNSWNounWithMetadata, field)
|
||||||
const val = metadata[field] ?? entity[field]
|
|
||||||
if (typeof val === 'number' && !isNaN(val)) return val
|
if (typeof val === 'number' && !isNaN(val)) return val
|
||||||
if (typeof val === 'string') {
|
if (typeof val === 'string') {
|
||||||
const num = parseFloat(val)
|
const num = parseFloat(val)
|
||||||
|
|
|
||||||
|
|
@ -18,7 +18,7 @@
|
||||||
* Private methods are prefixed with _ and not exposed in public API
|
* Private methods are prefixed with _ and not exposed in public API
|
||||||
*/
|
*/
|
||||||
|
|
||||||
import { Vector } from '../coreTypes.js'
|
import { Vector, resolveEntityField, type HNSWNounWithMetadata } from '../coreTypes.js'
|
||||||
import { cosineDistance, euclideanDistance } from '../utils/distance.js'
|
import { cosineDistance, euclideanDistance } from '../utils/distance.js'
|
||||||
import {
|
import {
|
||||||
SemanticCluster,
|
SemanticCluster,
|
||||||
|
|
@ -2927,22 +2927,20 @@ export class ImprovedNeuralAPI {
|
||||||
private _groupByDomain(items: any[], field: string): Map<string, any[]> {
|
private _groupByDomain(items: any[], field: string): Map<string, any[]> {
|
||||||
const groups = new Map()
|
const groups = new Map()
|
||||||
for (const item of items) {
|
for (const item of items) {
|
||||||
// Check multiple locations for the field value
|
// Resolve domain field value across legacy storage shapes.
|
||||||
let domain: any = 'unknown'
|
let domain: any = 'unknown'
|
||||||
|
|
||||||
// Special handling for type/nounType field
|
// Special handling for type/nounType — legacy storage compatibility
|
||||||
|
// where the type may live at item.nounType or metadata.noun/metadata.type
|
||||||
|
// rather than the current top-level item.type.
|
||||||
if (field === 'type' || field === 'nounType') {
|
if (field === 'type' || field === 'nounType') {
|
||||||
domain = item.nounType || item.metadata?.noun || item.metadata?.type || 'unknown'
|
domain = item.nounType || item.metadata?.noun || item.metadata?.type || 'unknown'
|
||||||
} else {
|
} else {
|
||||||
// Check root level first
|
// Primary path: single-source-of-truth resolver (standard top-level,
|
||||||
domain = item[field]
|
// custom in metadata).
|
||||||
|
domain = resolveEntityField(item as HNSWNounWithMetadata, field)
|
||||||
|
|
||||||
// Then check metadata
|
// Legacy fallback: some producers stash data under item.data.
|
||||||
if (domain == null) {
|
|
||||||
domain = item.metadata?.[field]
|
|
||||||
}
|
|
||||||
|
|
||||||
// Then check data
|
|
||||||
if (domain == null) {
|
if (domain == null) {
|
||||||
domain = item.data?.[field]
|
domain = item.data?.[field]
|
||||||
}
|
}
|
||||||
|
|
@ -3127,17 +3125,16 @@ export class ImprovedNeuralAPI {
|
||||||
|
|
||||||
const entity = item.entity
|
const entity = item.entity
|
||||||
|
|
||||||
// Get timestamp value from various possible locations
|
// Resolve the timestamp field through the single-source-of-truth
|
||||||
let timestamp: any = null
|
// helper so standard top-level fields (createdAt, updatedAt) and
|
||||||
|
// custom fields in metadata are both handled uniformly. Fall back
|
||||||
// Check root level first
|
// to entity.data for legacy producers that stash timestamps there.
|
||||||
if (timeField === 'createdAt' || timeField === 'updatedAt') {
|
let timestamp: any = resolveEntityField(
|
||||||
timestamp = entity[timeField]
|
entity as HNSWNounWithMetadata,
|
||||||
}
|
timeField
|
||||||
|
)
|
||||||
// Check metadata/data
|
|
||||||
if (timestamp == null) {
|
if (timestamp == null) {
|
||||||
timestamp = entity.metadata?.[timeField] || entity.data?.[timeField]
|
timestamp = entity.data?.[timeField]
|
||||||
}
|
}
|
||||||
|
|
||||||
if (timestamp == null) {
|
if (timestamp == null) {
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue