soulcraft/brainy
2
0
Fork 0
Code Issues 1 Pull requests Projects Releases 335 Packages Wiki Activity Actions

fix(8.0): distinctCount aggregates distinct values of any type + edge-case regression tests

Browse source 
distinctCount previously routed through getNumericField, so it silently returned 0 for
string/categorical fields — its primary use case (distinct categories / users / tags).
It now tracks the raw value (any type, keyed by string form) on both the add and remove
contribution paths; numeric ops (sum/avg/min/max/stddev/variance/percentile) are unchanged.

Also adds regression coverage confirming two long-standing query behaviors hold on the 8.0
engine: find({ where: { field: { missing: true } } }) matches a never-registered field, and
find({ type: [...], orderBy, limit }) returns the full set on the first call after
mutate+query+get. (percentile/median were already correct.)

Tests: tests/unit/brainy/find-agg-edge-cases.test.ts + existing aggregation suite green.
This commit is contained in:
David Snelling 2026-06-17 12:03:06 -07:00
parent 7aad80395e
commit 574a8b147c
2 changed files with 130 additions and 4 deletions
Download patch file Download diff file Expand all files Collapse all files

32
src/aggregation/AggregationIndex.ts
View file

@ -200,8 +200,9 @@ function updateMetricAdd(state: MetricState, val: number, op: AggregationOp): vo
state.m2 = (state.m2 ?? 0) + (val - oldMean) * (val - mean)
}
// Percentile/distinctCount: track the value multiset (matches Cortex's value_counts).
if (op === 'percentile' || op === 'distinctCount') {
// Percentile: track the numeric value multiset (matches Cortex's value_counts) for the
// exact-percentile computation. (distinctCount is tracked separately, on raw values.)
if (op === 'percentile') {
if (!state.valueCounts) state.valueCounts = {}
const key = String(val)
state.valueCounts[key] = (state.valueCounts[key] ?? 0) + 1
@ -213,8 +214,9 @@ function updateMetricAdd(state: MetricState, val: number, op: AggregationOp): vo
* Note: removing from Welford's is the inverse update.
*/
function updateMetricRemove(state: MetricState, val: number, op: AggregationOp): void {
// Percentile/distinctCount: decrement the value multiset (drop the key at zero).
if ((op === 'percentile' || op === 'distinctCount') && state.valueCounts) {
// Percentile: decrement the numeric value multiset (drop the key at zero).
// (distinctCount is decremented separately, on raw values.)
if (op === 'percentile' && state.valueCounts) {
const key = String(val)
const c = state.valueCounts[key]
if (c !== undefined) {
@ -741,6 +743,17 @@ export class AggregationIndex {
if (metricDef.op === 'count') {
state.count++
state.sum++
} else if (metricDef.op === 'distinctCount') {
// distinctCount tracks distinct values of ANY type (strings, numbers, booleans),
// keyed by their string form — NOT numeric-coerced, since its primary use is
// categorical (distinct categories / users / tags), not numeric columns.
const raw = resolveEntityField(entity as unknown as HNSWNounWithMetadata, metricDef.field!)
if (raw !== undefined && raw !== null) {
if (!state.valueCounts) state.valueCounts = {}
const key = String(raw)
state.valueCounts[key] = (state.valueCounts[key] ?? 0) + 1
state.count++
}
} else {
const val = getNumericField(entity, metricDef.field!)
if (val !== undefined) {
@ -777,6 +790,17 @@ export class AggregationIndex {
if (metricDef.op === 'count') {
state.count = Math.max(0, state.count - 1)
state.sum = Math.max(0, state.sum - 1)
} else if (metricDef.op === 'distinctCount') {
const raw = resolveEntityField(entity as unknown as HNSWNounWithMetadata, metricDef.field!)
if (raw !== undefined && raw !== null && state.valueCounts) {
const key = String(raw)
const c = state.valueCounts[key]
if (c !== undefined) {
if (c <= 1) delete state.valueCounts[key]
else state.valueCounts[key] = c - 1
}
state.count = Math.max(0, state.count - 1)
}
} else {
const val = getNumericField(entity, metricDef.field!)
if (val !== undefined) {