feat: queryAggregate() + HAVING, plus aggregate backfill, traversal depth/via, extraction typing (BR-ADV-FEATURES-BUN)

New report APIs:
- brain.queryAggregate(name, params) returns the clean AggregateResult[] shape
  ({ groupKey, metrics, count }) directly, instead of the search-Result wrapper that
  find({ aggregate }) returns.
- HAVING: find({ aggregate, having }) and queryAggregate filter groups by computed
  metric values (e.g. revenue > 1000), complementing where (which filters group keys).
  Evaluated per group: O(groups), independent of entity count.

Fixes (all reproducible on Node; surfaced under Cortex+Bun by Memory):
- Aggregate backfill-on-define: defining an aggregate over a store that already holds
  matching entities returned []. It now backfills from existing entities on first query
  (storage-agnostic via getNouns), so it works under durable backends that reopen
  pre-populated. groupBy:['noun'] resolves to the entity type; find({ aggregate }) rows
  expose groupKey/metrics/count at the top level.
- Multi-hop find({ connected: { depth, via } }) honors depth and via at every hop
  (was 1-hop only, with verb filtering applied to hop 1 only); BFS bounded by limit.
- extractEntities no longer bleeds a neighbour's type indicator across candidates; each
  candidate is typed by its own span. Also fixes the "Dr." title pattern.

Adds real-embedding regression tests; full unit suite green.
This commit is contained in:
David Snelling 2026-05-26 13:55:43 -07:00
parent 513186d951
commit 1a98e4276a
9 changed files with 481 additions and 96 deletions

View file

@ -208,6 +208,15 @@ export class AggregationIndex {
/** Track which aggregates have dirty state needing persistence */
private dirty = new Set<string>()
/**
* Aggregates whose state must be backfilled from entities that already existed
* when the aggregate was defined (or whose persisted state was missing/stale at
* init). Write-time hooks only capture entities added *after* a definition, so
* without backfill an aggregate defined over a populated store stays empty.
* Drained by the owner (Brainy) which has the entity iterator; see `getPendingBackfills`.
*/
private needsBackfill = new Set<string>()
/** Track aggregates with stale MIN/MAX (need lazy recompute) */
private staleMinMax = new Map<string, Set<string>>()
@ -243,8 +252,10 @@ export class AggregationIndex {
}
this.states.set(def.name, groupMap)
} else {
// Definition changed or no saved state — start fresh (will be rebuilt)
// Definition changed or no saved state — start fresh and backfill from
// existing entities (the owner drains needsBackfill on first query).
this.states.set(def.name, new Map())
this.needsBackfill.add(def.name)
}
this.definitionHashes.set(def.name, currentHash)
@ -332,9 +343,12 @@ export class AggregationIndex {
this.definitions.set(def.name, def)
this.definitionHashes.set(def.name, newHash)
// Reset state if definition changed or doesn't exist yet
// Reset state if definition changed or doesn't exist yet, and flag it for
// backfill so already-stored entities are counted (write-time hooks only see
// future writes). The owner drains this on the next query via getPendingBackfills().
if (!this.states.has(def.name) || (oldHash && oldHash !== newHash)) {
this.states.set(def.name, new Map())
this.needsBackfill.add(def.name)
}
// Notify native provider of definition (caches compiled form for hot path)
@ -376,6 +390,50 @@ export class AggregationIndex {
return this.definitions.has(name)
}
// ============= Backfill =============
//
// Write-time hooks only capture entities added after a definition exists, so an
// aggregate defined over a populated store would stay empty. The owner (Brainy) has
// the entity iterator, so backfill is driven from there: it reads the pending set,
// clears the aggregate, streams every existing entity through `backfillEntity`, then
// calls `finishBackfill`. Clearing first means a concurrent write that landed via the
// incremental hook is wiped and re-counted exactly once by the rescan.
/** Names of aggregates whose state must be (re)built from existing entities. */
getPendingBackfills(): string[] {
return Array.from(this.needsBackfill)
}
/** Clear an aggregate's state so a full rescan cannot double-count. */
beginBackfill(name: string): void {
this.states.set(name, new Map())
// Reset native provider state for this aggregate too, if present.
const def = this.definitions.get(name)
if (def && this.nativeProvider?.removeAggregate && this.nativeProvider?.defineAggregate) {
this.nativeProvider.removeAggregate(name)
this.nativeProvider.defineAggregate(def)
}
}
/** Feed one already-stored entity into a single aggregate during backfill. */
backfillEntity(name: string, entity: Record<string, unknown>): void {
if (isAggregateEntity(entity)) return
const def = this.definitions.get(name)
if (!def || !matchesSource(entity, def.source)) return
if (this.nativeProvider) {
this.applyNativeResults(name, this.nativeProvider.incrementalUpdate(name, def, entity, 'add'))
} else {
this.addContribution(name, def, entity)
}
}
/** Mark an aggregate's backfill complete; rebuilt state persists on next flush(). */
finishBackfill(name: string): void {
this.needsBackfill.delete(name)
this.dirty.add(name)
}
// ============= Write-Time Hooks =============
/**
@ -549,6 +607,12 @@ export class AggregationIndex {
}
}
// HAVING: filter groups by computed metric values (post-compute, O(groups), before
// sort/pagination). Reuses the where-operator engine over metrics + `count`.
if (params.having && Object.keys(params.having).length > 0) {
if (!matchesMetadataFilter({ ...metrics, count: totalCount } as any, params.having)) continue
}
results.push({
groupKey: { ...group.groupKey },
metrics,