feat: queryAggregate() + HAVING, plus aggregate backfill, traversal depth/via, extraction typing (BR-ADV-FEATURES-BUN)
New report APIs:
- brain.queryAggregate(name, params) returns the clean AggregateResult[] shape
({ groupKey, metrics, count }) directly, instead of the search-Result wrapper that
find({ aggregate }) returns.
- HAVING: find({ aggregate, having }) and queryAggregate filter groups by computed
metric values (e.g. revenue > 1000), complementing where (which filters group keys).
Evaluated per group: O(groups), independent of entity count.
Fixes (all reproducible on Node; surfaced under Cortex+Bun by Memory):
- Aggregate backfill-on-define: defining an aggregate over a store that already holds
matching entities returned []. It now backfills from existing entities on first query
(storage-agnostic via getNouns), so it works under durable backends that reopen
pre-populated. groupBy:['noun'] resolves to the entity type; find({ aggregate }) rows
expose groupKey/metrics/count at the top level.
- Multi-hop find({ connected: { depth, via } }) honors depth and via at every hop
(was 1-hop only, with verb filtering applied to hop 1 only); BFS bounded by limit.
- extractEntities no longer bleeds a neighbour's type indicator across candidates; each
candidate is typed by its own span. Also fixes the "Dr." title pattern.
Adds real-embedding regression tests; full unit suite green.
This commit is contained in:
parent
513186d951
commit
1a98e4276a
9 changed files with 481 additions and 96 deletions
|
|
@ -208,6 +208,15 @@ export class AggregationIndex {
|
|||
/** Track which aggregates have dirty state needing persistence */
|
||||
private dirty = new Set<string>()
|
||||
|
||||
/**
|
||||
* Aggregates whose state must be backfilled from entities that already existed
|
||||
* when the aggregate was defined (or whose persisted state was missing/stale at
|
||||
* init). Write-time hooks only capture entities added *after* a definition, so
|
||||
* without backfill an aggregate defined over a populated store stays empty.
|
||||
* Drained by the owner (Brainy) which has the entity iterator; see `getPendingBackfills`.
|
||||
*/
|
||||
private needsBackfill = new Set<string>()
|
||||
|
||||
/** Track aggregates with stale MIN/MAX (need lazy recompute) */
|
||||
private staleMinMax = new Map<string, Set<string>>()
|
||||
|
||||
|
|
@ -243,8 +252,10 @@ export class AggregationIndex {
|
|||
}
|
||||
this.states.set(def.name, groupMap)
|
||||
} else {
|
||||
// Definition changed or no saved state — start fresh (will be rebuilt)
|
||||
// Definition changed or no saved state — start fresh and backfill from
|
||||
// existing entities (the owner drains needsBackfill on first query).
|
||||
this.states.set(def.name, new Map())
|
||||
this.needsBackfill.add(def.name)
|
||||
}
|
||||
|
||||
this.definitionHashes.set(def.name, currentHash)
|
||||
|
|
@ -332,9 +343,12 @@ export class AggregationIndex {
|
|||
this.definitions.set(def.name, def)
|
||||
this.definitionHashes.set(def.name, newHash)
|
||||
|
||||
// Reset state if definition changed or doesn't exist yet
|
||||
// Reset state if definition changed or doesn't exist yet, and flag it for
|
||||
// backfill so already-stored entities are counted (write-time hooks only see
|
||||
// future writes). The owner drains this on the next query via getPendingBackfills().
|
||||
if (!this.states.has(def.name) || (oldHash && oldHash !== newHash)) {
|
||||
this.states.set(def.name, new Map())
|
||||
this.needsBackfill.add(def.name)
|
||||
}
|
||||
|
||||
// Notify native provider of definition (caches compiled form for hot path)
|
||||
|
|
@ -376,6 +390,50 @@ export class AggregationIndex {
|
|||
return this.definitions.has(name)
|
||||
}
|
||||
|
||||
// ============= Backfill =============
|
||||
//
|
||||
// Write-time hooks only capture entities added after a definition exists, so an
|
||||
// aggregate defined over a populated store would stay empty. The owner (Brainy) has
|
||||
// the entity iterator, so backfill is driven from there: it reads the pending set,
|
||||
// clears the aggregate, streams every existing entity through `backfillEntity`, then
|
||||
// calls `finishBackfill`. Clearing first means a concurrent write that landed via the
|
||||
// incremental hook is wiped and re-counted exactly once by the rescan.
|
||||
|
||||
/** Names of aggregates whose state must be (re)built from existing entities. */
|
||||
getPendingBackfills(): string[] {
|
||||
return Array.from(this.needsBackfill)
|
||||
}
|
||||
|
||||
/** Clear an aggregate's state so a full rescan cannot double-count. */
|
||||
beginBackfill(name: string): void {
|
||||
this.states.set(name, new Map())
|
||||
// Reset native provider state for this aggregate too, if present.
|
||||
const def = this.definitions.get(name)
|
||||
if (def && this.nativeProvider?.removeAggregate && this.nativeProvider?.defineAggregate) {
|
||||
this.nativeProvider.removeAggregate(name)
|
||||
this.nativeProvider.defineAggregate(def)
|
||||
}
|
||||
}
|
||||
|
||||
/** Feed one already-stored entity into a single aggregate during backfill. */
|
||||
backfillEntity(name: string, entity: Record<string, unknown>): void {
|
||||
if (isAggregateEntity(entity)) return
|
||||
const def = this.definitions.get(name)
|
||||
if (!def || !matchesSource(entity, def.source)) return
|
||||
|
||||
if (this.nativeProvider) {
|
||||
this.applyNativeResults(name, this.nativeProvider.incrementalUpdate(name, def, entity, 'add'))
|
||||
} else {
|
||||
this.addContribution(name, def, entity)
|
||||
}
|
||||
}
|
||||
|
||||
/** Mark an aggregate's backfill complete; rebuilt state persists on next flush(). */
|
||||
finishBackfill(name: string): void {
|
||||
this.needsBackfill.delete(name)
|
||||
this.dirty.add(name)
|
||||
}
|
||||
|
||||
// ============= Write-Time Hooks =============
|
||||
|
||||
/**
|
||||
|
|
@ -549,6 +607,12 @@ export class AggregationIndex {
|
|||
}
|
||||
}
|
||||
|
||||
// HAVING: filter groups by computed metric values (post-compute, O(groups), before
|
||||
// sort/pagination). Reuses the where-operator engine over metrics + `count`.
|
||||
if (params.having && Object.keys(params.having).length > 0) {
|
||||
if (!matchesMetadataFilter({ ...metrics, count: totalCount } as any, params.having)) continue
|
||||
}
|
||||
|
||||
results.push({
|
||||
groupKey: { ...group.groupKey },
|
||||
metrics,
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue