feat: add aggregation engine with incremental SUM/COUNT/AVG/MIN/MAX, GROUP BY, and time windows
Add a write-time incremental aggregation engine that maintains running
totals on every add/update/delete for O(1) read performance. Integrates
into brain.find({ aggregate }) for a unified query API.
Core features:
- AggregationIndex with defineAggregate()/removeAggregate() API
- Five aggregation operations: SUM, COUNT, AVG, MIN, MAX
- GROUP BY with multiple dimensions including time windows
- Time window bucketing: hour, day, week, month, quarter, year, custom
- Materialization of results as NounType.Measurement entities
- Debounced persistence of definitions and state to storage
- Definition change detection via FNV-1a hashing with auto-rebuild
- Infinite loop prevention for materialized entities
- 'aggregation' plugin provider key for native acceleration
- Lazy initialization (created on first defineAggregate() call)
- 73 tests (unit + integration) covering all functionality
Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
parent
089a4d4141
commit
f024e56ee7
19 changed files with 2986 additions and 21 deletions
|
|
@ -288,6 +288,10 @@ export interface FindParams<T = any> {
|
|||
|
||||
// Performance options
|
||||
writeOnly?: boolean // Skip validation for high-speed ingestion
|
||||
|
||||
// Aggregation
|
||||
/** Query a named aggregate definition. String shorthand or full query params. */
|
||||
aggregate?: string | AggregateQueryParams
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -679,29 +683,152 @@ export interface TraverseParams {
|
|||
limit?: number // Max nodes to visit
|
||||
}
|
||||
|
||||
// ============= Aggregation Engine Types =============
|
||||
|
||||
/**
|
||||
* Aggregation parameters
|
||||
* Supported aggregation operations
|
||||
*/
|
||||
export interface AggregateParams<T = any> {
|
||||
query?: FindParams<T> // Base query to aggregate
|
||||
groupBy: string | string[] // Fields to group by
|
||||
metrics: AggregateMetric[] // Metrics to calculate
|
||||
having?: any // Post-aggregation filters
|
||||
orderBy?: string // Sort results
|
||||
limit?: number // Max groups
|
||||
export type AggregationOp = 'sum' | 'count' | 'avg' | 'min' | 'max'
|
||||
|
||||
/**
|
||||
* Time window granularity for GROUP BY time dimensions
|
||||
*/
|
||||
export type TimeWindowGranularity =
|
||||
| 'hour' | 'day' | 'week' | 'month' | 'quarter' | 'year'
|
||||
| { seconds: number }
|
||||
|
||||
/**
|
||||
* A GROUP BY dimension — either a plain metadata field or a time-windowed field
|
||||
*/
|
||||
export type GroupByDimension = string | { field: string; window: TimeWindowGranularity }
|
||||
|
||||
/**
|
||||
* Source filter for which entities feed into an aggregate
|
||||
*/
|
||||
export interface AggregateSource {
|
||||
/** Filter by entity type(s) */
|
||||
type?: NounType | NounType[]
|
||||
/** Metadata filter (same syntax as find({ where })) */
|
||||
where?: Record<string, unknown>
|
||||
/** Multi-tenancy service filter */
|
||||
service?: string
|
||||
}
|
||||
|
||||
/**
|
||||
* Aggregate metrics
|
||||
* Full aggregate definition — registered via brain.defineAggregate()
|
||||
*/
|
||||
export type AggregateMetric =
|
||||
| 'count'
|
||||
| 'sum'
|
||||
| 'avg'
|
||||
| 'min'
|
||||
| 'max'
|
||||
| 'stddev'
|
||||
| { custom: string; field: string }
|
||||
export interface AggregateDefinition {
|
||||
/** Unique name for this aggregate (used in queries) */
|
||||
name: string
|
||||
/** Which entities contribute to this aggregate */
|
||||
source: AggregateSource
|
||||
/** Dimensions to group by */
|
||||
groupBy: GroupByDimension[]
|
||||
/** Named metrics to compute */
|
||||
metrics: Record<string, AggregateMetricDef>
|
||||
/** Control materialization of results as NounType.Measurement entities */
|
||||
materialize?: boolean | { debounceMs?: number; trackSources?: boolean }
|
||||
}
|
||||
|
||||
/**
|
||||
* Single metric definition within an aggregate
|
||||
*/
|
||||
export interface AggregateMetricDef {
|
||||
/** Aggregation operation */
|
||||
op: AggregationOp
|
||||
/** Metadata field to aggregate (required for sum/avg/min/max; optional for count) */
|
||||
field?: string
|
||||
}
|
||||
|
||||
/**
|
||||
* Internal running state for a single metric.
|
||||
* Tracks enough to compute all operations incrementally.
|
||||
*/
|
||||
export interface MetricState {
|
||||
sum: number
|
||||
count: number
|
||||
min: number
|
||||
max: number
|
||||
}
|
||||
|
||||
/**
|
||||
* Internal state for one aggregate group (one combination of group key values)
|
||||
*/
|
||||
export interface AggregateGroupState {
|
||||
/** The group key values (e.g., { category: 'food', period: '2024-01' }) */
|
||||
groupKey: Record<string, string | number>
|
||||
/** Running metric states keyed by metric name */
|
||||
metrics: Record<string, MetricState>
|
||||
/** Entity ID of the materialized Measurement entity (if materialized) */
|
||||
materializedEntityId?: string
|
||||
/** Timestamp of last update */
|
||||
lastUpdated: number
|
||||
}
|
||||
|
||||
/**
|
||||
* Query parameters for reading aggregate results
|
||||
*/
|
||||
export interface AggregateQueryParams {
|
||||
/** Name of the aggregate to query */
|
||||
name: string
|
||||
/** Filter aggregate groups by their key values */
|
||||
where?: Record<string, unknown>
|
||||
/** Sort by metric name or group key field */
|
||||
orderBy?: string
|
||||
/** Sort direction */
|
||||
order?: 'asc' | 'desc'
|
||||
/** Max results */
|
||||
limit?: number
|
||||
/** Skip N results */
|
||||
offset?: number
|
||||
}
|
||||
|
||||
/**
|
||||
* A single aggregate result row
|
||||
*/
|
||||
export interface AggregateResult {
|
||||
/** Group key values for this row */
|
||||
groupKey: Record<string, string | number>
|
||||
/** Computed metric values (derived from MetricState based on op) */
|
||||
metrics: Record<string, number>
|
||||
/** Total entity count in this group */
|
||||
count: number
|
||||
/** Entity ID of materialized Measurement (if materialized) */
|
||||
entityId?: string
|
||||
}
|
||||
|
||||
/**
|
||||
* Provider interface for Cortex-accelerated aggregation.
|
||||
* When registered as 'aggregation' provider, Brainy delegates to this.
|
||||
*/
|
||||
export interface AggregationProvider {
|
||||
/** Incrementally update aggregation state when an entity changes */
|
||||
incrementalUpdate(
|
||||
name: string,
|
||||
def: AggregateDefinition,
|
||||
entity: Record<string, unknown>,
|
||||
op: 'add' | 'update' | 'delete',
|
||||
prev?: Record<string, unknown>
|
||||
): AggregateGroupState[]
|
||||
|
||||
/** Compute the group key for a given entity */
|
||||
computeGroupKey(
|
||||
entity: Record<string, unknown>,
|
||||
groupBy: GroupByDimension[]
|
||||
): Record<string, string | number>
|
||||
|
||||
/** Rebuild an entire aggregate from scratch */
|
||||
rebuildAggregate(
|
||||
def: AggregateDefinition,
|
||||
entities: Array<Record<string, unknown>>
|
||||
): Map<string, AggregateGroupState>
|
||||
|
||||
/** Query aggregate state with filtering/sorting/pagination */
|
||||
queryAggregate(
|
||||
state: Map<string, AggregateGroupState>,
|
||||
params: AggregateQueryParams
|
||||
): AggregateResult[]
|
||||
}
|
||||
|
||||
// ============= Configuration =============
|
||||
|
||||
|
|
|
|||
|
|
@ -8,7 +8,7 @@
|
|||
*/
|
||||
|
||||
import { Vector } from '../coreTypes.js'
|
||||
import { AddParams, RelateParams, Result, Entity, FindParams, SimilarParams } from './brainy.types.js'
|
||||
import { AddParams, RelateParams, Result, Entity, FindParams, SimilarParams, AggregateDefinition } from './brainy.types.js'
|
||||
import { NounType, VerbType } from './graphTypes.js'
|
||||
import type { MigrationPreview, MigrationResult, MigrateOptions } from '../migration/types.js'
|
||||
|
||||
|
|
@ -179,4 +179,18 @@ export interface BrainyInterface<T = unknown> {
|
|||
* ```
|
||||
*/
|
||||
migrate(options?: MigrateOptions): Promise<MigrationResult | MigrationPreview>
|
||||
|
||||
/**
|
||||
* Define a named aggregate for incremental computation
|
||||
*
|
||||
* @param def - Aggregate definition (name, source filter, groupBy, metrics)
|
||||
*/
|
||||
defineAggregate(def: AggregateDefinition): void
|
||||
|
||||
/**
|
||||
* Remove a named aggregate and clean up its state
|
||||
*
|
||||
* @param name - Name of the aggregate to remove
|
||||
*/
|
||||
removeAggregate(name: string): void
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue