feat: array-unnest groupBy for aggregates + batch-embed entity extraction

- groupBy now supports { field, unnest: true }: an entity contributes once per
  distinct element of an array field (tag frequency / faceted counts). Duplicate
  elements on one entity count once; an empty/missing array joins no group. The
  incremental add/update/delete paths fan out across the unnested groups.
- extractEntities/extractConcepts batch-embed the unique candidate spans in one
  embedBatch call instead of one embed() per candidate (N sequential model calls);
  falls back to per-candidate embedding if the batch fails. No behavior change.
This commit is contained in:
David Snelling 2026-05-26 14:20:40 -07:00
parent 2591001bd0
commit c2e21b7b3c
8 changed files with 214 additions and 100 deletions

View file

@ -230,4 +230,28 @@ describe('BR-ADV-FEATURES-BUN regression', () => {
await b.close()
})
})
describe('R2 — array-unnest groupBy (tag frequency)', () => {
it('groups by each distinct array element; dedups per entity; skips empty', async () => {
const b: any = new Brainy({ storage: { type: 'memory' } })
await b.init()
b.defineAggregate({
name: 'tags',
source: { type: NounType.Concept },
groupBy: [{ field: 'tags', unnest: true }],
metrics: { count: { op: 'count' } }
})
await b.add({ data: 'a', type: NounType.Concept, metadata: { tags: ['ml', 'ai'] } })
await b.add({ data: 'b', type: NounType.Concept, metadata: { tags: ['ai', 'rust'] } })
await b.add({ data: 'c', type: NounType.Concept, metadata: { tags: ['ai'] } })
await b.add({ data: 'd', type: NounType.Concept, metadata: { tags: ['rust', 'rust'] } }) // dup → once
await b.add({ data: 'e', type: NounType.Concept, metadata: {} }) // no tags → no group
await b.flush()
const rows = await b.queryAggregate('tags', { orderBy: 'count', order: 'desc' })
const freq = Object.fromEntries(rows.map((r: any) => [r.groupKey.tags, r.count]))
expect(freq).toEqual({ ai: 3, rust: 2, ml: 1 })
await b.close()
})
})
})