feat: array-unnest groupBy for aggregates + batch-embed entity extraction

- groupBy now supports { field, unnest: true }: an entity contributes once per
  distinct element of an array field (tag frequency / faceted counts). Duplicate
  elements on one entity count once; an empty/missing array joins no group. The
  incremental add/update/delete paths fan out across the unnested groups.
- extractEntities/extractConcepts batch-embed the unique candidate spans in one
  embedBatch call instead of one embed() per candidate (N sequential model calls);
  falls back to per-candidate embedding if the batch fails. No behavior change.
This commit is contained in:
David Snelling 2026-05-26 14:20:40 -07:00
parent 2591001bd0
commit c2e21b7b3c
8 changed files with 214 additions and 100 deletions

View file

@ -137,7 +137,27 @@ export class NeuralEntityExtractor {
// Step 1: Extract potential entities using patterns
const candidates = await this.extractCandidates(text)
// Step 1b: Batch-embed the unique candidate texts once, instead of one brain.embed()
// per candidate inside EmbeddingSignal (N sequential model calls). The vectors are
// forwarded into the signal; if the batch fails, the signal falls back to per-candidate
// embedding, so this is a pure performance optimization with no behavior change.
const vectorByText = new Map<string, Vector>()
if (useNeuralMatching) {
const uniqueTexts = [...new Set(candidates.map(c => c.text))]
if (uniqueTexts.length > 0) {
try {
const vectors = await this.brain.embedBatch(uniqueTexts)
uniqueTexts.forEach((t, i) => {
const v = vectors[i]
if (v) vectorByText.set(t, v)
})
} catch {
// Leave the map empty — EmbeddingSignal will embed per-candidate as before.
}
}
}
// Step 2: Classify each candidate using SmartExtractor
for (const candidate of candidates) {
// Use SmartExtractor for unified neural + rule-based classification.
@ -146,7 +166,8 @@ export class NeuralEntityExtractor {
// option had no loosening effect).
const classification = await this.smartExtractor.extract(candidate.text, {
definition: candidate.context,
allTerms: [candidate.text, candidate.context]
allTerms: [candidate.text, candidate.context],
vector: vectorByText.get(candidate.text)
}, minConfidence)
// SmartExtractor already gates at minConfidence; this guards against null only.