feat: array-unnest groupBy for aggregates + batch-embed entity extraction
- groupBy now supports { field, unnest: true }: an entity contributes once per
distinct element of an array field (tag frequency / faceted counts). Duplicate
elements on one entity count once; an empty/missing array joins no group. The
incremental add/update/delete paths fan out across the unnested groups.
- extractEntities/extractConcepts batch-embed the unique candidate spans in one
embedBatch call instead of one embed() per candidate (N sequential model calls);
falls back to per-candidate embedding if the batch fails. No behavior change.
This commit is contained in:
parent
2591001bd0
commit
c2e21b7b3c
8 changed files with 214 additions and 100 deletions
|
|
@ -34,6 +34,7 @@
|
|||
|
||||
import type { Brainy } from '../brainy.js'
|
||||
import type { NounType } from '../types/graphTypes.js'
|
||||
import type { Vector } from '../coreTypes.js'
|
||||
import { ExactMatchSignal } from './signals/ExactMatchSignal.js'
|
||||
import { PatternSignal } from './signals/PatternSignal.js'
|
||||
import { EmbeddingSignal } from './signals/EmbeddingSignal.js'
|
||||
|
|
@ -211,6 +212,8 @@ export class SmartExtractor {
|
|||
formatContext?: FormatContext
|
||||
allTerms?: string[]
|
||||
metadata?: any
|
||||
/** Pre-computed candidate embedding (from a batch embed) — forwarded to EmbeddingSignal. */
|
||||
vector?: Vector
|
||||
},
|
||||
minConfidence?: number
|
||||
): Promise<ExtractionResult | null> {
|
||||
|
|
@ -243,7 +246,8 @@ export class SmartExtractor {
|
|||
const enrichedContext = {
|
||||
definition: context?.definition,
|
||||
allTerms: [...(context?.allTerms || []), ...formatHints],
|
||||
metadata: context?.metadata
|
||||
metadata: context?.metadata,
|
||||
vector: context?.vector
|
||||
}
|
||||
|
||||
// Execute all signals in parallel
|
||||
|
|
|
|||
|
|
@ -137,7 +137,27 @@ export class NeuralEntityExtractor {
|
|||
|
||||
// Step 1: Extract potential entities using patterns
|
||||
const candidates = await this.extractCandidates(text)
|
||||
|
||||
|
||||
// Step 1b: Batch-embed the unique candidate texts once, instead of one brain.embed()
|
||||
// per candidate inside EmbeddingSignal (N sequential model calls). The vectors are
|
||||
// forwarded into the signal; if the batch fails, the signal falls back to per-candidate
|
||||
// embedding, so this is a pure performance optimization with no behavior change.
|
||||
const vectorByText = new Map<string, Vector>()
|
||||
if (useNeuralMatching) {
|
||||
const uniqueTexts = [...new Set(candidates.map(c => c.text))]
|
||||
if (uniqueTexts.length > 0) {
|
||||
try {
|
||||
const vectors = await this.brain.embedBatch(uniqueTexts)
|
||||
uniqueTexts.forEach((t, i) => {
|
||||
const v = vectors[i]
|
||||
if (v) vectorByText.set(t, v)
|
||||
})
|
||||
} catch {
|
||||
// Leave the map empty — EmbeddingSignal will embed per-candidate as before.
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Step 2: Classify each candidate using SmartExtractor
|
||||
for (const candidate of candidates) {
|
||||
// Use SmartExtractor for unified neural + rule-based classification.
|
||||
|
|
@ -146,7 +166,8 @@ export class NeuralEntityExtractor {
|
|||
// option had no loosening effect).
|
||||
const classification = await this.smartExtractor.extract(candidate.text, {
|
||||
definition: candidate.context,
|
||||
allTerms: [candidate.text, candidate.context]
|
||||
allTerms: [candidate.text, candidate.context],
|
||||
vector: vectorByText.get(candidate.text)
|
||||
}, minConfidence)
|
||||
|
||||
// SmartExtractor already gates at minConfidence; this guards against null only.
|
||||
|
|
|
|||
|
|
@ -151,6 +151,8 @@ export class EmbeddingSignal {
|
|||
definition?: string
|
||||
allTerms?: string[]
|
||||
metadata?: any
|
||||
/** Pre-computed candidate embedding (from a batch embed) — skips the per-candidate embed. */
|
||||
vector?: Vector
|
||||
}
|
||||
): Promise<TypeSignal | null> {
|
||||
this.stats.calls++
|
||||
|
|
@ -167,8 +169,8 @@ export class EmbeddingSignal {
|
|||
}
|
||||
|
||||
try {
|
||||
// Embed candidate once (efficiency!)
|
||||
const vector = await this.embedWithTimeout(candidate)
|
||||
// Use the pre-computed vector when the caller batch-embedded; otherwise embed once here.
|
||||
const vector = context?.vector ?? await this.embedWithTimeout(candidate)
|
||||
|
||||
// Check all three sources in parallel
|
||||
const [typeMatch, graphMatch, historyMatch] = await Promise.all([
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue