diff --git a/RELEASES.md b/RELEASES.md index bd3efcdd..7eea67d7 100644 --- a/RELEASES.md +++ b/RELEASES.md @@ -11,6 +11,39 @@ Collective. The SDK wraps it — most products never call Brainy directly. Read --- +## v7.22.1 — 2026-05-26 + +**Affected products:** Anyone using `extractEntities()` / `extractConcepts()`, native +aggregation (`find({ aggregate })`), or multi-hop graph traversal +(`find({ connected: { depth } })`). Three advanced-API correctness fixes — all reproduce on +Node, so they were **not** Bun-specific despite the original report (BR-ADV-FEATURES-BUN). + +### Entity/concept extraction no longer returns `[]` + +`extractEntities()` / `extractConcepts()` returned empty for entity-rich text. The SmartExtractor +ensemble scored agreeing signals with a weighted **sum** against an absolute 0.60 gate, so a +confident low-weight signal (e.g. a name pattern at 0.82) lost selection to a mediocre +high-weight signal that then failed the gate — dropping the whole result. It now selects and +gates on a normalized weighted **average**. Also: the `confidence` option now actually controls +the threshold (was a dead hardcoded 0.60), and the embedding-signal timeout was raised +100ms → 2000ms so the neural signal isn't silently dropped on slower runtimes. + +*Known limitation:* type accuracy on dense multi-entity sentences is still imperfect — a strong +indicator in one entity's surrounding text can bleed onto neighbours. Tracked separately. + +### `find({ aggregate })` exposes `groupKey` / `metrics` / `count` + +Aggregation always computed correct values, but rows nested them under `.metadata`, so callers +expecting the documented `AggregateResult` shape saw empty-looking rows. Those three fields are +now also present at the top level of each result row. + +### Multi-hop `find({ connected: { depth } })` honours depth + +Previously returned only the immediate (1-hop) neighbour at any depth because the graph-search +path ignored `depth` / `via`. It now performs the full depth-aware traversal. + +--- + ## v7.22.0 — 2026-05-15 **Affected products:** All. Fixes a silent-data-loss class affecting `find()` and diff --git a/src/brainy.ts b/src/brainy.ts index 4bbc0aaf..8d1245af 100644 --- a/src/brainy.ts +++ b/src/brainy.ts @@ -4559,7 +4559,7 @@ export class Brainy implements BrainyInterface { } ): Promise { const entities = await this.extract(text, { - types: [NounType.Concept, NounType.Concept], + types: [NounType.Concept], confidence: options?.confidence || 0.7, neuralMatching: true }) @@ -6787,36 +6787,58 @@ export class Brainy implements BrainyInterface { } /** - * Execute graph search component with O(1) traversal + * Execute graph search component. + * + * Honors the full `GraphConstraints` contract: multi-hop `depth` (breadth-first via + * `neighbors()`), `via`/`type` verb-type filtering, and `direction`. Previously this read + * only `from`/`to`/`direction` and did a single 1-hop `getNeighbors()`, so `depth` and `via` + * were silently ignored — `find({ connected: { from, depth: 3 } })` returned only the + * immediate neighbour at every depth. */ private async executeGraphSearch(params: FindParams, existingResults: Result[]): Promise[]> { if (!params.connected) return existingResults - - const { from, to, direction = 'both' } = params.connected - const connectedIds: string[] = [] - + + const { from, to, depth, direction = 'both' } = params.connected + const via = params.connected.via ?? params.connected.type + + // GraphConstraints speaks 'in' | 'out' | 'both'; neighbors() speaks 'incoming' | 'outgoing' | 'both'. + const toNeighborDir = (d: 'in' | 'out' | 'both'): 'incoming' | 'outgoing' | 'both' => + d === 'in' ? 'incoming' : d === 'out' ? 'outgoing' : 'both' + + // Collect reachable ids honoring depth + verb-type filter. `via` may be a single VerbType + // or an array; reuse the depth-aware BFS in neighbors() (one pass per requested verb type). + const collect = async (id: string, dir: 'incoming' | 'outgoing' | 'both'): Promise => { + const verbTypes: Array = + via === undefined ? [undefined] : Array.isArray(via) ? via : [via] + const ids = new Set() + for (const verbType of verbTypes) { + const hops = await this.neighbors(id, { direction: dir, depth: depth ?? 1, verbType }) + for (const n of hops) ids.add(n) + } + return [...ids] + } + + const connectedIds = new Set() + if (from) { - const neighbors = await this.graphIndex.getNeighbors(from, direction) - connectedIds.push(...neighbors) + for (const id of await collect(from, toNeighborDir(direction))) connectedIds.add(id) } - + if (to) { - const reverseDirection = direction === 'in' ? 'out' : direction === 'out' ? 'in' : 'both' - const neighbors = await this.graphIndex.getNeighbors(to, reverseDirection) - connectedIds.push(...neighbors) + const reverse: 'in' | 'out' | 'both' = direction === 'in' ? 'out' : direction === 'out' ? 'in' : 'both' + for (const id of await collect(to, toNeighborDir(reverse))) connectedIds.add(id) } - + // Filter existing results to only connected entities if (existingResults.length > 0) { - const connectedIdSet = new Set(connectedIds) - return existingResults.filter(r => connectedIdSet.has(r.id)) + return existingResults.filter(r => connectedIds.has(r.id)) } - - // Batch-load connected entities for 10x faster cloud storage performance - // GCS: 20 entities = 1×50ms vs 20×50ms = 1000ms (20x faster) + + // Batch-load connected entities for fast cloud-storage performance const results: Result[] = [] - const entitiesMap = await this.batchGet(connectedIds) - for (const id of connectedIds) { + const ids = [...connectedIds] + const entitiesMap = await this.batchGet(ids) + for (const id of ids) { const entity = entitiesMap.get(id) if (entity) { results.push(this.createResult(id, 1.0, entity)) @@ -7912,7 +7934,14 @@ export class Brainy implements BrainyInterface { type: NounType.Measurement, metadata: entity.metadata, data: entity.data, - entity + entity, + // Surface the documented AggregateResult fields at the top level so consumers can read + // groupKey/metrics/count directly. Previously these were only reachable under .metadata, + // so callers expecting an AggregateResult saw rows with no groupKey/metrics/count and + // interpreted the output as degenerate/empty. + groupKey: agg.groupKey, + metrics: agg.metrics, + count: agg.count } }) } diff --git a/src/neural/SmartExtractor.ts b/src/neural/SmartExtractor.ts index c34f3884..7312c87a 100644 --- a/src/neural/SmartExtractor.ts +++ b/src/neural/SmartExtractor.ts @@ -211,12 +211,18 @@ export class SmartExtractor { formatContext?: FormatContext allTerms?: string[] metadata?: any - } + }, + minConfidence?: number ): Promise { this.stats.calls++ + // Per-call confidence threshold (falls back to the instance default). This lets callers + // such as brain.extractEntities({ confidence }) actually loosen or tighten the gate; it is + // part of the cache key below so results computed at one threshold are not reused at another. + const threshold = minConfidence ?? this.options.minConfidence + // Check cache first - const cacheKey = this.getCacheKey(candidate, context) + const cacheKey = this.getCacheKey(candidate, context, threshold) const cached = this.getFromCache(cacheKey) if (cached !== undefined) { this.stats.cacheHits++ @@ -282,8 +288,8 @@ export class SmartExtractor { // Combine using ensemble or best signal const result = this.options.enableEnsemble - ? this.combineEnsemble(signalResults, formatHints, context?.formatContext) - : this.selectBestSignal(signalResults, formatHints, context?.formatContext) + ? this.combineEnsemble(signalResults, formatHints, context?.formatContext, threshold) + : this.selectBestSignal(signalResults, formatHints, context?.formatContext, threshold) // Cache result (including nulls to avoid recomputation) this.addToCache(cacheKey, result) @@ -509,7 +515,8 @@ export class SmartExtractor { private combineEnsemble( signalResults: SignalResult[], formatHints: string[], - formatContext?: FormatContext + formatContext?: FormatContext, + minConfidence: number = this.options.minConfidence ): ExtractionResult | null { // Filter out null results const validResults = signalResults.filter(r => r.type !== null) @@ -518,58 +525,63 @@ export class SmartExtractor { return null } - // Count votes by type with weighted confidence - const typeScores = new Map() + // Group the signals by the type they voted for + const typeScores = new Map() for (const result of validResults) { if (!result.type) continue - const weighted = result.confidence * result.weight const existing = typeScores.get(result.type) - if (existing) { - existing.score += weighted - existing.signals.push(result) + existing.push(result) } else { - typeScores.set(result.type, { score: weighted, signals: [result] }) + typeScores.set(result.type, [result]) } } - // Find best type + // Score each candidate type by a NORMALIZED, weighted-average confidence + // — Σ(confidence·weight) / Σ(weight of its signals) — plus a small boost when + // multiple signals concur, then select the type with the highest such confidence. + // + // Why an average and not a sum (this was the bug): a weighted *sum* lives on the + // signal-weight scale, so two signals agreeing on a fresh brain summed to ≈0.37 — + // below the 0.60 gate — meaning agreement was effectively *penalized*. Worse, selecting + // the best type by that sum let a high-weight signal with mediocre confidence + // (embedding @0.51 · 0.35 = 0.179) outrank a low-weight signal with high confidence + // (pattern @0.82 · 0.20 = 0.164); the wrong type won selection, then failed the threshold + // on its own 0.51 confidence, and the whole extraction returned null. Averaging keeps the + // score on the same 0–1 scale as the threshold, so selection and the gate agree and the + // most-confident type wins (pattern @0.82 here). let bestType: NounType | null = null - let bestScore = 0 + let finalConfidence = 0 let bestSignals: SignalResult[] = [] - for (const [type, data] of typeScores.entries()) { - // Apply agreement boost (multiple signals agree) - let finalScore = data.score - if (data.signals.length > 1) { - const agreementBoost = 0.05 * (data.signals.length - 1) - finalScore += agreementBoost - this.stats.agreementBoosts++ + for (const [type, signals] of typeScores.entries()) { + const weightSum = signals.reduce((sum, s) => sum + s.weight, 0) + const weightedConfidence = signals.reduce((sum, s) => sum + s.confidence * s.weight, 0) + let confidence = weightSum > 0 ? weightedConfidence / weightSum : 0 + + // Reward agreement between independent signals + if (signals.length > 1) { + confidence = Math.min(confidence + 0.05 * (signals.length - 1), 1.0) } - if (finalScore > bestScore) { - bestScore = finalScore + if (confidence > finalConfidence) { + finalConfidence = confidence bestType = type - bestSignals = data.signals + bestSignals = signals } } - // Determine final confidence score - // FIX: When only one signal matches, use its original confidence instead of weighted score - // The weighted score is too low when only one signal matches (e.g., 0.8 * 0.2 = 0.16 < 0.60 threshold) - let finalConfidence = bestScore - if (bestSignals.length === 1) { - // Single signal: use its original confidence - finalConfidence = bestSignals[0].confidence - } - // Check minimum confidence threshold - if (!bestType || finalConfidence < this.options.minConfidence) { + if (!bestType || finalConfidence < minConfidence) { return null } + if (bestSignals.length > 1) { + this.stats.agreementBoosts++ + } + // Track signal contributions const usedSignals = bestSignals.length this.stats.averageSignalsUsed = @@ -604,13 +616,17 @@ export class SmartExtractor { private selectBestSignal( signalResults: SignalResult[], formatHints: string[], - formatContext?: FormatContext + formatContext?: FormatContext, + minConfidence: number = this.options.minConfidence ): ExtractionResult | null { - // Filter valid results and sort by weighted confidence + // Select by confidence — the same metric the threshold checks below. Sorting by + // weighted score (confidence·weight) instead would let a high-weight, mediocre-confidence + // signal outrank a low-weight, high-confidence one and then fail the gate, dropping the + // result entirely (the same defect fixed in combineEnsemble). Weight matters for ensemble + // voting, not for picking the single best signal. const validResults = signalResults .filter(r => r.type !== null) - .map(r => ({ ...r, weightedScore: r.confidence * r.weight })) - .sort((a, b) => b.weightedScore - a.weightedScore) + .sort((a, b) => b.confidence - a.confidence) if (validResults.length === 0) { return null @@ -618,9 +634,7 @@ export class SmartExtractor { const best = validResults[0] - // FIX: Use original confidence, not weighted score for threshold check - // Weighted score is for ranking signals, not for absolute threshold - if (best.confidence < this.options.minConfidence) { + if (best.confidence < minConfidence) { return null } @@ -661,11 +675,12 @@ export class SmartExtractor { /** * Get cache key from candidate and context */ - private getCacheKey(candidate: string, context?: any): string { + private getCacheKey(candidate: string, context?: any, minConfidence?: number): string { const normalized = candidate.toLowerCase().trim() const defSnippet = context?.definition?.substring(0, 50) || '' const format = context?.formatContext?.format || '' - return `${normalized}:${defSnippet}:${format}` + const threshold = minConfidence ?? this.options.minConfidence + return `${normalized}:${defSnippet}:${format}:${threshold}` } /** diff --git a/src/neural/entityExtractor.ts b/src/neural/entityExtractor.ts index 736eaa05..98f05f48 100644 --- a/src/neural/entityExtractor.ts +++ b/src/neural/entityExtractor.ts @@ -140,14 +140,17 @@ export class NeuralEntityExtractor { // Step 2: Classify each candidate using SmartExtractor for (const candidate of candidates) { - // Use SmartExtractor for unified neural + rule-based classification + // Use SmartExtractor for unified neural + rule-based classification. + // Pass the caller's threshold through so `confidence` actually controls the gate + // (previously SmartExtractor applied a hardcoded 0.60 floor, so a low confidence + // option had no loosening effect). const classification = await this.smartExtractor.extract(candidate.text, { definition: candidate.context, allTerms: [candidate.text, candidate.context] - }) + }, minConfidence) - // Skip if SmartExtractor returns null (low confidence) or below threshold - if (!classification || classification.confidence < minConfidence) { + // SmartExtractor already gates at minConfidence; this guards against null only. + if (!classification) { continue } diff --git a/src/neural/signals/EmbeddingSignal.ts b/src/neural/signals/EmbeddingSignal.ts index a14207fd..154d7e71 100644 --- a/src/neural/signals/EmbeddingSignal.ts +++ b/src/neural/signals/EmbeddingSignal.ts @@ -43,7 +43,10 @@ export interface EmbeddingSignalOptions { minConfidence?: number // Minimum confidence threshold (default: 0.60) checkGraph?: boolean // Check against graph entities (default: true) checkHistory?: boolean // Check against historical data (default: true) - timeout?: number // Max time in ms (default: 100) + timeout?: number // Max time in ms (default: 2000). A real transformer embed of a single + // candidate is commonly 50–200ms (cold/under load higher); the previous + // 100ms default silently timed out, dropping the neural signal entirely — + // which alone left concept extraction (no regex fallback) returning nothing. cacheSize?: number // LRU cache size (default: 1000) } @@ -110,7 +113,7 @@ export class EmbeddingSignal { minConfidence: options?.minConfidence ?? 0.60, checkGraph: options?.checkGraph ?? true, checkHistory: options?.checkHistory ?? true, - timeout: options?.timeout ?? 100, + timeout: options?.timeout ?? 2000, cacheSize: options?.cacheSize ?? 1000 } } diff --git a/src/types/brainy.types.ts b/src/types/brainy.types.ts index 0265b5ca..ef49b6b0 100644 --- a/src/types/brainy.types.ts +++ b/src/types/brainy.types.ts @@ -122,6 +122,14 @@ export interface Result { semanticScore?: number // Semantic similarity score (0-1) matchSource?: 'text' | 'semantic' | 'both' // Where this result came from rrfScore?: number // Raw RRF fusion score (for advanced ranking analysis) + + // Aggregation: present only on rows returned by find({ aggregate }). These mirror the + // documented AggregateResult shape so callers can read group/metric data at the top level + // instead of digging into `metadata`. (The same values are also flattened into `metadata` + // for backward compatibility.) + groupKey?: Record // Group-by key values for this aggregate row + metrics?: Record // Computed metric values (sum/avg/min/max/count) + count?: number // Total entity count in this group } /** diff --git a/tests/integration/advanced-apis-regression.test.ts b/tests/integration/advanced-apis-regression.test.ts new file mode 100644 index 00000000..1a7593e4 --- /dev/null +++ b/tests/integration/advanced-apis-regression.test.ts @@ -0,0 +1,131 @@ +/** + * Regression tests for the three advanced-API defects reported as BR-ADV-FEATURES-BUN: + * + * 1. Entity/concept extraction returned `[]` for entity-rich text. Root cause: the + * SmartExtractor ensemble combined agreeing signals with a weighted *sum* compared against + * an absolute gate, so a confident-but-low-weight signal was discarded whenever a + * higher-weight signal voted a different (lower-confidence) type. Fixed by selecting and + * gating on a normalized weighted *average*. + * 2. `find({ aggregate })` rows did not expose the documented AggregateResult fields + * (`groupKey`/`metrics`/`count`) at the top level, so callers saw "empty" rows. + * 3. `find({ connected: { depth } })` ignored `depth`/`via` and returned only the 1-hop + * neighbour. Fixed by delegating to the depth-aware `neighbors()` BFS. + * + * These run with REAL embeddings (no mocks) — the original failures were invisible under the + * zero-vector mock embeddings used by the unit suite. + */ + +import { describe, it, expect, beforeAll, afterAll } from 'vitest' +import { Brainy } from '../../src/brainy.js' +import { NounType, VerbType } from '../../src/types/graphTypes.js' + +describe('BR-ADV-FEATURES-BUN regression', () => { + let brain: Brainy + + beforeAll(async () => { + brain = new Brainy({ storage: { type: 'memory' } }) + await brain.init() + // Warm the embedding model so the first real extraction isn't paying cold-start latency. + await brain.embed('warmup') + }) + + afterAll(async () => { + await brain.close() + }) + + describe('Bug 1 — entity extraction is not empty', () => { + const text = 'Sarah Chen founded Acme Corp in New York' + + it('extractEntities returns the entity spans (was [])', async () => { + const entities = await brain.extractEntities(text, { confidence: 0.1 }) + expect(entities.length).toBeGreaterThan(0) + const texts = entities.map(e => e.text) + // The candidate spans must be surfaced (type accuracy on dense sentences is a separate + // pre-existing concern; this guards the reported "returns nothing" regression). + expect(texts).toContain('Sarah Chen') + expect(texts).toContain('Acme Corp') + }) + + it('a confident single-signal candidate classifies correctly in isolation', async () => { + // "Acme Corp" carries a strong Organization indicator ("Corp") with no competing context. + const entities = await brain.extractEntities('Acme Corp', { confidence: 0.6 }) + expect(entities.length).toBeGreaterThan(0) + expect(entities.some(e => e.type === NounType.Organization)).toBe(true) + }) + + it('the confidence option actually controls the gate (no longer dead)', async () => { + const loose = await brain.extractEntities(text, { confidence: 0.1 }) + const strict = await brain.extractEntities(text, { confidence: 0.99 }) + expect(loose.length).toBeGreaterThan(0) + // Previously confidence had no effect because SmartExtractor applied a hardcoded 0.60 + // floor; a near-1.0 threshold must now admit no fewer results than a loose one, and here + // strictly fewer (nothing scores ≥ 0.99). + expect(strict.length).toBeLessThanOrEqual(loose.length) + }) + }) + + describe('Bug 3 — multi-hop traversal honors depth', () => { + let a: string + let b: string + let c: string + + beforeAll(async () => { + a = await brain.add({ data: 'graph node A', type: NounType.Concept, metadata: { name: 'A' } }) + b = await brain.add({ data: 'graph node B', type: NounType.Concept, metadata: { name: 'B' } }) + c = await brain.add({ data: 'graph node C', type: NounType.Concept, metadata: { name: 'C' } }) + await brain.relate({ from: a, to: b, type: VerbType.RelatedTo }) + await brain.relate({ from: b, to: c, type: VerbType.RelatedTo }) + }) + + const names = (rows: Array<{ metadata?: any }>) => + rows.map(r => r.metadata?.name).filter(Boolean).sort() + + it('depth 1 returns only the immediate neighbour', async () => { + const rows = await brain.find({ connected: { from: a, depth: 1, direction: 'out' } }) + expect(names(rows)).toEqual(['B']) + }) + + it('depth 2 reaches the second hop (was 1-hop only)', async () => { + const rows = await brain.find({ connected: { from: a, depth: 2, direction: 'out' } }) + expect(names(rows)).toEqual(['B', 'C']) + }) + + it('depth 3 on a 2-edge chain still returns B and C', async () => { + const rows = await brain.find({ connected: { from: a, depth: 3, direction: 'out' } }) + expect(names(rows)).toEqual(['B', 'C']) + }) + }) + + describe('Bug 2 — find({ aggregate }) exposes AggregateResult fields', () => { + beforeAll(async () => { + brain.defineAggregate({ + name: 'spend_by_cat', + source: { type: NounType.Event }, + groupBy: ['category'], + metrics: { total: { op: 'sum', field: 'amount' }, count: { op: 'count' } } + }) + await brain.add({ data: 'coffee', type: NounType.Event, metadata: { category: 'food', amount: 5 } }) + await brain.add({ data: 'lunch', type: NounType.Event, metadata: { category: 'food', amount: 12 } }) + await brain.add({ data: 'bus', type: NounType.Event, metadata: { category: 'transit', amount: 3 } }) + await brain.flush() + await new Promise(r => setTimeout(r, 500)) // let materialization debounce settle + }) + + it('rows carry top-level groupKey / metrics / count (was hidden under metadata)', async () => { + const rows: any[] = await brain.find({ aggregate: 'spend_by_cat' }) + expect(rows.length).toBe(2) + + const food = rows.find(r => r.groupKey?.category === 'food') + const transit = rows.find(r => r.groupKey?.category === 'transit') + + expect(food).toBeDefined() + expect(food.groupKey).toEqual({ category: 'food' }) + expect(food.metrics.total).toBe(17) + expect(food.count).toBe(2) + + expect(transit).toBeDefined() + expect(transit.metrics.total).toBe(3) + expect(transit.count).toBe(1) + }) + }) +}) diff --git a/tests/unit/neural/NaturalLanguageProcessor.test.ts b/tests/unit/neural/NaturalLanguageProcessor.test.ts index 19f65b42..00f02012 100644 --- a/tests/unit/neural/NaturalLanguageProcessor.test.ts +++ b/tests/unit/neural/NaturalLanguageProcessor.test.ts @@ -118,10 +118,11 @@ describe('NaturalLanguageProcessor', () => { expect(extraction).toBeDefined() expect(Array.isArray(extraction)).toBe(true) - // Entity extraction uses neural matching with type embeddings - // Extraction quality depends on text context and entity similarity to known types - // For simple text without rich context, extraction may return empty array - // This is correct behavior - it's better to return nothing than false positives + // This unit suite runs with mock (zero-vector) embeddings, so the neural embedding + // signal can't fire here — only the array shape is asserted. Real, non-empty extraction + // is verified against actual embeddings in + // tests/integration/advanced-apis-regression.test.ts (BR-ADV-FEATURES-BUN). An empty + // result here reflects the mock embeddings, not expected production behaviour. }) it('should extract topics and concepts', async () => {