From 1a98e4276a6e15be6c2336fff822cb0a2ae6a88c Mon Sep 17 00:00:00 2001 From: David Snelling Date: Tue, 26 May 2026 13:55:43 -0700 Subject: [PATCH] feat: queryAggregate() + HAVING, plus aggregate backfill, traversal depth/via, extraction typing (BR-ADV-FEATURES-BUN) New report APIs: - brain.queryAggregate(name, params) returns the clean AggregateResult[] shape ({ groupKey, metrics, count }) directly, instead of the search-Result wrapper that find({ aggregate }) returns. - HAVING: find({ aggregate, having }) and queryAggregate filter groups by computed metric values (e.g. revenue > 1000), complementing where (which filters group keys). Evaluated per group: O(groups), independent of entity count. Fixes (all reproducible on Node; surfaced under Cortex+Bun by Memory): - Aggregate backfill-on-define: defining an aggregate over a store that already holds matching entities returned []. It now backfills from existing entities on first query (storage-agnostic via getNouns), so it works under durable backends that reopen pre-populated. groupBy:['noun'] resolves to the entity type; find({ aggregate }) rows expose groupKey/metrics/count at the top level. - Multi-hop find({ connected: { depth, via } }) honors depth and via at every hop (was 1-hop only, with verb filtering applied to hop 1 only); BFS bounded by limit. - extractEntities no longer bleeds a neighbour's type indicator across candidates; each candidate is typed by its own span. Also fixes the "Dr." title pattern. Adds real-embedding regression tests; full unit suite green. --- RELEASES.md | 41 ++++ docs/guides/aggregation.md | 56 +++++- docs/neural-extraction.md | 20 +- src/aggregation/AggregationIndex.ts | 68 ++++++- src/brainy.ts | 185 +++++++++++++----- src/coreTypes.ts | 8 + src/neural/signals/PatternSignal.ts | 86 +++++--- src/types/brainy.types.ts | 7 + .../advanced-apis-regression.test.ts | 106 +++++++++- 9 files changed, 481 insertions(+), 96 deletions(-) diff --git a/RELEASES.md b/RELEASES.md index 7eea67d7..3d3b243f 100644 --- a/RELEASES.md +++ b/RELEASES.md @@ -11,6 +11,47 @@ Collective. The SDK wraps it — most products never call Brainy directly. Read --- +## v7.23.0 — 2026-05-26 + +**Affected products:** anyone using aggregation (stats/dashboards), graph traversal, or entity +extraction. Adds two report APIs and fixes three correctness gaps surfaced by BR-ADV-FEATURES-BUN. +Drop-in upgrade from 7.22.x. + +### New — `brain.queryAggregate(name, params)` + +A first-class report API returning the clean `AggregateResult[]` shape +(`{ groupKey, metrics, count }[]`) directly, instead of the search-`Result` wrapper that +`find({ aggregate })` returns. Accepts `where` / `having` / `orderBy` / `order` / `limit` / `offset`. + +### New — HAVING (filter groups by metric value) + +`find({ aggregate, having: { revenue: { greaterThan: 1000 } } })` (and the same on +`queryAggregate`) filters groups by their computed metrics — the analytics equivalent of SQL +`HAVING`, complementing `where` (which filters group keys). Evaluated per group: **O(groups), +independent of entity count.** + +### Fix — aggregates now backfill when defined over existing data (R1) + +Defining an aggregate on a store that already holds matching entities returned `[]` because +write-time hooks only saw *future* writes. It now backfills from existing entities on first query +(one-time scan via `getNouns`, then incremental) — storage-agnostic, so it works under durable +backends (Cortex) where a brain reopens pre-populated. Also: `groupBy:['noun']` now resolves to +the entity type instead of a single null group. `find({ aggregate })` rows now expose +`groupKey`/`metrics`/`count` at the top level (previously only under `.metadata`). + +### Fix — multi-hop `find({ connected: { depth, via } })` (traversal) + +`depth` and `via` are now honored at **every hop** (previously only the immediate neighbour was +returned, and verb filtering applied to hop 1 only). The BFS is bounded by `limit`. + +### Fix — entity extraction type accuracy (R3) + +`extractEntities` no longer lets a type indicator in one candidate's surrounding text bleed onto +neighbours (e.g. "Corp" in "Sarah Chen founded Acme Corp" no longer types "Sarah Chen" as +Organization). Each candidate is typed by its own span; context may only reinforce the same type. + +--- + ## v7.22.1 — 2026-05-26 **Affected products:** Anyone using `extractEntities()` / `extractConcepts()`, native diff --git a/docs/guides/aggregation.md b/docs/guides/aggregation.md index 1b39edf5..d87fb8c6 100644 --- a/docs/guides/aggregation.md +++ b/docs/guides/aggregation.md @@ -8,6 +8,11 @@ Brainy's aggregation engine computes running totals at write time, so reading ag No batch jobs. No scheduled recalculations. Aggregates stay current with every write. +**Defining over existing data:** if you define an aggregate on a store that already holds +matching entities, Brainy backfills it from those entities on the first query (a one-time scan, +then purely incremental). So `defineAggregate()` behaves the same whether you define it before +or after the data exists — including when a persisted brain reopens already populated. + ## Quick Start ```typescript @@ -216,6 +221,22 @@ const foodOnly = await brain.find({ }) ``` +### Filter by Metric Value (HAVING) + +Use `having` to filter groups by their **computed metric values** — the analytics equivalent of +SQL `HAVING`. (`where` filters group *keys*; `having` filters *metrics*.) + +```typescript +const bigCategories = await brain.find({ + aggregate: 'sales_by_category', + having: { revenue: { greaterThan: 1000 } } +}) +``` + +`having` accepts the same operators as `where`, applied to each group's metric results plus +`count`. It is evaluated per group — **O(groups), independent of entity count** — before sorting +and pagination, so it stays cheap even over billions of entities. + ### Sort and Paginate Sort by any metric or group key field: @@ -248,24 +269,47 @@ const recentTopSpenders = await brain.find({ ### Result Format -Each result is returned as a `Result` with `type: NounType.Measurement`: +`find({ aggregate })` returns `Result` rows (for uniformity with the rest of `find()`), +with the aggregate fields surfaced **both** at the top level and, for backward compatibility, +flattened into `metadata`: ```typescript { id: string, score: 1.0, type: NounType.Measurement, - metadata: { + groupKey: { category: 'food' }, // top-level — the group key values + metrics: { revenue: 17.50, count: 2, average: 8.75 }, // top-level — computed metrics + count: 2, // top-level — entities in the group + metadata: { // legacy mirror of the same data __aggregate: 'sales_by_category', - category: 'food', // Group key values - revenue: 17.50, // Computed metrics - count: 2, - average: 8.75 + category: 'food', + revenue: 17.50, count: 2, average: 8.75 }, entity: Entity } ``` +### `queryAggregate()` — the report-friendly view + +For dashboards and reports, prefer `brain.queryAggregate(name, params)`. It returns the clean +`AggregateResult[]` shape directly — no search-result wrapper: + +```typescript +const rows = await brain.queryAggregate('sales_by_category', { + orderBy: 'revenue', + order: 'desc', + limit: 10 +}) +// [ +// { groupKey: { category: 'electronics' }, metrics: { revenue: 1200, count: 1, average: 1200 }, count: 1 }, +// { groupKey: { category: 'food' }, metrics: { revenue: 17.50, count: 2, average: 8.75 }, count: 2 } +// ] +``` + +It accepts the same `where` / `having` / `orderBy` / `order` / `limit` / `offset` params as the +`find({ aggregate })` form. + ## Source Filtering Control which entities feed into an aggregate with the `source` property. diff --git a/docs/neural-extraction.md b/docs/neural-extraction.md index 8f183339..989b1b60 100644 --- a/docs/neural-extraction.md +++ b/docs/neural-extraction.md @@ -30,12 +30,11 @@ const brain = new Brainy() await brain.init() // Extract all entities -const entities = await brain.extractEntities('John Smith founded Acme Corp in New York') -// Returns: +const entities = await brain.extractEntities('Sarah Chen founded Acme Corp') +// Returns (fast pattern + embedding ensemble; confidences are approximate): // [ -// { text: 'John Smith', type: NounType.Person, confidence: 0.95 }, -// { text: 'Acme Corp', type: NounType.Organization, confidence: 0.92 }, -// { text: 'New York', type: NounType.Location, confidence: 0.88 } +// { text: 'Sarah Chen', type: NounType.Person, confidence: 0.68 }, +// { text: 'Acme Corp', type: NounType.Organization, confidence: 0.85 } // ] // Extract with filters @@ -46,6 +45,17 @@ const people = await brain.extractEntities('...', { }) ``` +> **What this is (and isn't).** `extractEntities` is a fast, dependency-free **heuristic +> ensemble** (regex/pattern + type-embedding similarity + context), not a trained NER model. +> Each candidate is typed by its own span — a type indicator in a neighbour's text (e.g. "Corp" +> in "Sarah Chen founded Acme Corp") will **not** bleed onto another candidate. +> +> **Known limitation:** a bare proper-noun place name with no structural cue (e.g. "New York", +> no comma, state code, or "in"/"at" preposition handling) can be typed as `Person` by the +> generic full-name pattern. For high-precision typing, pass `types` to constrain results, supply +> richer context, or post-validate. For state-of-the-art NER, drive extraction from an LLM at the +> application layer and store the results in Brainy. + ### Method 2: Direct Import (Advanced) ```typescript diff --git a/src/aggregation/AggregationIndex.ts b/src/aggregation/AggregationIndex.ts index 71dd9e41..aaa4658d 100644 --- a/src/aggregation/AggregationIndex.ts +++ b/src/aggregation/AggregationIndex.ts @@ -208,6 +208,15 @@ export class AggregationIndex { /** Track which aggregates have dirty state needing persistence */ private dirty = new Set() + /** + * Aggregates whose state must be backfilled from entities that already existed + * when the aggregate was defined (or whose persisted state was missing/stale at + * init). Write-time hooks only capture entities added *after* a definition, so + * without backfill an aggregate defined over a populated store stays empty. + * Drained by the owner (Brainy) which has the entity iterator; see `getPendingBackfills`. + */ + private needsBackfill = new Set() + /** Track aggregates with stale MIN/MAX (need lazy recompute) */ private staleMinMax = new Map>() @@ -243,8 +252,10 @@ export class AggregationIndex { } this.states.set(def.name, groupMap) } else { - // Definition changed or no saved state — start fresh (will be rebuilt) + // Definition changed or no saved state — start fresh and backfill from + // existing entities (the owner drains needsBackfill on first query). this.states.set(def.name, new Map()) + this.needsBackfill.add(def.name) } this.definitionHashes.set(def.name, currentHash) @@ -332,9 +343,12 @@ export class AggregationIndex { this.definitions.set(def.name, def) this.definitionHashes.set(def.name, newHash) - // Reset state if definition changed or doesn't exist yet + // Reset state if definition changed or doesn't exist yet, and flag it for + // backfill so already-stored entities are counted (write-time hooks only see + // future writes). The owner drains this on the next query via getPendingBackfills(). if (!this.states.has(def.name) || (oldHash && oldHash !== newHash)) { this.states.set(def.name, new Map()) + this.needsBackfill.add(def.name) } // Notify native provider of definition (caches compiled form for hot path) @@ -376,6 +390,50 @@ export class AggregationIndex { return this.definitions.has(name) } + // ============= Backfill ============= + // + // Write-time hooks only capture entities added after a definition exists, so an + // aggregate defined over a populated store would stay empty. The owner (Brainy) has + // the entity iterator, so backfill is driven from there: it reads the pending set, + // clears the aggregate, streams every existing entity through `backfillEntity`, then + // calls `finishBackfill`. Clearing first means a concurrent write that landed via the + // incremental hook is wiped and re-counted exactly once by the rescan. + + /** Names of aggregates whose state must be (re)built from existing entities. */ + getPendingBackfills(): string[] { + return Array.from(this.needsBackfill) + } + + /** Clear an aggregate's state so a full rescan cannot double-count. */ + beginBackfill(name: string): void { + this.states.set(name, new Map()) + // Reset native provider state for this aggregate too, if present. + const def = this.definitions.get(name) + if (def && this.nativeProvider?.removeAggregate && this.nativeProvider?.defineAggregate) { + this.nativeProvider.removeAggregate(name) + this.nativeProvider.defineAggregate(def) + } + } + + /** Feed one already-stored entity into a single aggregate during backfill. */ + backfillEntity(name: string, entity: Record): void { + if (isAggregateEntity(entity)) return + const def = this.definitions.get(name) + if (!def || !matchesSource(entity, def.source)) return + + if (this.nativeProvider) { + this.applyNativeResults(name, this.nativeProvider.incrementalUpdate(name, def, entity, 'add')) + } else { + this.addContribution(name, def, entity) + } + } + + /** Mark an aggregate's backfill complete; rebuilt state persists on next flush(). */ + finishBackfill(name: string): void { + this.needsBackfill.delete(name) + this.dirty.add(name) + } + // ============= Write-Time Hooks ============= /** @@ -549,6 +607,12 @@ export class AggregationIndex { } } + // HAVING: filter groups by computed metric values (post-compute, O(groups), before + // sort/pagination). Reuses the where-operator engine over metrics + `count`. + if (params.having && Object.keys(params.having).length > 0) { + if (!matchesMetadataFilter({ ...metrics, count: totalCount } as any, params.having)) continue + } + results.push({ groupKey: { ...group.groupKey }, metrics, diff --git a/src/brainy.ts b/src/brainy.ts index 8d1245af..24c91fbf 100644 --- a/src/brainy.ts +++ b/src/brainy.ts @@ -2238,6 +2238,35 @@ export class Brainy implements BrainyInterface { } } + /** + * Query a named aggregate, returning the documented `AggregateResult[]` shape + * (`{ groupKey, metrics, count }`) directly — the report-friendly view. + * + * `find({ aggregate })` returns the same data wrapped as search `Result` rows (with + * `score`/`type`/`entity`) for uniformity with the rest of `find()`; this method is the + * first-class analytics path for dashboards and reports. + * + * @param name - Aggregate name (must be defined via `defineAggregate`) + * @param params - Optional `where` (group-key filter), `having` (metric filter), `orderBy`, `order`, `limit`, `offset` + * @returns Computed group rows + * + * @example + * const rows = await brain.queryAggregate('sales_by_category', { orderBy: 'revenue', order: 'desc', limit: 10 }) + * // [{ groupKey: { category: 'food' }, metrics: { revenue: 17.5, count: 2 }, count: 2 }, ...] + */ + async queryAggregate( + name: string, + params?: Omit + ): Promise { + await this.ensureInitialized() + this.ensureAggregationIndex() + if (!this._aggregationIndex!.hasAggregate(name)) { + throw new Error(`Aggregate '${name}' is not defined. Call defineAggregate() first.`) + } + await this.backfillAggregateIfNeeded(name) + return this._aggregationIndex!.queryAggregate({ name, ...params }) + } + /** * Lazily create the AggregationIndex on first use. * Checks for a native 'aggregation' provider from plugins. @@ -4479,12 +4508,19 @@ export class Brainy implements BrainyInterface { * @param options - Extraction options * @returns Array of extracted entities with types and confidence * + * Fast heuristic ensemble (pattern + type-embedding + context), not a trained NER — + * each candidate is typed by its own span (no cross-candidate bleed). Confidences are + * approximate; pass `types` to constrain results when precision matters. + * + * @param text - Text to extract entities from + * @param options - Extraction options + * @returns Array of extracted entities with types and confidence + * * @example - * const entities = await brain.extract('John Smith founded Acme Corp in New York') + * const entities = await brain.extract('Sarah Chen founded Acme Corp') * // [ - * // { text: 'John Smith', type: NounType.Person, confidence: 0.95 }, - * // { text: 'Acme Corp', type: NounType.Organization, confidence: 0.92 }, - * // { text: 'New York', type: NounType.Location, confidence: 0.88 } + * // { text: 'Sarah Chen', type: NounType.Person, confidence: 0.68 }, + * // { text: 'Acme Corp', type: NounType.Organization, confidence: 0.85 } * // ] */ async extract( @@ -6275,7 +6311,7 @@ export class Brainy implements BrainyInterface { options?: { direction?: 'outgoing' | 'incoming' | 'both' depth?: number - verbType?: VerbType + verbType?: VerbType | VerbType[] limit?: number } ): Promise { @@ -6283,66 +6319,43 @@ export class Brainy implements BrainyInterface { const direction = options?.direction || 'both' const limit = options?.limit + const verbTypes = options?.verbType === undefined + ? undefined + : new Set(Array.isArray(options.verbType) ? options.verbType : [options.verbType]) // Map our API direction to graphIndex direction const graphDirection = direction === 'outgoing' ? 'out' : direction === 'incoming' ? 'in' : 'both' - // Get neighbors from graph index - let neighbors = await this.graphIndex.getNeighbors(entityId, { - direction: graphDirection, - limit - }) + let neighbors = await this.getTypedNeighbors(entityId, graphDirection, verbTypes, limit) - // Filter by verb type if specified - if (options?.verbType) { - const filteredNeighbors: string[] = [] - const verbIds = direction !== 'incoming' - ? await this.graphIndex.getVerbIdsBySource(entityId) - : await this.graphIndex.getVerbIdsByTarget(entityId) - - // Load verbs to check their types - const verbs = await this.graphIndex.getVerbsBatchCached(verbIds) - - for (const [, verb] of verbs) { - if (verb.type === options.verbType || verb.verb === options.verbType) { - const neighborId = verb.sourceId === entityId ? verb.targetId : verb.sourceId - if (neighbors.includes(neighborId)) { - filteredNeighbors.push(neighborId) - } - } - } - - neighbors = filteredNeighbors - } - - // Handle depth > 1 (multi-hop traversal) + // Handle depth > 1 (multi-hop traversal). The verb-type filter is applied at EVERY hop + // (previously only the first), and the BFS is bounded by `limit` so a dense graph can't + // expand without limit before the final slice. if (options?.depth && options.depth > 1) { const visited = new Set([entityId, ...neighbors]) let currentLevel = neighbors for (let d = 1; d < options.depth; d++) { + if (limit && neighbors.length >= limit) break const nextLevel: string[] = [] - for (const nodeId of currentLevel) { - const nodeNeighbors = await this.graphIndex.getNeighbors(nodeId, { - direction: graphDirection - }) - + outer: for (const nodeId of currentLevel) { + const nodeNeighbors = await this.getTypedNeighbors(nodeId, graphDirection, verbTypes) for (const neighbor of nodeNeighbors) { if (!visited.has(neighbor)) { visited.add(neighbor) nextLevel.push(neighbor) + neighbors.push(neighbor) + if (limit && neighbors.length >= limit) break outer } } } if (nextLevel.length === 0) break currentLevel = nextLevel - neighbors.push(...nextLevel) } - // Apply limit after multi-hop traversal if (limit && neighbors.length > limit) { neighbors = neighbors.slice(0, limit) } @@ -6351,6 +6364,39 @@ export class Brainy implements BrainyInterface { return neighbors } + /** + * Neighbours of a single node, optionally filtered to a set of verb types. + * + * Extracted so depth traversal can apply the verb-type filter at every hop (not just the + * first). For `direction: 'both'` the verb scan unions out-edges and in-edges so the filter + * isn't silently limited to outgoing edges. + */ + private async getTypedNeighbors( + nodeId: string, + graphDirection: 'in' | 'out' | 'both', + verbTypes?: Set, + limit?: number + ): Promise { + const neighbors = await this.graphIndex.getNeighbors(nodeId, { direction: graphDirection, limit }) + if (!verbTypes || verbTypes.size === 0) return neighbors + + // Gather candidate edges in the relevant direction(s). + const verbIds: string[] = [] + if (graphDirection !== 'in') verbIds.push(...await this.graphIndex.getVerbIdsBySource(nodeId)) + if (graphDirection !== 'out') verbIds.push(...await this.graphIndex.getVerbIdsByTarget(nodeId)) + + const verbs = await this.graphIndex.getVerbsBatchCached(verbIds) + const neighborSet = new Set(neighbors) + const filtered: string[] = [] + for (const [, verb] of verbs) { + if (verbTypes.has(verb.type as VerbType) || verbTypes.has(verb.verb as VerbType)) { + const neighborId = verb.sourceId === nodeId ? verb.targetId : verb.sourceId + if (neighborSet.has(neighborId)) filtered.push(neighborId) + } + } + return filtered + } + /** * Find semantic duplicates in the database * @@ -6805,18 +6851,11 @@ export class Brainy implements BrainyInterface { const toNeighborDir = (d: 'in' | 'out' | 'both'): 'incoming' | 'outgoing' | 'both' => d === 'in' ? 'incoming' : d === 'out' ? 'outgoing' : 'both' - // Collect reachable ids honoring depth + verb-type filter. `via` may be a single VerbType - // or an array; reuse the depth-aware BFS in neighbors() (one pass per requested verb type). - const collect = async (id: string, dir: 'incoming' | 'outgoing' | 'both'): Promise => { - const verbTypes: Array = - via === undefined ? [undefined] : Array.isArray(via) ? via : [via] - const ids = new Set() - for (const verbType of verbTypes) { - const hops = await this.neighbors(id, { direction: dir, depth: depth ?? 1, verbType }) - for (const n of hops) ids.add(n) - } - return [...ids] - } + // Collect reachable ids honoring depth + verb-type filter via the depth-aware BFS in + // neighbors(), which takes the whole verb-type set and filters every hop against it — so a + // mixed-verb path (a-[likes]->b-[knows]->c with via:[likes,knows]) is followed correctly. + const collect = (id: string, dir: 'incoming' | 'outgoing' | 'both'): Promise => + this.neighbors(id, { direction: dir, depth: depth ?? 1, verbType: via }) const connectedIds = new Set() @@ -7909,6 +7948,10 @@ export class Brainy implements BrainyInterface { aggParams.offset = params.offset } + // Backfill from already-stored entities if this aggregate was defined over a + // populated store (write-time hooks only capture entities added after define). + await this.backfillAggregateIfNeeded(aggParams.name) + const aggregateResults = this._aggregationIndex.queryAggregate(aggParams) // Convert AggregateResult[] to Result[] for API consistency @@ -7946,6 +7989,44 @@ export class Brainy implements BrainyInterface { }) } + /** + * Backfill a named aggregate from entities already in storage. + * + * Write-time hooks (`onEntityAdded` etc.) only capture entities added *after* an + * aggregate is defined, so an aggregate defined over a populated store — the common + * case under durable storage, where a brain reopens pre-populated — would otherwise + * return `[]`. On first query we clear the aggregate's state and stream every stored + * noun back through it. Storage-agnostic: `getNouns()` works for in-memory, the + * filesystem adapter, and native (Cortex) storage alike. One-time per definition — + * the rebuilt state is persisted on flush() and reloaded on the next session. + */ + private async backfillAggregateIfNeeded(name: string): Promise { + const index = this._aggregationIndex + if (!index || !index.getPendingBackfills().includes(name)) return + + index.beginBackfill(name) + + const PAGE = 500 + let offset = 0 + let cursor: string | undefined + for (;;) { + const page = await this.storage.getNouns({ + pagination: cursor ? { limit: PAGE, cursor } : { limit: PAGE, offset } + }) + for (const noun of page.items) { + index.backfillEntity(name, noun as unknown as Record) + } + if (!page.hasMore || page.items.length === 0) break + if (page.nextCursor) { + cursor = page.nextCursor + } else { + offset += page.items.length + } + } + + index.finishBackfill(name) + } + /** * Close and cleanup * diff --git a/src/coreTypes.ts b/src/coreTypes.ts index f8a67144..f3093750 100644 --- a/src/coreTypes.ts +++ b/src/coreTypes.ts @@ -279,6 +279,14 @@ export function resolveEntityField( entity: HNSWNounWithMetadata, field: string ): unknown { + // `noun` is an alias for the entity's type — some shapes carry `noun`, the + // canonical HNSWNounWithMetadata carries `type`. Resolve it consistently so + // groupBy/source/sort on 'noun' isn't silently null (it otherwise falls through + // to metadata.noun, which doesn't exist). Mirrors matchesSource's `type ?? noun`. + if (field === 'noun') { + const e = entity as unknown as Record + return e.type ?? e.noun + } if (STANDARD_ENTITY_FIELDS.has(field)) { return (entity as unknown as Record)[field] } diff --git a/src/neural/signals/PatternSignal.ts b/src/neural/signals/PatternSignal.ts index f2ab1ee2..df4dc016 100644 --- a/src/neural/signals/PatternSignal.ts +++ b/src/neural/signals/PatternSignal.ts @@ -135,7 +135,7 @@ export class PatternSignal { // Person patterns - SPECIFIC INDICATORS (high confidence) this.addPatterns(NounType.Person, 0.82, [ - /\b(?:Dr|Prof|Mr|Mrs|Ms|Sir|Lady|Lord)\s+[A-Z][a-z]+/, // Titles + /\b(?:Dr|Prof|Mr|Mrs|Ms|Sir|Lady|Lord)\.?\s+[A-Z][a-z]+/, // Titles (optional period: "Dr." / "Dr") /\b(?:CEO|CTO|CFO|COO|VP|Director|Manager|Engineer|Developer|Designer)\b/i, // Roles /\b(?:author|creator|founder|inventor|contributor|maintainer)\b/i ]) @@ -294,7 +294,8 @@ export class PatternSignal { return cached } - // Try regex patterns (primary method) + // Try regex patterns (primary method). The candidate span is authoritative for its + // type; context can only reinforce the same type, never override it (see matchRegexPatterns). const regexMatch = this.matchRegexPatterns(candidate, context?.definition) if (regexMatch && regexMatch.confidence >= this.options.minConfidence) { this.stats.regexMatches++ @@ -328,49 +329,76 @@ export class PatternSignal { } /** - * Match against pre-compiled regex patterns + * Match against pre-compiled regex patterns. * - * Checks candidate and optional definition text + * Matches the CANDIDATE SPAN ONLY — never the surrounding context. Folding the + * candidate's own span decides the type; the surrounding `definition`/context may only + * REINFORCE that same type (small boost), never override it with a different one. + * + * This is the fix for cross-candidate type bleed: folding the 30-char context window into + * one match let an indicator in a neighbour's span (e.g. "Corp" in "Sarah Chen founded + * Acme Corp") flip "Sarah Chen" to Organization. Now a different-type context match is + * ignored. When the candidate has no pattern of its own, the context's type is used as a + * fallback — a descriptive cue (a role, "conference", …) still classifies an otherwise + * shapeless candidate, which is the legitimate use of context the ContextSignal complements. */ private matchRegexPatterns( candidate: string, definition?: string ): TypeSignal | null { - const textToMatch = definition ? `${candidate} ${definition}` : candidate - const matches: Array<{ pattern: CompiledPattern, count: number }> = [] + const candidateMatch = this.bestPatternMatch(candidate) + const contextMatch = definition ? this.bestPatternMatch(definition) : null - // Check all patterns - for (const pattern of this.patterns) { - const matchCount = (textToMatch.match(pattern.regex) || []).length - if (matchCount > 0) { - matches.push({ pattern, count: matchCount }) + if (candidateMatch) { + const chosen = candidateMatch.pattern + let confidence = Math.min(chosen.confidence, 0.85) // Cap at 0.85 + let evidence = `Pattern match: ${chosen.name} (${candidateMatch.count} occurrence${candidateMatch.count > 1 ? 's' : ''})` + + // Context of the SAME type reinforces (use the stronger evidence + a small agreement + // boost); a different-type context match is deliberately ignored — no override. + if (contextMatch && contextMatch.pattern.type === chosen.type) { + confidence = Math.min(Math.max(confidence, contextMatch.pattern.confidence) + 0.05, 0.95) + evidence += ' + context' + } + + return { + source: 'pattern-regex', + type: chosen.type, + confidence, + evidence, + metadata: { patternName: chosen.name, matchCount: candidateMatch.count } } } - if (matches.length === 0) return null + // Candidate has no shape of its own — fall back to the context's descriptive cue. + if (contextMatch) { + const chosen = contextMatch.pattern + return { + source: 'pattern-regex', + type: chosen.type, + confidence: Math.min(chosen.confidence, 0.85), + evidence: `Context pattern match: ${chosen.name} (${contextMatch.count} occurrence${contextMatch.count > 1 ? 's' : ''})`, + metadata: { patternName: chosen.name, matchCount: contextMatch.count } + } + } - // Find best match (highest confidence * match count) - let best = matches[0] - let bestScore = best.pattern.confidence * Math.log(best.count + 1) + return null + } - for (const match of matches.slice(1)) { - const score = match.pattern.confidence * Math.log(match.count + 1) + /** Best pattern match (ranked by confidence × log(count+1)) for a single text span. */ + private bestPatternMatch(text: string): { pattern: CompiledPattern, count: number } | null { + let best: { pattern: CompiledPattern, count: number } | null = null + let bestScore = -1 + for (const pattern of this.patterns) { + const count = (text.match(pattern.regex) || []).length + if (count === 0) continue + const score = pattern.confidence * Math.log(count + 1) if (score > bestScore) { - best = match + best = { pattern, count } bestScore = score } } - - return { - source: 'pattern-regex', - type: best.pattern.type, - confidence: Math.min(best.pattern.confidence, 0.85), // Cap at 0.85 - evidence: `Pattern match: ${best.pattern.name} (${best.count} occurrence${best.count > 1 ? 's' : ''})`, - metadata: { - patternName: best.pattern.name, - matchCount: best.count - } - } + return best } /** diff --git a/src/types/brainy.types.ts b/src/types/brainy.types.ts index ef49b6b0..177456e9 100644 --- a/src/types/brainy.types.ts +++ b/src/types/brainy.types.ts @@ -783,6 +783,13 @@ export interface AggregateQueryParams { name: string /** Filter aggregate groups by their key values */ where?: Record + /** + * Filter groups by their computed METRIC values (SQL HAVING). Same BFO operators as + * `where`, but applied to the derived metric results plus `count`, e.g. + * `{ revenue: { greaterThan: 1000 } }`. Evaluated per group (O(groups), independent of + * entity count), before sort/pagination. + */ + having?: Record /** Sort by metric name or group key field */ orderBy?: string /** Sort direction */ diff --git a/tests/integration/advanced-apis-regression.test.ts b/tests/integration/advanced-apis-regression.test.ts index 1a7593e4..19bf1d6f 100644 --- a/tests/integration/advanced-apis-regression.test.ts +++ b/tests/integration/advanced-apis-regression.test.ts @@ -40,12 +40,20 @@ describe('BR-ADV-FEATURES-BUN regression', () => { const entities = await brain.extractEntities(text, { confidence: 0.1 }) expect(entities.length).toBeGreaterThan(0) const texts = entities.map(e => e.text) - // The candidate spans must be surfaced (type accuracy on dense sentences is a separate - // pre-existing concern; this guards the reported "returns nothing" regression). expect(texts).toContain('Sarah Chen') expect(texts).toContain('Acme Corp') }) + it('does not bleed a neighbour\'s type indicator across candidates (R3)', async () => { + // "Corp" belongs to "Acme Corp" — it must not flip "Sarah Chen" to Organization. + const entities = await brain.extractEntities(text, { confidence: 0.1 }) + const byText = (t: string) => entities.find(e => e.text === t) + expect(byText('Sarah Chen')?.type).toBe(NounType.Person) + expect(byText('Acme Corp')?.type).toBe(NounType.Organization) + // (Type accuracy for "New York" → Location needs semantic/gazetteer support and is a + // documented heuristic limitation, not the reported context-bleed bug.) + }) + it('a confident single-signal candidate classifies correctly in isolation', async () => { // "Acme Corp" carries a strong Organization indicator ("Corp") with no competing context. const entities = await brain.extractEntities('Acme Corp', { confidence: 0.6 }) @@ -128,4 +136,98 @@ describe('BR-ADV-FEATURES-BUN regression', () => { expect(transit.count).toBe(1) }) }) + + describe('R1 — aggregate backfills over a pre-populated store', () => { + it('defineAggregate AFTER entities exist still returns correct rows', async () => { + // The durable-storage case: a brain reopens pre-populated, then an aggregate is + // defined. Write-time hooks never saw these entities, so without backfill this is []. + const b: any = new Brainy({ storage: { type: 'memory' } }) + await b.init() + await b.add({ data: 'a', type: NounType.Event, metadata: { category: 'food', amount: 5 } }) + await b.add({ data: 'b', type: NounType.Event, metadata: { category: 'food', amount: 7 } }) + await b.add({ data: 'c', type: NounType.Event, metadata: { category: 'transit', amount: 3 } }) + await b.flush() + + b.defineAggregate({ + name: 'after', + source: { type: NounType.Event }, + groupBy: ['category'], + metrics: { total: { op: 'sum', field: 'amount' }, count: { op: 'count' } } + }) + + const rows: any[] = await b.find({ aggregate: 'after' }) + const food = rows.find(r => r.groupKey?.category === 'food') + const transit = rows.find(r => r.groupKey?.category === 'transit') + expect(food?.metrics.total).toBe(12) + expect(food?.count).toBe(2) + expect(transit?.count).toBe(1) + await b.close() + }) + + it('groupBy "noun" resolves to the entity type, not null', async () => { + const b: any = new Brainy({ storage: { type: 'memory' } }) + await b.init() + await b.add({ data: 'p', type: NounType.Person }) + b.defineAggregate({ + name: 'byNoun', + source: { type: NounType.Person }, + groupBy: ['noun'], + metrics: { count: { op: 'count' } } + }) + const rows: any[] = await b.find({ aggregate: 'byNoun' }) + expect(rows.length).toBe(1) + expect(rows[0].groupKey.noun).toBe(NounType.Person) + await b.close() + }) + }) + + describe('Traversal — via filter applies at every hop', () => { + it('depth-2 via traversal follows the typed chain to the second hop', async () => { + const b: any = new Brainy({ storage: { type: 'memory' } }) + await b.init() + const a = await b.add({ data: 'a', type: NounType.Concept, metadata: { name: 'A' } }) + const m = await b.add({ data: 'm', type: NounType.Concept, metadata: { name: 'M' } }) + const z = await b.add({ data: 'z', type: NounType.Concept, metadata: { name: 'Z' } }) + await b.relate({ from: a, to: m, type: VerbType.RelatedTo }) + await b.relate({ from: m, to: z, type: VerbType.RelatedTo }) + + const viaMatch = await b.find({ connected: { from: a, depth: 2, direction: 'out', via: VerbType.RelatedTo } }) + expect(viaMatch.map((x: any) => x.metadata?.name).sort()).toEqual(['M', 'Z']) + + // A non-matching verb type filters out hop 1 (and therefore everything) — proving the + // filter is honored, not silently ignored as it was before. + const viaMiss = await b.find({ connected: { from: a, depth: 2, direction: 'out', via: VerbType.Contains } }) + expect(viaMiss.length).toBe(0) + await b.close() + }) + }) + + describe('queryAggregate() + HAVING', () => { + it('returns AggregateResult[] and filters groups by metric value', async () => { + const b: any = new Brainy({ storage: { type: 'memory' } }) + await b.init() + b.defineAggregate({ + name: 'rev', + source: { type: NounType.Event }, + groupBy: ['cat'], + metrics: { total: { op: 'sum', field: 'amt' }, count: { op: 'count' } } + }) + await b.add({ data: '1', type: NounType.Event, metadata: { cat: 'a', amt: 1500 } }) + await b.add({ data: '2', type: NounType.Event, metadata: { cat: 'a', amt: 200 } }) + await b.add({ data: '3', type: NounType.Event, metadata: { cat: 'b', amt: 50 } }) + await b.flush() + + const all = await b.queryAggregate('rev', { orderBy: 'total', order: 'desc' }) + expect(all.length).toBe(2) + expect(all[0].groupKey.cat).toBe('a') // highest total first + expect(all[0].metrics.total).toBe(1700) + expect(all[0].count).toBe(2) + + // HAVING: only groups whose computed total exceeds 1000 + const big = await b.queryAggregate('rev', { having: { total: { greaterThan: 1000 } } }) + expect(big.length).toBe(1) + expect(big[0].groupKey.cat).toBe('a') + await b.close() + }) + }) })