From 945d92d29e64ec84bee370b22692637ffed31e4c Mon Sep 17 00:00:00 2001 From: David Snelling Date: Sun, 19 Jul 2026 10:54:36 -0700 Subject: [PATCH] fix: one field-resolution law across aggregation hooks, source.where, removeMany, and find() spellings MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Four fixes from a consumer conformance report, one root disease — two field-resolution regimes where there must be one: - The delete/update aggregation hooks fed the engine a partial entity view (type/service/data/metadata only), so a reserved-field groupBy (subtype, visibility, ...) resolved to a nonexistent group on the way down: counts drifted upward forever after deletes, and updates moving an entity between reserved-field groups double-counted. The hooks now pass the full-fidelity view via entityForAggFromRawRecord (every reserved field top-level, mirroring the add path); the update sites pass the full get() view instead of a hand-rolled subset. - Aggregation source.where resolved fields only against the custom metadata bag, so where on a reserved field silently matched nothing. The matcher now resolves each filtered field through resolveEntityField — the same single source of truth groupBy uses. - removeMany() with no usable selector (bare array passed positionally, empty params, ids: []) resolved successfully having deleted nothing. All three now throw; the two legacy tests that pinned the silent no-op as 'graceful' now pin the refusal. - find() where keys accept both spellings: a metadata.-prefixed key falls back to its flattened spelling when the prefixed one is not indexed (metadata is flattened at index time). A literal nested custom key named metadata still wins when indexed as spelled. Five regression pins in aggregate-reserved-fields.test.ts (4 of 5 vary red on the unfixed code). --- RELEASES.md | 26 +++ src/aggregation/AggregationIndex.ts | 14 +- src/brainy.ts | 104 +++++++---- src/utils/metadataIndex.ts | 21 ++- .../aggregate-reserved-fields.test.ts | 169 ++++++++++++++++++ .../metadata-index-cleanup.unit.test.ts | 8 +- tests/unit/brainy/batch-operations.test.ts | 6 +- 7 files changed, 303 insertions(+), 45 deletions(-) create mode 100644 tests/integration/aggregate-reserved-fields.test.ts diff --git a/RELEASES.md b/RELEASES.md index fd9d64ec..d1374d5a 100644 --- a/RELEASES.md +++ b/RELEASES.md @@ -10,6 +10,32 @@ Full auto-generated changelog: `CHANGELOG.md` · Releases: https://github.com/so --- +## v8.8.2 — 2026-07-19 (one field-resolution law: reserved-field aggregates stop drifting) + +Four fixes from a consumer conformance audit, all rooted in the same disease — two field-resolution +regimes where there must be one: + +- **Aggregates grouped by a RESERVED field (`subtype`, `visibility`, …) now decrement on + delete.** The delete/update hooks fed the aggregation engine a partial entity view (type, + service, data, metadata only), so a reserved-field `groupBy` resolved to a nonexistent group + on the way DOWN — counts drifted upward forever after any delete, and updates that moved an + entity between reserved-field groups double-counted it. The hooks now pass the full-fidelity + entity view (every reserved field top-level, the same shape the add path uses). If your + deployment derives stats from reserved-field aggregates, re-define those aggregates once + after upgrading (a changed definition triggers one rescan) or run them fresh — the drifted + persisted counts do not self-heal retroactively. +- **Aggregation `source.where` on reserved fields now filters** instead of silently matching + nothing: the matcher resolves fields through the same resolver `groupBy` uses (top-level + standard fields + custom metadata), so `where: { subtype: 'note' }` means what it says. +- **`removeMany()` refuses empty/invalid selectors loudly.** A bare array passed positionally + (`removeMany([id])` instead of `removeMany({ ids: [id] })`), an empty params object, or + `ids: []` used to resolve successfully having deleted nothing. All three now throw. +- **`find()` accepts both where-key spellings.** Metadata is flattened at index time + (`metadata.entry.title` indexes as `entry.title`); a `metadata.`-prefixed where key now + falls back to its flattened spelling when the prefixed one isn't indexed — the + "unindexed field(s), returning []" confusion for storage-shaped spellings is gone. (A + literal nested custom key named `metadata` still wins when indexed as spelled.) + ## v8.8.1 — 2026-07-18 (flush no longer walks the whole generation history + the import dedup off-switch is now honest) ### The flush-storm fix (production incident, reported by a long-running deployment) diff --git a/src/aggregation/AggregationIndex.ts b/src/aggregation/AggregationIndex.ts index 7f7ffb27..f9382218 100644 --- a/src/aggregation/AggregationIndex.ts +++ b/src/aggregation/AggregationIndex.ts @@ -88,10 +88,18 @@ function matchesSource(entity: Record, source: AggregateDefinit if (entity.service !== source.service) return false } - // Metadata where filter — match against the entity's metadata sub-object + // Where filter — resolve each filtered field through resolveEntityField, + // the SAME single source of truth groupBy uses (top-level standard fields + // + custom metadata). Matching only the metadata sub-object made + // where:{subtype}/{visibility}/… a silent no-op: reserved fields never + // live in the custom bag, so those filters could never match anything. if (source.where && Object.keys(source.where).length > 0) { - const metadata = (entity.metadata ?? entity) as Record - if (!matchesMetadataFilter(metadata, source.where)) return false + const e = entity as unknown as HNSWNounWithMetadata + const resolved: Record = {} + for (const key of Object.keys(source.where)) { + resolved[key] = resolveEntityField(e, key) + } + if (!matchesMetadataFilter(resolved, source.where)) return false } return true diff --git a/src/brainy.ts b/src/brainy.ts index 008733b9..21eab679 100644 --- a/src/brainy.ts +++ b/src/brainy.ts @@ -1883,6 +1883,29 @@ export class Brainy implements BrainyInterface { } } + /** + * @description Build the AGGREGATION view of an entity from a stored flat + * metadata record — EVERY reserved field mapped to its top-level entity + * name (stored `noun` → `type`), custom metadata in `metadata`. This must + * mirror the add-path `entityForIndexing` shape exactly: the aggregation + * engine resolves groupBy/where fields via `resolveEntityField` + * (top-level standard fields + custom metadata), so a view that drops a + * reserved field makes every aggregate grouped by that field decrement a + * group that does not exist — counts then drift upward forever after + * deletes (SELF-AGGREGATE-DELETE-DRIFT). Do not hand-roll subsets of this. + * @param record - The stored flat metadata record (before-image or pre-delete read). + * @returns The full-fidelity entity view for aggregation hooks. + */ + private entityForAggFromRawRecord(record: Record): Record { + const { reserved, custom } = splitNounMetadataRecord(record) + const { noun, ...rest } = reserved + return { + type: noun, + ...rest, + metadata: custom + } + } + /** * @description Add an entity (noun) to the brain. Embeds `data` into a vector and * indexes the entity across all three intelligences — vector similarity, graph @@ -3125,15 +3148,16 @@ export class Brainy implements BrainyInterface { ] : undefined) - // Aggregation hook (outside transaction — derived data) + // Aggregation hook (outside transaction — derived data). `existing` is + // the full get() view — every reserved field top-level — and must be + // passed whole: a subset view makes the old-side decrement miss any + // reserved-field group (update would then double-count it). if (this._aggregationIndex) { - const oldEntityForAgg = { - type: existing.type, - service: existing.service, - data: existing.data, - metadata: existing.metadata - } - this._aggregationIndex.onEntityUpdated(params.id, entityForIndexing, oldEntityForAgg) + this._aggregationIndex.onEntityUpdated( + params.id, + entityForIndexing, + existing as unknown as Record + ) } } @@ -3245,19 +3269,15 @@ export class Brainy implements BrainyInterface { ] : undefined) - // Aggregation hook (outside transaction — derived data) + // Aggregation hook (outside transaction — derived data). The view must + // carry EVERY reserved field top-level (not a subset): a groupBy on + // subtype/visibility/etc. otherwise decrements a nonexistent group and + // the real count never comes down. if (this._aggregationIndex && metadata) { - // Reconstruct entity-like object from stored metadata via the - // canonical reserved/custom split (the hand-rolled destructure here - // missed subtype/_rev, leaking them into the aggregation view). - const { reserved, custom } = splitNounMetadataRecord(metadata) - const entityForAgg = { - type: reserved.noun, - service: reserved.service, - data: reserved.data, - metadata: custom - } - this._aggregationIndex.onEntityDeleted(id, entityForAgg) + this._aggregationIndex.onEntityDeleted( + id, + this.entityForAggFromRawRecord(metadata as Record) + ) } } @@ -6885,6 +6905,30 @@ export class Brainy implements BrainyInterface { this.assertWritable('removeMany') await this.ensureInitialized() + // Loud selector validation: a call with no usable selector used to + // resolve successfully having deleted NOTHING (total: 0) — the classic + // silent no-op being a bare array passed positionally + // (removeMany([id]) instead of removeMany({ ids: [id] })). The caller + // believes the delete happened; every count derived afterwards is "wrong" + // while the engine was never even asked. Refuse instead. + if (Array.isArray(params)) { + throw new Error( + `removeMany() takes a params object, not a bare array — use removeMany({ ids: [...] })` + ) + } + if (!params || (!params.ids && !params.type && !params.where)) { + throw new Error( + `removeMany() requires a selector: { ids } and/or { type, where }. ` + + `An empty selector would silently delete nothing — refusing.` + ) + } + if (params.ids && params.ids.length === 0) { + throw new Error( + `removeMany() received ids: [] — an empty id list deletes nothing. ` + + `Pass the ids to delete, or omit ids and select by { type, where }.` + ) + } + // Determine what to delete let idsToDelete: string[] = [] @@ -9388,12 +9432,9 @@ export class Brainy implements BrainyInterface { ) plan.touchedNouns.push(params.id) - const oldEntityForAgg = { - type: existing.type, - service: existing.service, - data: existing.data, - metadata: existing.metadata - } + // The full planGetEntity view, passed whole — a subset view makes the + // old-side decrement miss reserved-field groups (double-count on update). + const oldEntityForAgg = existing as unknown as Record plan.postCommit.push(() => { if (this._aggregationIndex) { this._aggregationIndex.onEntityUpdated(params.id, entityForIndexing, oldEntityForAgg) @@ -9514,14 +9555,9 @@ export class Brainy implements BrainyInterface { } if (metadata) { - // Canonical reserved/custom split — mirror of remove()'s aggregation hook. - const { reserved, custom } = splitNounMetadataRecord(metadata) - const entityForAgg = { - type: reserved.noun, - service: reserved.service, - data: reserved.data, - metadata: custom - } + // Mirror of remove()'s aggregation hook — the FULL reserved view, so + // reserved-field groupBy decrements find their group. + const entityForAgg = this.entityForAggFromRawRecord(metadata as Record) plan.postCommit.push(() => { if (this._aggregationIndex) { this._aggregationIndex.onEntityDeleted(id, entityForAgg) diff --git a/src/utils/metadataIndex.ts b/src/utils/metadataIndex.ts index fd325319..c772bce4 100644 --- a/src/utils/metadataIndex.ts +++ b/src/utils/metadataIndex.ts @@ -1867,9 +1867,26 @@ export class MetadataIndexManager implements MetadataIndexProvider { // not once per AND-clause inside it. const unindexedFields: string[] = [] - for (const [field, condition] of Object.entries(filter)) { + for (const [rawField, condition] of Object.entries(filter)) { // Skip logical operators - if (field === 'allOf' || field === 'anyOf' || field === 'not') continue + if (rawField === 'allOf' || rawField === 'anyOf' || rawField === 'not') continue + + // Metadata is FLATTENED at index time (metadata.entry.title indexes as + // entry.title), so a `metadata.`-prefixed where key is almost always + // the caller spelling the STORAGE shape rather than the index shape. + // Accept both spellings: when the key as spelled is unindexed but its + // stripped spelling is, query the stripped one. A literal nested + // custom key named `metadata` still wins when indexed as spelled + // (checked first), so that rare shape keeps working. + let field = rawField + if ( + rawField.startsWith('metadata.') && + this.columnStore && + !this.columnStore.hasField(rawField) && + this.columnStore.hasField(rawField.slice('metadata.'.length)) + ) { + field = rawField.slice('metadata.'.length) + } let fieldResults: string[] = [] diff --git a/tests/integration/aggregate-reserved-fields.test.ts b/tests/integration/aggregate-reserved-fields.test.ts new file mode 100644 index 00000000..e81692d6 --- /dev/null +++ b/tests/integration/aggregate-reserved-fields.test.ts @@ -0,0 +1,169 @@ +/** + * @module tests/integration/aggregate-reserved-fields + * @description One field-resolution law across the whole aggregation + query + * surface (SELF-AGGREGATE-DELETE-DRIFT). Laws: + * (1) aggregates grouped by a RESERVED field (subtype) decrement on delete — + * the delete-side entity view carries every reserved field, so the + * decrement finds its group (counts must never drift from ground truth); + * (2) same for update: moving an entity between reserved-field groups + * decrements the old group and increments the new one (no double-count); + * (3) aggregation source.where on a reserved field (subtype) FILTERS instead + * of silently matching nothing; + * (4) removeMany refuses empty/invalid selectors loudly (bare array, empty + * object, ids: []) instead of resolving as a silent no-op; + * (5) find() accepts both where spellings: flattened (entry.title) and + * storage-shaped (metadata.entry.title) resolve to the same rows. + */ +import { describe, it, expect, beforeEach, afterEach } from 'vitest' +import { Brainy } from '../../src/brainy.js' +import { NounType } from '../../src/types/graphTypes.js' + +const stubEmbedding = async (text: string): Promise => { + const hash = text.split('').reduce((acc, char) => acc + char.charCodeAt(0), 0) + return new Array(384).fill(0).map((_, i) => Math.sin(hash + i)) +} + +describe('aggregation + query field-resolution law', () => { + let brain: Brainy + + beforeEach(async () => { + brain = new Brainy({ + requireSubtype: false, + storage: { type: 'memory' as const }, + embeddingFunction: stubEmbedding + }) + await brain.init() + }) + + afterEach(async () => { + await brain.close() + }) + + it('reserved-field groupBy decrements on delete (the drift bug)', async () => { + brain.defineAggregate({ + name: 'by_subtype', + source: { type: NounType.Document }, + groupBy: ['subtype'], + metrics: { count: { op: 'count' } } + }) + + const ids: string[] = [] + for (let i = 0; i < 5; i++) { + ids.push( + await brain.add({ + data: `doc-${i}`, + type: NounType.Document, + subtype: 'note', + metadata: { team: 'alpha' } + }) + ) + } + let groups = await brain.queryAggregate('by_subtype') + expect(groups).toHaveLength(1) + expect(groups[0].groupKey).toEqual({ subtype: 'note' }) + expect(groups[0].metrics.count).toBe(5) + + await brain.remove(ids[0]) + await brain.flush() + + groups = await brain.queryAggregate('by_subtype') + expect(groups[0].metrics.count).toBe(4) + const live = await brain.find({ type: NounType.Document, limit: 100 }) + expect(groups[0].metrics.count).toBe(live.length) + }) + + it('reserved-field groupBy moves between groups on update (no double-count)', async () => { + brain.defineAggregate({ + name: 'by_subtype', + source: { type: NounType.Document }, + groupBy: ['subtype'], + metrics: { count: { op: 'count' } } + }) + const id = await brain.add({ + data: 'doc-move', + type: NounType.Document, + subtype: 'draft' + }) + await brain.update({ id, subtype: 'published' }) + + const groups = await brain.queryAggregate('by_subtype') + const byKey = Object.fromEntries( + groups.map((g) => [String(g.groupKey.subtype), g.metrics.count]) + ) + expect(byKey['published']).toBe(1) + // The old group must be gone or zero — never still counting the entity. + expect(byKey['draft'] ?? 0).toBe(0) + }) + + it('source.where on a reserved field filters instead of matching nothing', async () => { + brain.defineAggregate({ + name: 'notes_only', + source: { type: NounType.Document, where: { subtype: 'note' } }, + groupBy: ['team'], + metrics: { count: { op: 'count' } } + }) + await brain.add({ + data: 'n1', + type: NounType.Document, + subtype: 'note', + metadata: { team: 'alpha' } + }) + await brain.add({ + data: 'd1', + type: NounType.Document, + subtype: 'draft', + metadata: { team: 'alpha' } + }) + + const groups = await brain.queryAggregate('notes_only') + expect(groups).toHaveLength(1) + expect(groups[0].metrics.count).toBe(1) // the note, never the draft + }) + + it('removeMany refuses empty/invalid selectors loudly', async () => { + const id = await brain.add({ data: 'keep-me', type: NounType.Document }) + + // Bare array passed positionally — the classic silent no-op. + await expect( + brain.removeMany([id] as unknown as Parameters[0]) + ).rejects.toThrow(/bare array/) + // Empty selector object. + await expect( + brain.removeMany({} as Parameters[0]) + ).rejects.toThrow(/requires a selector/) + // Explicit empty id list. + await expect(brain.removeMany({ ids: [] })).rejects.toThrow(/ids: \[\]/) + + // Nothing was deleted by any of the refused calls. + expect(await brain.get(id)).toBeTruthy() + }) + + it('find() accepts both flattened and metadata.-prefixed where spellings', async () => { + await brain.add({ + data: 'nested-doc', + type: NounType.Document, + metadata: { entry: { title: 'T1' }, classifier: { contextHints: { vfsPath: '/n/a.md' } } } + }) + await brain.flush() + + const flat = await brain.find({ + type: NounType.Document, + where: { 'entry.title': 'T1' }, + limit: 10 + }) + const prefixed = await brain.find({ + type: NounType.Document, + where: { 'metadata.entry.title': 'T1' }, + limit: 10 + }) + const deepPrefixed = await brain.find({ + type: NounType.Document, + where: { 'metadata.classifier.contextHints.vfsPath': '/n/a.md' }, + limit: 10 + }) + expect(flat).toHaveLength(1) + expect(prefixed).toHaveLength(1) + expect(prefixed[0].id).toBe(flat[0].id) + expect(deepPrefixed).toHaveLength(1) + }) +}) diff --git a/tests/regression/metadata-index-cleanup.unit.test.ts b/tests/regression/metadata-index-cleanup.unit.test.ts index 266b9a4d..0984d727 100644 --- a/tests/regression/metadata-index-cleanup.unit.test.ts +++ b/tests/regression/metadata-index-cleanup.unit.test.ts @@ -205,10 +205,10 @@ describe('Metadata index cleanup after remove / removeMany', () => { } }) - it('handles empty ids array gracefully', async () => { - const result = await brain.removeMany({ ids: [] }) - expect(result.successful).toHaveLength(0) - expect(result.failed).toHaveLength(0) + it('refuses an empty ids array loudly (a silent no-op is not "graceful")', async () => { + // 8.8.2: an empty selector used to resolve successfully having deleted + // NOTHING — the caller believed the delete happened. Now it throws. + await expect(brain.removeMany({ ids: [] })).rejects.toThrow(/ids: \[\]/) }) it('handles large batch (> 1 chunk) without leaving stale index entries', async () => { diff --git a/tests/unit/brainy/batch-operations.test.ts b/tests/unit/brainy/batch-operations.test.ts index b2cc7f09..58b25744 100644 --- a/tests/unit/brainy/batch-operations.test.ts +++ b/tests/unit/brainy/batch-operations.test.ts @@ -533,8 +533,10 @@ describe('Brainy Batch Operations', () => { expect(result.successful).toHaveLength(0) await brain.updateMany({ items: [] }) - await brain.removeMany({ ids: [] }) - // Should not throw + // removeMany is the exception (8.8.2): an empty id list is a refused + // selector, not an empty batch — deleting "nothing" silently was the + // bug class (a positional/bare-array call looked identical). + await expect(brain.removeMany({ ids: [] })).rejects.toThrow(/ids: \[\]/) }) it('should validate batch size limits', async () => {