fix: validate where-operators and align the in-memory matcher to the documented set

`find({ where })` had two gaps that could silently return nothing. The in-memory
matcher (`matchesQuery`, used for egress re-validation and historical reads) was
missing several documented operators — `in`, `greaterThanOrEqual`,
`lessThanOrEqual` — so it disagreed with the index path, which does handle them.
And an unknown operator key (a typo, or `notIn` written where `not: { in }` was
meant) fell through to a nested-object-field interpretation and matched nothing.

- Align `matchesQuery` to the full documented operator set (add the missing
  aliases; the default branch now throws instead of treating an unknown key as a
  nested field — dotted paths remain the supported nested form).
- Validate the `where` filter up front (`validateWhereFilter`), throwing a typed
  `BrainyError('INVALID_QUERY')` that names the unknown operator, recursing into
  `allOf` / `anyOf` / `not`.
- Stop excluding a user field literally named `level` from the metadata index
  (it collided with an internal never-index key), so numeric/exact filters on it
  resolve instead of returning 0.

Adds unit coverage for the operator set and the throw-on-unknown behavior.
This commit is contained in:
David Snelling 2026-07-07 12:23:36 -07:00
parent 68da66024c
commit 6821e1980b
4 changed files with 176 additions and 13 deletions

View file

@ -10,6 +10,7 @@ export type BrainyErrorType =
| 'NOT_FOUND'
| 'RETRY_EXHAUSTED'
| 'VALIDATION'
| 'INVALID_QUERY'
| 'FIELD_NOT_INDEXED'
| 'GRAPH_INDEX_NOT_READY'
| 'METADATA_INDEX_NOT_READY'

View file

@ -5,6 +5,7 @@
*/
import { SearchResult, HNSWNoun, HNSWNounWithMetadata } from '../coreTypes.js'
import { BrainyError } from '../errors/brainyError.js'
/**
* Brainy Field Operators (BFO) - Our own field query system
@ -30,6 +31,7 @@ export interface BrainyFieldOperators {
// Array/Set operators
oneOf?: any[]
in?: any[] // documented alias for oneOf
noneOf?: any[]
contains?: any
excludes?: any
@ -70,6 +72,65 @@ export interface MetadataFilterOptions {
}
}
/**
* Value-level operators (the keys inside `{ field: { <op>: operand } }`) the
* complete documented set, kept in lockstep with the `matchesQuery` switch below
* and the metadata-index path. `in` is the documented alias for `oneOf`.
*/
const VALUE_OPERATORS = new Set<string>([
'equals', 'eq', 'notEquals', 'ne',
'greaterThan', 'gt', 'greaterThanOrEqual', 'gte',
'lessThan', 'lt', 'lessThanOrEqual', 'lte',
'between', 'oneOf', 'in', 'noneOf',
'contains', 'excludes', 'hasAll', 'length',
'exists', 'missing', 'matches', 'startsWith', 'endsWith'
])
/** Filter-level logical operators (siblings of field names). */
const LOGICAL_OPERATORS = new Set<string>(['allOf', 'anyOf', 'not'])
/**
* Validate a `where` filter's operators up front, throwing a typed
* `BrainyError('INVALID_QUERY')` on the first unrecognized operator so a typo
* like `{ subtype: { notIn: [...] } }` fails LOUD instead of silently matching
* nothing (the invisible-degrade class). Called by `find()` before either the
* index path or the in-memory matcher runs, so the throw fires even when the
* result set is empty (the index path would otherwise return `[]` without ever
* invoking the matcher). Nested fields use dot notation (`{ 'a.b': v }`), so a
* field's object value carries operators, never sub-field names.
*
* @param filter - the user-supplied `where` clause (validated raw, before any
* internal field injection like visibility or `type``noun`).
* @throws BrainyError('INVALID_QUERY') naming the bad operator + the valid set.
*/
export function validateWhereFilter(filter: unknown): void {
if (!filter || typeof filter !== 'object' || Array.isArray(filter)) return
for (const [key, value] of Object.entries(filter as Record<string, unknown>)) {
if (LOGICAL_OPERATORS.has(key)) {
if (key === 'not') {
validateWhereFilter(value)
} else if (Array.isArray(value)) {
for (const sub of value) validateWhereFilter(sub)
}
continue
}
// A field key. Its value is a scalar/array (equality) — nothing to validate —
// or an object of value-operators, every key of which must be recognized.
if (value && typeof value === 'object' && !Array.isArray(value)) {
for (const op of Object.keys(value as Record<string, unknown>)) {
if (!VALUE_OPERATORS.has(op)) {
throw new BrainyError(
`Unknown filter operator "${op}" on field "${key}". Valid operators: ` +
`${[...VALUE_OPERATORS].sort().join(', ')}. For nested fields use dot ` +
`notation, e.g. { '${key}.subfield': value }.`,
'INVALID_QUERY'
)
}
}
}
}
}
/**
* Check if a value matches a query with operators
*/
@ -103,6 +164,7 @@ function matchesQuery(value: any, query: any): boolean {
if (typeof value !== 'number' || typeof operand !== 'number' || !(value > operand)) return false
break
case 'gte':
case 'greaterThanOrEqual':
if (typeof value !== 'number' || typeof operand !== 'number' || !(value >= operand)) return false
break
case 'lessThan':
@ -110,6 +172,7 @@ function matchesQuery(value: any, query: any): boolean {
if (typeof value !== 'number' || typeof operand !== 'number' || !(value < operand)) return false
break
case 'lte':
case 'lessThanOrEqual':
if (typeof value !== 'number' || typeof operand !== 'number' || !(value <= operand)) return false
break
case 'between':
@ -119,6 +182,7 @@ function matchesQuery(value: any, query: any): boolean {
// Array/Set operators
case 'oneOf':
case 'in': // documented alias for oneOf
if (!Array.isArray(operand) || !operand.includes(value)) return false
break
case 'noneOf':
@ -161,22 +225,26 @@ function matchesQuery(value: any, query: any): boolean {
break
default:
// Unknown operator, treat as field name
if (!matchesFieldQuery(value, op, operand)) return false
// Unknown operator. The old behavior treated any unknown key as a
// nested-object field name (an UNDOCUMENTED fallback — dot notation
// `{ 'a.b': v }` is the supported nested form), which silently swallowed
// operator typos: `{ x: { notIn: [...] } }` matched nothing instead of
// erroring. Fail loud. find() validates the whole where clause up front
// (validateWhereFilter), so a typo throws even on an empty result set;
// this is the belt-and-suspenders for any matcher caller that bypasses
// that path.
throw new BrainyError(
`Unknown filter operator "${op}". Valid operators: ` +
`${[...VALUE_OPERATORS].sort().join(', ')}. For nested fields use dot ` +
`notation, e.g. { 'address.city': 'NYC' }.`,
'INVALID_QUERY'
)
}
}
return true
}
/**
* Check if a field matches a query
*/
function matchesFieldQuery(obj: any, field: string, query: any): boolean {
const value = getNestedValue(obj, field)
return matchesQuery(value, query)
}
/**
* Get nested value from object using dot notation
*/

View file

@ -1166,8 +1166,17 @@ export class MetadataIndexManager implements MetadataIndexProvider {
private extractIndexableFields(data: any): Array<{ field: string, value: any }> {
const fields: Array<{ field: string, value: any }> = []
// Fields that should NEVER be indexed (vectors, embeddings, large arrays, HNSW internals)
const NEVER_INDEX = new Set(['vector', 'embedding', 'embeddings', 'connections', 'level', 'id'])
// Fields that should NEVER be indexed: bulk structural payloads that would
// blow up the index (the 384-dim vector, embeddings, the adjacency list).
// These are also caught by the array-size guard below, but naming them is
// belt-and-suspenders. NOTE: `level` was previously here (an HNSW node's
// layer) but it never actually reaches this path — every caller passes a
// metadata bag or Entity record, neither of which carries the node's
// `level` — so its only effect was to silently drop a legitimate USER
// metadata field named `level` (log level, skill level, access level…),
// making `where: { level: … }` return nothing. Removed. (`id` stays: it is
// the reserved entity-identity field, resolved specially by find().)
const NEVER_INDEX = new Set(['vector', 'embedding', 'embeddings', 'connections', 'id'])
const extract = (obj: any, prefix = ''): void => {
for (const [key, value] of Object.entries(obj)) {
@ -1278,7 +1287,10 @@ export class MetadataIndexManager implements MetadataIndexProvider {
return data.map(d => this.extractTextContent(d)).filter(Boolean).join(' ')
}
if (typeof data === 'object') {
const skipKeys = new Set(['vector', 'embedding', 'embeddings', 'connections', 'level', 'id'])
// Mirror of NEVER_INDEX for the text-extraction path: bulk structural
// payloads only. `level` removed for the same reason (it silently dropped
// a real user field from hybrid text search too).
const skipKeys = new Set(['vector', 'embedding', 'embeddings', 'connections', 'id'])
const texts: string[] = []
for (const [key, value] of Object.entries(data)) {
// Skip internal fields and numeric keys (array indices)