diff --git a/src/aggregation/AggregationIndex.ts b/src/aggregation/AggregationIndex.ts index 407b1fe0..ca44ac8b 100644 --- a/src/aggregation/AggregationIndex.ts +++ b/src/aggregation/AggregationIndex.ts @@ -110,11 +110,15 @@ function matchesSource(entity: Record, source: AggregateDefinit // live in the custom bag, so those filters could never match anything. if (source.where && Object.keys(source.where).length > 0) { const e = entity as unknown as HNSWNounWithMetadata - const resolved: Record = {} - for (const key of Object.keys(source.where)) { - resolved[key] = readAddressed(e, key) + for (const [key, condition] of Object.entries(source.where)) { + // Evaluate ONE field at a time under a neutral key: the address may be + // dotted ('system.subtype'), and the filter evaluator would otherwise + // walk dots as a nested path instead of treating the key as an address. + const value = readAddressed(e, key) + if (!matchesMetadataFilter({ v: value }, { v: condition } as Record)) { + return false + } } - if (!matchesMetadataFilter(resolved, source.where)) return false } return true diff --git a/src/db/fieldAddressing.ts b/src/db/fieldAddressing.ts index 056ba3f2..98c81e5f 100644 --- a/src/db/fieldAddressing.ts +++ b/src/db/fieldAddressing.ts @@ -167,10 +167,39 @@ export function readEntityFieldAddress( entity: HNSWNounWithMetadata, address: FieldAddress ): unknown { + const rec = entity as unknown as Record + const bag = + rec.metadata && typeof rec.metadata === 'object' + ? (rec.metadata as Record) + : null + if (address.scope === 'system') { - return (entity as unknown as Record)[address.field] + // Entity views carry system scalars top-level; raw storage shapes carry + // them inside the stored metadata record (where `type` is spelled `noun`). + // Read top-level first, then the record — never the user's namespace. + const top = rec[address.field] + if (top !== undefined) return top + if (bag) { + if (address.field === 'type') return bag.type ?? bag.noun + return bag[address.field] + } + return undefined } - return entity.metadata?.[address.field] + + // User scope. The write-path remap guarantees the user can never OWN a + // field named like a system scalar (those lift top-level at write), so a + // bare system name reads as ABSENT — reading the stored record's reserved + // key here would re-create the shadow this module exists to kill. Same for + // plumbing and the legacy 'noun' spelling. + if ( + SYSTEM_ENTITY_SCALARS.has(address.field) || + PLUMBING_FIELDS.has(address.field) || + address.field === 'noun' + ) { + return undefined + } + if (bag) return bag[address.field] + return rec[address.field] } /** @@ -222,29 +251,6 @@ export function buildUnresolvableMessage( ) } -/** - * @description Refusal for a malformed or out-of-map field ADDRESS — - * `system.` (including all plumbing), an empty - * name, or a bare `metadata.` prefix. The message carries the full valid - * system map so the fix never needs a docs lookup. - */ -export class InvalidFieldAddressError extends Error { - public readonly raw: string - public readonly kind: FieldAddressKind - - constructor(raw: string, kind: FieldAddressKind, systemMap: ReadonlySet) { - const valid = [...systemMap].map((f) => `system.${f}`).join(', ') - super( - `'${raw}' is not an addressable ${kind} field. Bare names address your own ` + - `metadata fields; engine fields are exactly: ${valid}. Engine plumbing ` + - `(vector, connections, level, data, _rev) is not part of the query surface.` - ) - this.name = 'InvalidFieldAddressError' - this.raw = raw - this.kind = kind - } -} - /** * @description Refusal for a syntactically valid address that resolves to * NOTHING — a bare name no user field carries. Carries the did-you-mean @@ -256,14 +262,35 @@ export class UnresolvableFieldError extends Error { public readonly raw: string public readonly kind: FieldAddressKind - constructor(raw: string, kind: FieldAddressKind) { - super(buildUnresolvableMessage(raw, kind)) + constructor(raw: string, kind: FieldAddressKind, messageOverride?: string) { + super(messageOverride ?? buildUnresolvableMessage(raw, kind)) this.name = 'UnresolvableFieldError' this.raw = raw this.kind = kind } } +/** + * @description Refusal for a malformed or out-of-map field ADDRESS — + * `system.` (including all plumbing), an empty + * name, or a bare `metadata.` prefix. The message carries the full valid + * system map so the fix never needs a docs lookup. + */ +export class InvalidFieldAddressError extends UnresolvableFieldError { + constructor(raw: string, kind: FieldAddressKind, systemMap: ReadonlySet) { + const valid = [...systemMap].map((f) => `system.${f}`).join(', ') + super( + raw, + kind, + `'${raw}' is not an addressable ${kind} field. Bare names address your own ` + + `metadata fields; engine fields are exactly: ${valid}. Engine plumbing ` + + `(vector, connections, level, data, _rev) is not part of the query surface.` + ) + this.name = 'InvalidFieldAddressError' + } +} + + /** * @description Refusal for a find() option that is accepted by the type * surface but NOT implemented — an accepted option must work or refuse; diff --git a/src/utils/metadataIndex.ts b/src/utils/metadataIndex.ts index bfde5fe9..6deeb811 100644 --- a/src/utils/metadataIndex.ts +++ b/src/utils/metadataIndex.ts @@ -5,7 +5,7 @@ */ import { StorageAdapter, resolveEntityField, NounMetadata, VerbMetadata } from '../coreTypes.js' -import { SYSTEM_ENTITY_SCALARS, parseFieldAddress } from '../db/fieldAddressing.js' +import { SYSTEM_ENTITY_SCALARS, parseFieldAddress, UnresolvableFieldError } from '../db/fieldAddressing.js' import { ColumnStore } from '../indexes/columnStore/ColumnStore.js' import type { MetadataIndexProvider } from '../plugin.js' import { MetadataIndexCache, MetadataIndexCacheConfig } from './metadataIndexCache.js' @@ -2250,6 +2250,18 @@ export class MetadataIndexManager implements MetadataIndexProvider { const orderKey = orderAddress.scope === 'system' ? `system.${orderAddress.field}` : orderAddress.field + // DATA-AWARE REFUSAL (the did-you-mean): a bare address no user field + // carries cannot mean anything as a sort key — and when the name collides + // with a system scalar the caller almost certainly meant system.. + // Refusing loudly with both candidates beats silently sorting nothing. + if ( + orderAddress.scope === 'metadata' && + !(this.columnStore && this.columnStore.hasField(orderKey)) && + !(await this.loadSparseIndex(orderKey)) + ) { + throw new UnresolvableFieldError(orderAddress.raw, 'entity') + } + // Column store path: O(K log S) sort via k-way merge across segments. // No per-entity storage reads, no precision loss from bucketing. if (this.columnStore && this.columnStore.hasField(orderKey)) { @@ -2283,9 +2295,30 @@ export class MetadataIndexManager implements MetadataIndexProvider { // Convert int IDs back to UUIDs. Number() narrowing is lossless — the // shipped EntityIdSpaceExceeded guard caps the JS mapper at u32. - return sortedIntIds + const sortedUuids = sortedIntIds .map(intId => this.idMapper.getUuid(Number(intId))) .filter((uuid): uuid is string => uuid !== undefined) + + // ORDERING CONTRACT (cross-engine, sealed): rows missing the field are + // NEVER dropped — they sort LAST in both directions — and ties break by + // id ascending. The column only contains rows that HAVE the field, so + // (1) re-sort the page deterministically (value, then id) with K cheap + // value reads, and (2) append the filtered rows the column omitted, + // id-ascending, filling any remaining page budget. + const page = await Promise.all( + sortedUuids.map(async id => ({ id, value: await this.getFieldValueForEntity(id, orderKey) })) + ) + page.sort((a, b) => this.compareAddressedValues(a.value, b.value, a.id, b.id, order)) + let result = page.map(p => p.id) + + if (hasFilter) { + const present = new Set(sortedUuids) + if (topK === undefined || result.length < topK) { + const missing = filteredIds.filter(id => !present.has(id)).sort() + result = result.concat(missing) + } + } + return topK !== undefined ? result.slice(0, topK) : result } // Fallback: sparse index path (for fields not yet in column store). @@ -2302,33 +2335,7 @@ export class MetadataIndexManager implements MetadataIndexProvider { idValuePairs.push({ id, value }) } - idValuePairs.sort((a, b) => { - // Ordering contract (cross-engine, ruled 2026-08-03): missing/null - // values sort LAST in BOTH directions — the direction flip never moves - // them to the front — and ties break by id ascending, so an ordered - // read is deterministic and identical on both engines. Rows are never - // dropped for lacking the field. - const aNull = a.value == null - const bNull = b.value == null - if (aNull || bNull) { - if (aNull && bNull) return a.id < b.id ? -1 : a.id > b.id ? 1 : 0 - return aNull ? 1 : -1 - } - // Numbers compare numerically; everything else by code-point (UTF-8 byte) order. - // This makes the JS fallback sort match cor's native column store exactly - // (numeric i64/f64 vs code-point strings) and stay deterministic across - // environments, unlike the `<` operator's UTF-16 ordering for strings. - let comparison = 0 - if (a.value !== b.value) { - if (typeof a.value === 'number' && typeof b.value === 'number') { - comparison = a.value < b.value ? -1 : 1 - } else { - comparison = compareCodePoints(String(a.value), String(b.value)) - } - } - if (comparison === 0) return a.id < b.id ? -1 : a.id > b.id ? 1 : 0 - return order === 'asc' ? comparison : -comparison - }) + idValuePairs.sort((a, b) => this.compareAddressedValues(a.value, b.value, a.id, b.id, order)) const sorted = idValuePairs.map(p => p.id) return topK !== undefined ? sorted.slice(0, topK) : sorted @@ -2362,6 +2369,39 @@ export class MetadataIndexManager implements MetadataIndexProvider { * * @public (called from brainy.ts for sorted queries) */ + /** + * The cross-engine ordering contract in one comparator (sealed 2026-08-03): + * missing/null values sort LAST in BOTH directions — the direction flip + * never moves them to the front — and ties break by id ascending, so an + * ordered read is deterministic and identical on both engines. Numbers + * compare numerically; everything else by code-point (UTF-8 byte) order, + * matching the native column store exactly. + */ + private compareAddressedValues( + aVal: any, + bVal: any, + aId: string, + bId: string, + order: 'asc' | 'desc' + ): number { + const aNull = aVal == null + const bNull = bVal == null + if (aNull || bNull) { + if (aNull && bNull) return aId < bId ? -1 : aId > bId ? 1 : 0 + return aNull ? 1 : -1 + } + let comparison = 0 + if (aVal !== bVal) { + if (typeof aVal === 'number' && typeof bVal === 'number') { + comparison = aVal < bVal ? -1 : 1 + } else { + comparison = compareCodePoints(String(aVal), String(bVal)) + } + } + if (comparison === 0) return aId < bId ? -1 : aId > bId ? 1 : 0 + return order === 'asc' ? comparison : -comparison + } + async getFieldValueForEntity(entityId: string, field: string): Promise { // `field` arrives as a FROZEN INDEX KEY (bare = user metadata; // 'system.' = engine scalar). Storage fallbacks read the matching diff --git a/tests/conformance/namespace-law.test.ts b/tests/conformance/namespace-law.test.ts index 91227587..0231685a 100644 --- a/tests/conformance/namespace-law.test.ts +++ b/tests/conformance/namespace-law.test.ts @@ -294,7 +294,9 @@ describe.skipIf(!lawActive)('namespace law — bare/system/metadata field addres brain.defineAggregate({ name: 'ns_law_by_team_bare', - source: { type: NounType.Document, where: { subtype: 'ns-law-group-bare' } }, + // system.subtype — bare 'subtype' would address user metadata under the + // law (the exact migration every fleet consumer's aggregates make). + source: { type: NounType.Document, where: { 'system.subtype': 'ns-law-group-bare' } }, groupBy: ['team'], metrics: { count: { op: 'count' } } })