feat(namespace): conformance green 19/19 — data-aware did-you-mean on unindexed bare addresses, ordering contract on the column top-K path (never drop, nulls last, ties by id), shape-complete addressed reads (entity views AND raw storage shapes, shadow-proof both scopes), per-key source matching for dotted addresses; refusal classes unified under UnresolvableFieldError
This commit is contained in:
parent
7492b6cb59
commit
8e962dabda
4 changed files with 134 additions and 61 deletions
|
|
@ -110,11 +110,15 @@ function matchesSource(entity: Record<string, unknown>, source: AggregateDefinit
|
||||||
// live in the custom bag, so those filters could never match anything.
|
// live in the custom bag, so those filters could never match anything.
|
||||||
if (source.where && Object.keys(source.where).length > 0) {
|
if (source.where && Object.keys(source.where).length > 0) {
|
||||||
const e = entity as unknown as HNSWNounWithMetadata
|
const e = entity as unknown as HNSWNounWithMetadata
|
||||||
const resolved: Record<string, unknown> = {}
|
for (const [key, condition] of Object.entries(source.where)) {
|
||||||
for (const key of Object.keys(source.where)) {
|
// Evaluate ONE field at a time under a neutral key: the address may be
|
||||||
resolved[key] = readAddressed(e, key)
|
// dotted ('system.subtype'), and the filter evaluator would otherwise
|
||||||
|
// walk dots as a nested path instead of treating the key as an address.
|
||||||
|
const value = readAddressed(e, key)
|
||||||
|
if (!matchesMetadataFilter({ v: value }, { v: condition } as Record<string, unknown>)) {
|
||||||
|
return false
|
||||||
|
}
|
||||||
}
|
}
|
||||||
if (!matchesMetadataFilter(resolved, source.where)) return false
|
|
||||||
}
|
}
|
||||||
|
|
||||||
return true
|
return true
|
||||||
|
|
|
||||||
|
|
@ -167,10 +167,39 @@ export function readEntityFieldAddress(
|
||||||
entity: HNSWNounWithMetadata,
|
entity: HNSWNounWithMetadata,
|
||||||
address: FieldAddress
|
address: FieldAddress
|
||||||
): unknown {
|
): unknown {
|
||||||
|
const rec = entity as unknown as Record<string, unknown>
|
||||||
|
const bag =
|
||||||
|
rec.metadata && typeof rec.metadata === 'object'
|
||||||
|
? (rec.metadata as Record<string, unknown>)
|
||||||
|
: null
|
||||||
|
|
||||||
if (address.scope === 'system') {
|
if (address.scope === 'system') {
|
||||||
return (entity as unknown as Record<string, unknown>)[address.field]
|
// Entity views carry system scalars top-level; raw storage shapes carry
|
||||||
|
// them inside the stored metadata record (where `type` is spelled `noun`).
|
||||||
|
// Read top-level first, then the record — never the user's namespace.
|
||||||
|
const top = rec[address.field]
|
||||||
|
if (top !== undefined) return top
|
||||||
|
if (bag) {
|
||||||
|
if (address.field === 'type') return bag.type ?? bag.noun
|
||||||
|
return bag[address.field]
|
||||||
|
}
|
||||||
|
return undefined
|
||||||
}
|
}
|
||||||
return entity.metadata?.[address.field]
|
|
||||||
|
// User scope. The write-path remap guarantees the user can never OWN a
|
||||||
|
// field named like a system scalar (those lift top-level at write), so a
|
||||||
|
// bare system name reads as ABSENT — reading the stored record's reserved
|
||||||
|
// key here would re-create the shadow this module exists to kill. Same for
|
||||||
|
// plumbing and the legacy 'noun' spelling.
|
||||||
|
if (
|
||||||
|
SYSTEM_ENTITY_SCALARS.has(address.field) ||
|
||||||
|
PLUMBING_FIELDS.has(address.field) ||
|
||||||
|
address.field === 'noun'
|
||||||
|
) {
|
||||||
|
return undefined
|
||||||
|
}
|
||||||
|
if (bag) return bag[address.field]
|
||||||
|
return rec[address.field]
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
|
@ -222,29 +251,6 @@ export function buildUnresolvableMessage(
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
|
||||||
* @description Refusal for a malformed or out-of-map field ADDRESS —
|
|
||||||
* `system.<anything-not-in-the-map>` (including all plumbing), an empty
|
|
||||||
* name, or a bare `metadata.` prefix. The message carries the full valid
|
|
||||||
* system map so the fix never needs a docs lookup.
|
|
||||||
*/
|
|
||||||
export class InvalidFieldAddressError extends Error {
|
|
||||||
public readonly raw: string
|
|
||||||
public readonly kind: FieldAddressKind
|
|
||||||
|
|
||||||
constructor(raw: string, kind: FieldAddressKind, systemMap: ReadonlySet<string>) {
|
|
||||||
const valid = [...systemMap].map((f) => `system.${f}`).join(', ')
|
|
||||||
super(
|
|
||||||
`'${raw}' is not an addressable ${kind} field. Bare names address your own ` +
|
|
||||||
`metadata fields; engine fields are exactly: ${valid}. Engine plumbing ` +
|
|
||||||
`(vector, connections, level, data, _rev) is not part of the query surface.`
|
|
||||||
)
|
|
||||||
this.name = 'InvalidFieldAddressError'
|
|
||||||
this.raw = raw
|
|
||||||
this.kind = kind
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* @description Refusal for a syntactically valid address that resolves to
|
* @description Refusal for a syntactically valid address that resolves to
|
||||||
* NOTHING — a bare name no user field carries. Carries the did-you-mean
|
* NOTHING — a bare name no user field carries. Carries the did-you-mean
|
||||||
|
|
@ -256,14 +262,35 @@ export class UnresolvableFieldError extends Error {
|
||||||
public readonly raw: string
|
public readonly raw: string
|
||||||
public readonly kind: FieldAddressKind
|
public readonly kind: FieldAddressKind
|
||||||
|
|
||||||
constructor(raw: string, kind: FieldAddressKind) {
|
constructor(raw: string, kind: FieldAddressKind, messageOverride?: string) {
|
||||||
super(buildUnresolvableMessage(raw, kind))
|
super(messageOverride ?? buildUnresolvableMessage(raw, kind))
|
||||||
this.name = 'UnresolvableFieldError'
|
this.name = 'UnresolvableFieldError'
|
||||||
this.raw = raw
|
this.raw = raw
|
||||||
this.kind = kind
|
this.kind = kind
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @description Refusal for a malformed or out-of-map field ADDRESS —
|
||||||
|
* `system.<anything-not-in-the-map>` (including all plumbing), an empty
|
||||||
|
* name, or a bare `metadata.` prefix. The message carries the full valid
|
||||||
|
* system map so the fix never needs a docs lookup.
|
||||||
|
*/
|
||||||
|
export class InvalidFieldAddressError extends UnresolvableFieldError {
|
||||||
|
constructor(raw: string, kind: FieldAddressKind, systemMap: ReadonlySet<string>) {
|
||||||
|
const valid = [...systemMap].map((f) => `system.${f}`).join(', ')
|
||||||
|
super(
|
||||||
|
raw,
|
||||||
|
kind,
|
||||||
|
`'${raw}' is not an addressable ${kind} field. Bare names address your own ` +
|
||||||
|
`metadata fields; engine fields are exactly: ${valid}. Engine plumbing ` +
|
||||||
|
`(vector, connections, level, data, _rev) is not part of the query surface.`
|
||||||
|
)
|
||||||
|
this.name = 'InvalidFieldAddressError'
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* @description Refusal for a find() option that is accepted by the type
|
* @description Refusal for a find() option that is accepted by the type
|
||||||
* surface but NOT implemented — an accepted option must work or refuse;
|
* surface but NOT implemented — an accepted option must work or refuse;
|
||||||
|
|
|
||||||
|
|
@ -5,7 +5,7 @@
|
||||||
*/
|
*/
|
||||||
|
|
||||||
import { StorageAdapter, resolveEntityField, NounMetadata, VerbMetadata } from '../coreTypes.js'
|
import { StorageAdapter, resolveEntityField, NounMetadata, VerbMetadata } from '../coreTypes.js'
|
||||||
import { SYSTEM_ENTITY_SCALARS, parseFieldAddress } from '../db/fieldAddressing.js'
|
import { SYSTEM_ENTITY_SCALARS, parseFieldAddress, UnresolvableFieldError } from '../db/fieldAddressing.js'
|
||||||
import { ColumnStore } from '../indexes/columnStore/ColumnStore.js'
|
import { ColumnStore } from '../indexes/columnStore/ColumnStore.js'
|
||||||
import type { MetadataIndexProvider } from '../plugin.js'
|
import type { MetadataIndexProvider } from '../plugin.js'
|
||||||
import { MetadataIndexCache, MetadataIndexCacheConfig } from './metadataIndexCache.js'
|
import { MetadataIndexCache, MetadataIndexCacheConfig } from './metadataIndexCache.js'
|
||||||
|
|
@ -2250,6 +2250,18 @@ export class MetadataIndexManager implements MetadataIndexProvider {
|
||||||
const orderKey =
|
const orderKey =
|
||||||
orderAddress.scope === 'system' ? `system.${orderAddress.field}` : orderAddress.field
|
orderAddress.scope === 'system' ? `system.${orderAddress.field}` : orderAddress.field
|
||||||
|
|
||||||
|
// DATA-AWARE REFUSAL (the did-you-mean): a bare address no user field
|
||||||
|
// carries cannot mean anything as a sort key — and when the name collides
|
||||||
|
// with a system scalar the caller almost certainly meant system.<field>.
|
||||||
|
// Refusing loudly with both candidates beats silently sorting nothing.
|
||||||
|
if (
|
||||||
|
orderAddress.scope === 'metadata' &&
|
||||||
|
!(this.columnStore && this.columnStore.hasField(orderKey)) &&
|
||||||
|
!(await this.loadSparseIndex(orderKey))
|
||||||
|
) {
|
||||||
|
throw new UnresolvableFieldError(orderAddress.raw, 'entity')
|
||||||
|
}
|
||||||
|
|
||||||
// Column store path: O(K log S) sort via k-way merge across segments.
|
// Column store path: O(K log S) sort via k-way merge across segments.
|
||||||
// No per-entity storage reads, no precision loss from bucketing.
|
// No per-entity storage reads, no precision loss from bucketing.
|
||||||
if (this.columnStore && this.columnStore.hasField(orderKey)) {
|
if (this.columnStore && this.columnStore.hasField(orderKey)) {
|
||||||
|
|
@ -2283,9 +2295,30 @@ export class MetadataIndexManager implements MetadataIndexProvider {
|
||||||
|
|
||||||
// Convert int IDs back to UUIDs. Number() narrowing is lossless — the
|
// Convert int IDs back to UUIDs. Number() narrowing is lossless — the
|
||||||
// shipped EntityIdSpaceExceeded guard caps the JS mapper at u32.
|
// shipped EntityIdSpaceExceeded guard caps the JS mapper at u32.
|
||||||
return sortedIntIds
|
const sortedUuids = sortedIntIds
|
||||||
.map(intId => this.idMapper.getUuid(Number(intId)))
|
.map(intId => this.idMapper.getUuid(Number(intId)))
|
||||||
.filter((uuid): uuid is string => uuid !== undefined)
|
.filter((uuid): uuid is string => uuid !== undefined)
|
||||||
|
|
||||||
|
// ORDERING CONTRACT (cross-engine, sealed): rows missing the field are
|
||||||
|
// NEVER dropped — they sort LAST in both directions — and ties break by
|
||||||
|
// id ascending. The column only contains rows that HAVE the field, so
|
||||||
|
// (1) re-sort the page deterministically (value, then id) with K cheap
|
||||||
|
// value reads, and (2) append the filtered rows the column omitted,
|
||||||
|
// id-ascending, filling any remaining page budget.
|
||||||
|
const page = await Promise.all(
|
||||||
|
sortedUuids.map(async id => ({ id, value: await this.getFieldValueForEntity(id, orderKey) }))
|
||||||
|
)
|
||||||
|
page.sort((a, b) => this.compareAddressedValues(a.value, b.value, a.id, b.id, order))
|
||||||
|
let result = page.map(p => p.id)
|
||||||
|
|
||||||
|
if (hasFilter) {
|
||||||
|
const present = new Set(sortedUuids)
|
||||||
|
if (topK === undefined || result.length < topK) {
|
||||||
|
const missing = filteredIds.filter(id => !present.has(id)).sort()
|
||||||
|
result = result.concat(missing)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return topK !== undefined ? result.slice(0, topK) : result
|
||||||
}
|
}
|
||||||
|
|
||||||
// Fallback: sparse index path (for fields not yet in column store).
|
// Fallback: sparse index path (for fields not yet in column store).
|
||||||
|
|
@ -2302,33 +2335,7 @@ export class MetadataIndexManager implements MetadataIndexProvider {
|
||||||
idValuePairs.push({ id, value })
|
idValuePairs.push({ id, value })
|
||||||
}
|
}
|
||||||
|
|
||||||
idValuePairs.sort((a, b) => {
|
idValuePairs.sort((a, b) => this.compareAddressedValues(a.value, b.value, a.id, b.id, order))
|
||||||
// Ordering contract (cross-engine, ruled 2026-08-03): missing/null
|
|
||||||
// values sort LAST in BOTH directions — the direction flip never moves
|
|
||||||
// them to the front — and ties break by id ascending, so an ordered
|
|
||||||
// read is deterministic and identical on both engines. Rows are never
|
|
||||||
// dropped for lacking the field.
|
|
||||||
const aNull = a.value == null
|
|
||||||
const bNull = b.value == null
|
|
||||||
if (aNull || bNull) {
|
|
||||||
if (aNull && bNull) return a.id < b.id ? -1 : a.id > b.id ? 1 : 0
|
|
||||||
return aNull ? 1 : -1
|
|
||||||
}
|
|
||||||
// Numbers compare numerically; everything else by code-point (UTF-8 byte) order.
|
|
||||||
// This makes the JS fallback sort match cor's native column store exactly
|
|
||||||
// (numeric i64/f64 vs code-point strings) and stay deterministic across
|
|
||||||
// environments, unlike the `<` operator's UTF-16 ordering for strings.
|
|
||||||
let comparison = 0
|
|
||||||
if (a.value !== b.value) {
|
|
||||||
if (typeof a.value === 'number' && typeof b.value === 'number') {
|
|
||||||
comparison = a.value < b.value ? -1 : 1
|
|
||||||
} else {
|
|
||||||
comparison = compareCodePoints(String(a.value), String(b.value))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if (comparison === 0) return a.id < b.id ? -1 : a.id > b.id ? 1 : 0
|
|
||||||
return order === 'asc' ? comparison : -comparison
|
|
||||||
})
|
|
||||||
|
|
||||||
const sorted = idValuePairs.map(p => p.id)
|
const sorted = idValuePairs.map(p => p.id)
|
||||||
return topK !== undefined ? sorted.slice(0, topK) : sorted
|
return topK !== undefined ? sorted.slice(0, topK) : sorted
|
||||||
|
|
@ -2362,6 +2369,39 @@ export class MetadataIndexManager implements MetadataIndexProvider {
|
||||||
*
|
*
|
||||||
* @public (called from brainy.ts for sorted queries)
|
* @public (called from brainy.ts for sorted queries)
|
||||||
*/
|
*/
|
||||||
|
/**
|
||||||
|
* The cross-engine ordering contract in one comparator (sealed 2026-08-03):
|
||||||
|
* missing/null values sort LAST in BOTH directions — the direction flip
|
||||||
|
* never moves them to the front — and ties break by id ascending, so an
|
||||||
|
* ordered read is deterministic and identical on both engines. Numbers
|
||||||
|
* compare numerically; everything else by code-point (UTF-8 byte) order,
|
||||||
|
* matching the native column store exactly.
|
||||||
|
*/
|
||||||
|
private compareAddressedValues(
|
||||||
|
aVal: any,
|
||||||
|
bVal: any,
|
||||||
|
aId: string,
|
||||||
|
bId: string,
|
||||||
|
order: 'asc' | 'desc'
|
||||||
|
): number {
|
||||||
|
const aNull = aVal == null
|
||||||
|
const bNull = bVal == null
|
||||||
|
if (aNull || bNull) {
|
||||||
|
if (aNull && bNull) return aId < bId ? -1 : aId > bId ? 1 : 0
|
||||||
|
return aNull ? 1 : -1
|
||||||
|
}
|
||||||
|
let comparison = 0
|
||||||
|
if (aVal !== bVal) {
|
||||||
|
if (typeof aVal === 'number' && typeof bVal === 'number') {
|
||||||
|
comparison = aVal < bVal ? -1 : 1
|
||||||
|
} else {
|
||||||
|
comparison = compareCodePoints(String(aVal), String(bVal))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (comparison === 0) return aId < bId ? -1 : aId > bId ? 1 : 0
|
||||||
|
return order === 'asc' ? comparison : -comparison
|
||||||
|
}
|
||||||
|
|
||||||
async getFieldValueForEntity(entityId: string, field: string): Promise<any> {
|
async getFieldValueForEntity(entityId: string, field: string): Promise<any> {
|
||||||
// `field` arrives as a FROZEN INDEX KEY (bare = user metadata;
|
// `field` arrives as a FROZEN INDEX KEY (bare = user metadata;
|
||||||
// 'system.<field>' = engine scalar). Storage fallbacks read the matching
|
// 'system.<field>' = engine scalar). Storage fallbacks read the matching
|
||||||
|
|
|
||||||
|
|
@ -294,7 +294,9 @@ describe.skipIf(!lawActive)('namespace law — bare/system/metadata field addres
|
||||||
|
|
||||||
brain.defineAggregate({
|
brain.defineAggregate({
|
||||||
name: 'ns_law_by_team_bare',
|
name: 'ns_law_by_team_bare',
|
||||||
source: { type: NounType.Document, where: { subtype: 'ns-law-group-bare' } },
|
// system.subtype — bare 'subtype' would address user metadata under the
|
||||||
|
// law (the exact migration every fleet consumer's aggregates make).
|
||||||
|
source: { type: NounType.Document, where: { 'system.subtype': 'ns-law-group-bare' } },
|
||||||
groupBy: ['team'],
|
groupBy: ['team'],
|
||||||
metrics: { count: { op: 'count' } }
|
metrics: { count: { op: 'count' } }
|
||||||
})
|
})
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue