From 8f9a9989e947c6b3a714f3f2c7452a2b27762c28 Mon Sep 17 00:00:00 2001 From: David Snelling Date: Mon, 3 Aug 2026 13:27:36 -0700 Subject: [PATCH 01/13] =?UTF-8?q?feat(namespace):=20the=20one=20field-addr?= =?UTF-8?q?essing=20law=20as=20a=20single=20source=20of=20truth=20?= =?UTF-8?q?=E2=80=94=20parseFieldAddress=20+=20the=20ruled=20ten-scalar=20?= =?UTF-8?q?system=20maps=20+=20plumbing=20invisibility=20+=20refusal=20bui?= =?UTF-8?q?lders=20(module=20only;=20query=20surfaces=20wire=20in=20next)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/db/fieldAddressing.ts | 246 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 246 insertions(+) create mode 100644 src/db/fieldAddressing.ts diff --git a/src/db/fieldAddressing.ts b/src/db/fieldAddressing.ts new file mode 100644 index 00000000..0ee09a05 --- /dev/null +++ b/src/db/fieldAddressing.ts @@ -0,0 +1,246 @@ +/** + * @module db/fieldAddressing + * @description The one field-addressing law for every query surface (find()'s + * `where` / `orderBy` / `groupBy`, aggregation `source.where`), ruled + * 2026-08-03 after a production incident in which a user metadata field + * named `level` was silently shadowed by the engine's internal HNSW node + * layer (VENUE-BRAINY-ORDERBY-NOOP — thread id kept verbatim as the audit + * key; it names no product): + * + * 1. A BARE field name addresses the user's metadata field. Always. + * No priority resolution, no fallback chain — `orderBy: 'level'` + * reads `entity.metadata.level`, full stop. + * 2. `system.` addresses an engine scalar, reachable ONLY with the + * explicit prefix. The entity map is exactly ten scalars; the relation + * map mirrors it with `verb`/`sourceId`/`targetId` as the structural + * members. + * 3. Engine plumbing (`vector`, `connections`, `level`, `data`, `_rev`) is + * INVISIBLE to the query surface in either spelling — `system.level` + * refuses; bare `level` is the user's field. + * 4. `metadata.` is the explicit spelling of the bare form — + * identical semantics on every path. + * 5. Anything unresolvable refuses with a TYPED error naming both + * candidate spellings — an accepted name either works or refuses; + * there is no third state. + * + * This module is the SINGLE source of truth for the law: parsing, the maps, + * and the refusal builders live here so the JS engine, the provider seams, + * and the cross-engine conformance suite can never drift on the contract. + */ + +import type { HNSWNounWithMetadata, HNSWVerbWithMetadata } from '../coreTypes.js' + +/** + * @description The entity-side `system.*` map — EXACTLY the ten engine + * scalars David ruled queryable (2026-08-03). Adding a name here is a + * cross-engine contract change: the native accelerator's conformance suite + * pins this list verbatim, so any edit must ship as a paired release. + */ +export const SYSTEM_ENTITY_SCALARS: ReadonlySet = new Set([ + 'id', + 'type', + 'subtype', + 'createdAt', + 'updatedAt', + 'confidence', + 'weight', + 'visibility', + 'service', + 'createdBy' +]) + +/** + * @description The relation-side `system.*` map — the verb mirror of + * {@link SYSTEM_ENTITY_SCALARS}: `verb`, `sourceId`, `targetId` are the + * structural members beside the eight shared scalars. Same one law, same + * pairing rule for edits. + */ +export const SYSTEM_RELATION_SCALARS: ReadonlySet = new Set([ + 'verb', + 'sourceId', + 'targetId', + 'subtype', + 'createdAt', + 'updatedAt', + 'confidence', + 'weight', + 'visibility', + 'service', + 'createdBy' +]) + +/** + * @description Engine plumbing — never addressable from the query surface in + * ANY spelling. `level` is the HNSW node layer (the incident field: listing + * it as resolvable shadowed real user data); `data` is the payload container, + * not a scalar — content is reached through the content/text-search APIs, + * and addressing it as a sortable field would lie about its shape. + */ +export const PLUMBING_FIELDS: ReadonlySet = new Set([ + 'vector', + 'connections', + 'level', + 'data', + '_rev' +]) + +/** @description Which record kind a field address is being resolved against. */ +export type FieldAddressKind = 'entity' | 'relation' + +/** + * @description A parsed, law-valid field address. `scope` says which side of + * the record the name lives on; `field` is the unprefixed name to read. + */ +export interface FieldAddress { + /** 'metadata' = the user's field (bare or `metadata.`-prefixed); 'system' = an engine scalar. */ + scope: 'metadata' | 'system' + /** The field name with any scope prefix removed. */ + field: string + /** The exact spelling the caller used — preserved for error text and telemetry. */ + raw: string +} + +/** + * Parse a query-surface field name under the one law. Pure and data-blind: + * this validates the ADDRESS (spelling + map membership), not whether any + * row actually carries the field — data-aware refusals (the did-you-mean + * for a bare system-scalar name no row carries) belong to the query layer, + * which calls {@link buildUnresolvableMessage} with index knowledge. + * + * @param raw - The field name as the caller wrote it (`level`, + * `metadata.level`, `system.createdAt`, …) + * @param kind - Entity or relation resolution (selects the system map) + * @returns The parsed {@link FieldAddress} + * @throws {InvalidFieldAddressError} for a `system.*` name outside the ruled + * map (including every plumbing field) or a malformed spelling — the error + * text enumerates the valid system scalars so the fix is in the message. + * + * @example + * parseFieldAddress('level', 'entity') // { scope: 'metadata', field: 'level' } + * parseFieldAddress('metadata.level', 'entity') // { scope: 'metadata', field: 'level' } + * parseFieldAddress('system.createdAt', 'entity') // { scope: 'system', field: 'createdAt' } + * parseFieldAddress('system.level', 'entity') // throws — plumbing is invisible + */ +export function parseFieldAddress( + raw: string, + kind: FieldAddressKind +): FieldAddress { + const systemMap = + kind === 'entity' ? SYSTEM_ENTITY_SCALARS : SYSTEM_RELATION_SCALARS + + if (raw.startsWith('system.')) { + const field = raw.slice('system.'.length) + if (!systemMap.has(field)) { + throw new InvalidFieldAddressError(raw, kind, systemMap) + } + return { scope: 'system', field, raw } + } + + if (raw.startsWith('metadata.')) { + const field = raw.slice('metadata.'.length) + if (field.length === 0) { + throw new InvalidFieldAddressError(raw, kind, systemMap) + } + return { scope: 'metadata', field, raw } + } + + if (raw.length === 0) { + throw new InvalidFieldAddressError(raw, kind, systemMap) + } + + // Bare name = the user's metadata field. Always. Even when the same name + // exists in the system map — `confidence` as a bare name is the user's + // metadata field named confidence; the engine scalar is system.confidence. + return { scope: 'metadata', field: raw, raw } +} + +/** + * Read the addressed value off an entity. The ONLY sanctioned way a query + * surface turns a {@link FieldAddress} into a value — direct property reads + * against records re-create the shadow class this module exists to kill. + * + * @returns The value, or `undefined` when the record does not carry it + * (missing values sort LAST in both directions per the ordering contract — + * they are never grounds for dropping a row). + */ +export function readEntityFieldAddress( + entity: HNSWNounWithMetadata, + address: FieldAddress +): unknown { + if (address.scope === 'system') { + return (entity as unknown as Record)[address.field] + } + return entity.metadata?.[address.field] +} + +/** + * Relation twin of {@link readEntityFieldAddress}. The stored flat record + * keys the relation type under `verb`; public Relation shapes may carry it + * as `type` — both spellings of the record are read, the ADDRESS is always + * `system.verb`. + */ +export function readRelationFieldAddress( + verb: HNSWVerbWithMetadata, + address: FieldAddress +): unknown { + if (address.scope === 'system') { + const rec = verb as unknown as Record + if (address.field === 'verb') return rec.verb ?? rec.type + return rec[address.field] + } + return verb.metadata?.[address.field] +} + +/** + * Build the ruled did-you-mean refusal text for a bare name that resolved to + * metadata but is UNKNOWN to the index — the data-aware half of the law, + * called by the query layer once it has consulted the known-field set: + * + * "no metadata field 'createdAt' — did you mean system.createdAt or + * metadata.createdAt?" + * + * When the bare name is NOT a system scalar the system candidate is omitted + * (there is only one thing the caller could have meant; the refusal exists + * because refusing beats silently sorting nothing). + */ +export function buildUnresolvableMessage( + raw: string, + kind: FieldAddressKind +): string { + const systemMap = + kind === 'entity' ? SYSTEM_ENTITY_SCALARS : SYSTEM_RELATION_SCALARS + if (systemMap.has(raw)) { + return ( + `no metadata field '${raw}' — did you mean system.${raw} or metadata.${raw}? ` + + `(bare names always address your metadata; engine fields need the system. prefix)` + ) + } + return ( + `no metadata field '${raw}' on this store — nothing carries it, so an ordered or ` + + `filtered read against it cannot mean anything. Spell it metadata.${raw} once the ` + + `field exists, or check the field name.` + ) +} + +/** + * @description Refusal for a malformed or out-of-map field ADDRESS — + * `system.` (including all plumbing), an empty + * name, or a bare `metadata.` prefix. The message carries the full valid + * system map so the fix never needs a docs lookup. + */ +export class InvalidFieldAddressError extends Error { + public readonly raw: string + public readonly kind: FieldAddressKind + + constructor(raw: string, kind: FieldAddressKind, systemMap: ReadonlySet) { + const valid = [...systemMap].map((f) => `system.${f}`).join(', ') + super( + `'${raw}' is not an addressable ${kind} field. Bare names address your own ` + + `metadata fields; engine fields are exactly: ${valid}. Engine plumbing ` + + `(vector, connections, level, data, _rev) is not part of the query surface.` + ) + this.name = 'InvalidFieldAddressError' + this.raw = raw + this.kind = kind + } +} From d8d0b55f9d85bf044c80a464a692db8931b2b595 Mon Sep 17 00:00:00 2001 From: David Snelling Date: Mon, 3 Aug 2026 13:36:05 -0700 Subject: [PATCH 02/13] =?UTF-8?q?test(namespace)+docs:=20the=20cross-engin?= =?UTF-8?q?e=20conformance=20suite=20(self-arming=20=E2=80=94=20skips=20un?= =?UTF-8?q?til=20the=20resolver=20exports=20land)=20+=20the=20public=20fie?= =?UTF-8?q?ld-addressing=20docs=20page;=20sidebar=20order=20deconflicted?= =?UTF-8?q?=20to=207?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/concepts/field-addressing.md | 196 ++++++++++ tests/conformance/namespace-law.test.ts | 484 ++++++++++++++++++++++++ 2 files changed, 680 insertions(+) create mode 100644 docs/concepts/field-addressing.md create mode 100644 tests/conformance/namespace-law.test.ts diff --git a/docs/concepts/field-addressing.md b/docs/concepts/field-addressing.md new file mode 100644 index 00000000..863f7474 --- /dev/null +++ b/docs/concepts/field-addressing.md @@ -0,0 +1,196 @@ +--- +title: Field addressing: your fields and system fields +slug: concepts/field-addressing +public: true +category: concepts +template: concept +order: 7 +description: The one rule for every query-surface field name — a bare name always means your metadata, system. reaches the ten engine scalars explicitly, and anything else refuses by name. +next: + - concepts/consistency-model +--- + +# Field addressing: your fields and system fields + +Every query surface in Brainy — `find()`'s `where`, `orderBy`, aggregation +`groupBy`, and aggregation `source.where` — resolves field names by one rule, +with no exceptions: + +> **A bare field name always means your metadata. `system.` reaches an +> engine scalar, and only when you spell it explicitly.** + +```typescript +await brain.find({ orderBy: 'level' }) // reads entity.metadata.level — YOUR field +await brain.find({ orderBy: 'system.createdAt' }) // reads the engine's createdAt scalar +await brain.find({ orderBy: 'metadata.level' }) // identical to bare 'level' — explicit scope +``` + +There is no priority list, no "try the system field, fall back to metadata" +behavior, and no name that resolves differently depending on what else +happens to exist on your entities. A field called `level`, `score`, +`createdAt`, or `type` in your own `metadata` is read as *your* field, every +time, by its bare name. + +## Why this rule exists + +An internal report from a production deployment found that a user metadata +field literally named `level` was being silently shadowed by the engine's +own internal index layer field of the same name — every sort by `level` +returned insertion order, with no error raised. This rule makes that class of +bug structurally impossible: bare names belong to you, unconditionally, and +anything that isn't yours has to be spelled out. + +## The system scalars + +`system.` addresses exactly ten scalars on an entity — no more, no +fewer: + +| System field | What it is | +|---|---| +| `system.id` | The entity's id | +| `system.type` | The entity's `NounType` | +| `system.subtype` | The per-app sub-classification passed to `add()` | +| `system.createdAt` | When the entity was created | +| `system.updatedAt` | When the entity was last written | +| `system.confidence` | The `confidence` param (0–1) | +| `system.weight` | The `weight` param | +| `system.visibility` | `'public'` / `'internal'` (see the visibility tiers in [Consistency Model](./consistency-model.md)) | +| `system.service` | The multi-tenancy `service` tag | +| `system.createdBy` | Who/what created the entity | + +Relationships mirror the same eight shared scalars (`subtype`, `createdAt`, +`updatedAt`, `confidence`, `weight`, `visibility`, `service`, `createdBy`) +plus three of their own: + +| System field (relationship) | What it is | +|---|---| +| `system.verb` | The relationship's `VerbType` | +| `system.sourceId` | The id of the entity the relationship starts from | +| `system.targetId` | The id of the entity the relationship points to | + +Anything not on these two lists is not a system scalar — `system.` for +any other name refuses (see "Refusal semantics" below), even if that name +sounds like it should be engine-owned. + +## Invisible plumbing — never addressable, in either spelling + +Five names are pure engine internals. They are not reachable as a bare name, +and not reachable as `system.` either — they simply have no place on +the query surface: + +- **`vector`** — the stored embedding. It participates in similarity search + (`query`, `near`, vector `find()`), never in `where`/`orderBy`/`groupBy`. +- **`connections`** — graph adjacency. Reached through `connected` and + `brain.related()`, not through field addressing. +- **`level`** — the internal index layer number used by the nearest-neighbor + graph. It is pure index plumbing with no query-surface meaning at all — + which is exactly why a user field of the same name must never be shadowed + by it. `level` as a bare name is always yours; there is no engine-owned + spelling of it to compete with. +- **`data`** — your entity's content payload, not a scalar. It can be a + string, a number, or an arbitrary object, so sorting or filtering it as a + single comparable value would lie about its actual shape. Content is + reached through the content/text-search APIs (`query`, `searchMode: + 'text'`), not through `where`/`orderBy`. +- **`_rev`** — the per-entity revision counter used for optimistic + concurrency (`ifRev`). It is a CAS token, not a queryable dimension. + +`system.level`, `system.vector`, and `system.data` all refuse for the same +reason: they are not in the ten-scalar system map, full stop. + +## `metadata.` — the explicit spelling of "mine" + +Prefix any field with `metadata.` to say the same thing a bare name already +says, spelled out. The two are interchangeable everywhere a field name is +accepted, including `orderBy`: + +```typescript +await brain.find({ where: { 'customer.tier': 'gold' } }) +await brain.find({ where: { 'metadata.customer.tier': 'gold' } }) // identical +await brain.find({ orderBy: 'metadata.score', order: 'desc' }) // identical to orderBy: 'score' +``` + +Reach for the explicit spelling when it reads more clearly next to a +`system.` field in the same query — for example, sorting by your own `score` +while filtering on `system.confidence`. + +## Refusal semantics + +A name that resolves to neither your metadata nor a system scalar is a typed +refusal, not a silent empty result and not a guess. Refusals name **both** +candidates, so the fix is always in the error text: + +```typescript +await brain.find({ orderBy: 'createdAt' }) +// UnresolvableFieldError: no metadata field 'createdAt' — did you mean +// system.createdAt or metadata.createdAt? +``` + +`UnresolvableFieldError` is exported from the package root: + +```typescript +import { UnresolvableFieldError } from '@soulcraft/brainy' + +try { + await brain.find({ orderBy: 'createdAt' }) +} catch (err) { + if (err instanceof UnresolvableFieldError) { + // err.message names both candidates — usually enough to fix the call site. + } +} +``` + +A handful of `find()` options are not implemented yet: `cursor`, +`includeRelations`, and `writeOnly`. Rather than accepting them and quietly +ignoring the option, `find()` refuses with `UnsupportedFindOptionError` — +also exported from the package root — so a call site can never believe an +unimplemented option took effect when it didn't. + +## The ordering contract + +`orderBy` behaves identically regardless of which engine (the pure-TypeScript +path or a native accelerator) is serving the query: + +- An entity missing the `orderBy` field, or holding `null` on it, sorts + **LAST — in both `asc` and `desc`**. It is never treated as "smaller than + everything" in one direction and "larger than everything" in the other; it + is simply last, either way. +- Rows are **never dropped** from an ordered read because they lack the + field — a missing value changes position, never presence. +- Ties on the `orderBy` field break by **id ascending**, regardless of the + primary sort direction. + +```typescript +// employees: [{ score: 9 }, { score: 5 }, { /* no score field */ }] +await brain.find({ orderBy: 'score', order: 'desc' }) // [9, 5, missing] — missing is last +await brain.find({ orderBy: 'score', order: 'asc' }) // [5, 9, missing] — missing is STILL last +``` + +## Migrating existing call sites + +If you have call sites written before this rule shipped that rely on a bare +system name — `orderBy: 'createdAt'`, `where: { confidence: { greaterThan: +0.8 } }`, and similar — they now refuse instead of silently resolving to the +engine field. The fix is always in the error: swap the bare name for +`system.` (or `metadata.` if you actually meant your own field +of that name, and it happens to share a name with a system scalar): + +```typescript +// Before: bare 'createdAt' silently meant the engine's timestamp. +await brain.find({ orderBy: 'createdAt' }) + +// After: say which one you meant. +await brain.find({ orderBy: 'system.createdAt' }) // the engine timestamp +await brain.find({ orderBy: 'metadata.createdAt' }) // your own field named createdAt, if you have one +``` + +There is no silent migration path by design — every ambiguous call site +surfaces as a refusal naming its own fix, once, the first time it runs +against the new rule. + +## Where to go next + +- [Consistency Model](./consistency-model.md) — the separate (and + longer-standing) contract for *reserved* fields: which names may never + appear inside a `metadata` bag at write time, distinct from this page's + read-time addressing rule. diff --git a/tests/conformance/namespace-law.test.ts b/tests/conformance/namespace-law.test.ts new file mode 100644 index 00000000..91227587 --- /dev/null +++ b/tests/conformance/namespace-law.test.ts @@ -0,0 +1,484 @@ +/** + * @module tests/conformance/namespace-law + * @description Conformance suite for the ruled field-addressing contract + * announced in RELEASES.md ("Coming next... one field-addressing law — bare + * names = user metadata, `system.` for engine fields, typed refusals + * for unresolvable names"). This suite is the drift-proof shared by this + * engine and its native accelerator: both must satisfy every test here + * bit-for-bit, because they implement the SAME contract independently. + * + * The rule, in full: + * 1. A bare field name in `where` / `orderBy` / `groupBy` / aggregation + * `source.where` ALWAYS means the caller's own `metadata` field. No + * priority resolution, no engine fallback — ever. + * 2. `system.` reaches an engine scalar, and ONLY an engine scalar, + * and ONLY when spelled explicitly. The addressable entity map is exactly + * ten names: id, type, subtype, createdAt, updatedAt, confidence, weight, + * visibility, service, createdBy. The relationship map is system.verb, + * system.sourceId, system.targetId, plus the eight scalars shared with + * entities. + * 3. Some names are invisible plumbing and are never addressable in either + * spelling: vector, connections, level, data, _rev. `system.level`, + * `system.vector`, and `system.data` all refuse — they are not in the + * system map. Bare `level` is a perfectly ordinary user field. + * 4. `metadata.` is the explicit-user-scope spelling: identical + * semantics to the bare spelling, valid everywhere the bare spelling is. + * 5. Anything that resolves to neither a user field nor a system scalar is a + * typed refusal naming both candidates (`UnresolvableFieldError`). + * Unimplemented `find()` options (`cursor`, `includeRelations`, + * `writeOnly`) refuse with `UnsupportedFindOptionError` instead of being + * silently accepted and ignored. + * 6. Ordering is identical on both engines: rows missing/null on the + * `orderBy` field sort LAST in BOTH directions and are never dropped; + * ties break by id ascending. + * + * The motivating incident (told generically — see CLAUDE.md naming rule): an + * internal report from a production deployment showed a user metadata field + * literally named `level` silently shadowed by the engine's internal HNSW + * node layer, breaking sort order with zero errors raised. This contract + * makes that class of bug impossible, and testable forever. + * + * SELF-SKIP: the resolver this suite pins is being built in a parallel + * session and has not landed on every branch yet. Rather than going red on + * a branch that simply hasn't caught up, the suite detects whether the + * contract is live by the one thing any conformant implementation must + * export — `UnresolvableFieldError` from the package root — and skips + * loudly (never silently) until it does. This is the house pattern: a + * sibling engine's gate once went red because a test armed before its + * feature existed. + */ +import { describe, it, expect, beforeEach, afterEach } from 'vitest' +import { Brainy } from '../../src/brainy.js' +import { NounType } from '../../src/types/graphTypes.js' +import * as brainyExports from '../../src/index.js' + +const stubEmbedding = async (text: string): Promise => { + const hash = text.split('').reduce((acc, char) => acc + char.charCodeAt(0), 0) + return new Array(384).fill(0).map((_, i) => Math.sin(hash + i)) +} + +// Detected purely by the exported error-class NAME — never by reaching into +// implementation internals. Both engines building this contract must export +// it from the package root, so this is a legitimate, implementation-agnostic +// readiness probe. +const lawActive = 'UnresolvableFieldError' in brainyExports +const UnresolvableFieldError = (brainyExports as Record).UnresolvableFieldError as new ( + ...args: any[] +) => Error +const UnsupportedFindOptionError = (brainyExports as Record) + .UnsupportedFindOptionError as new (...args: any[]) => Error + +// Always runs, regardless of lawActive — the loud signal that the rest of +// this file was skipped, and why. +it('namespace law armed?', () => { + if (!lawActive) { + console.warn( + '[conformance] namespace-law suite SKIPPED — UnresolvableFieldError not exported yet; arms when the resolver lands' + ) + } + expect(true).toBe(true) +}) + +/** + * Awaits `promise`, asserting it rejects with an instance of `ErrorClass` + * whose `.message` contains every string in `mustContain`. Fails loudly if + * the promise resolves instead of rejecting. + */ +async function expectRefusal( + promise: Promise, + ErrorClass: new (...args: any[]) => Error, + ...mustContain: string[] +): Promise { + let threw = false + try { + await promise + } catch (err) { + threw = true + expect(err).toBeInstanceOf(ErrorClass) + for (const fragment of mustContain) { + expect((err as Error).message).toContain(fragment) + } + } + expect(threw).toBe(true) +} + +describe.skipIf(!lawActive)('namespace law — bare/system/metadata field addressing', () => { + let brain: Brainy + + beforeEach(async () => { + brain = new Brainy({ + requireSubtype: false, + storage: { type: 'memory' as const }, + embeddingFunction: stubEmbedding + }) + await brain.init() + }) + + afterEach(async () => { + await brain.close() + }) + + /** The star case from the motivating incident: metadata.level 3/9/6. */ + async function addLevelRows(): Promise { + const ids: string[] = [] + for (const level of [3, 9, 6]) { + ids.push( + await brain.add({ + data: `probe level ${level}`, + type: NounType.Person, + subtype: 'ns-law-level', + metadata: { name: `p-${level}`, level } + }) + ) + } + return ids + } + + // ------------------------------------------------------------------- + // Rule 1 — bare field name = the user's metadata field, always. + // ------------------------------------------------------------------- + + it("bare orderBy 'level' reads user metadata, desc and asc (the star case)", async () => { + await addLevelRows() + + const desc = await brain.find({ + type: NounType.Person, + subtype: 'ns-law-level', + orderBy: 'level', + order: 'desc', + limit: 100 + }) + expect(desc.map((r: any) => r.metadata?.level)).toEqual([9, 6, 3]) + + const asc = await brain.find({ + type: NounType.Person, + subtype: 'ns-law-level', + orderBy: 'level', + order: 'asc', + limit: 100 + }) + expect(asc.map((r: any) => r.metadata?.level)).toEqual([3, 6, 9]) + }) + + it("bare where { level: N } matches the user's field", async () => { + const ids = await addLevelRows() + const hit = await brain.find({ type: NounType.Person, subtype: 'ns-law-level', where: { level: 9 } }) + expect(hit).toHaveLength(1) + expect(hit[0].id).toBe(ids[1]) + expect(hit[0].metadata?.level).toBe(9) + }) + + // ------------------------------------------------------------------- + // Rule 4 — metadata. is the explicit-user-scope spelling, + // identical semantics to bare, valid on every path including orderBy. + // ------------------------------------------------------------------- + + it("'metadata.level' resolves identically to bare 'level'", async () => { + await addLevelRows() + const desc = await brain.find({ + type: NounType.Person, + subtype: 'ns-law-level', + orderBy: 'metadata.level', + order: 'desc', + limit: 100 + }) + expect(desc.map((r: any) => r.metadata?.level)).toEqual([9, 6, 3]) + }) + + // ------------------------------------------------------------------- + // Rule 2 — system. reaches an engine scalar explicitly. + // ------------------------------------------------------------------- + + it('system.createdAt sorts by entity age', async () => { + const ids: string[] = [] + for (const name of ['first', 'second', 'third']) { + ids.push( + await brain.add({ + data: `aged ${name}`, + type: NounType.Person, + subtype: 'ns-law-aged', + metadata: { name } + }) + ) + // Guarantee distinct createdAt timestamps between adds. + await new Promise((resolve) => setTimeout(resolve, 5)) + } + + const asc = await brain.find({ + type: NounType.Person, + subtype: 'ns-law-aged', + orderBy: 'system.createdAt', + order: 'asc', + limit: 100 + }) + expect(asc.map((r: any) => r.id)).toEqual(ids) + + const desc = await brain.find({ + type: NounType.Person, + subtype: 'ns-law-aged', + orderBy: 'system.createdAt', + order: 'desc', + limit: 100 + }) + expect(desc.map((r: any) => r.id)).toEqual([...ids].reverse()) + }) + + it('where on system.confidence filters by the engine scalar', async () => { + const highId = await brain.add({ + data: 'high confidence row', + type: NounType.Person, + subtype: 'ns-law-confidence', + confidence: 0.95, + metadata: { name: 'hi' } + }) + await brain.add({ + data: 'low confidence row', + type: NounType.Person, + subtype: 'ns-law-confidence', + confidence: 0.4, + metadata: { name: 'lo' } + }) + + const hit = await brain.find({ + type: NounType.Person, + subtype: 'ns-law-confidence', + where: { 'system.confidence': 0.95 } + }) + expect(hit).toHaveLength(1) + expect(hit[0].id).toBe(highId) + }) + + it('groupBy on system.subtype groups by the engine scalar, not user metadata', async () => { + await brain.add({ data: 'i1', type: NounType.Document, subtype: 'invoice' }) + await brain.add({ data: 'i2', type: NounType.Document, subtype: 'invoice' }) + await brain.add({ data: 'r1', type: NounType.Document, subtype: 'receipt' }) + + brain.defineAggregate({ + name: 'ns_law_by_subtype_system', + source: { type: NounType.Document }, + groupBy: ['system.subtype'], + metrics: { count: { op: 'count' } } + }) + + const groups = await brain.queryAggregate('ns_law_by_subtype_system') + const invoiceGroup = groups.find((g) => Object.values(g.groupKey).includes('invoice')) + const receiptGroup = groups.find((g) => Object.values(g.groupKey).includes('receipt')) + expect(invoiceGroup?.metrics.count).toBe(2) + expect(receiptGroup?.metrics.count).toBe(1) + }) + + // ------------------------------------------------------------------- + // Rule 1 (groupBy face) — bare groupBy dimensions read user metadata, + // never the engine's own notion of the same-sounding name. + // ------------------------------------------------------------------- + + it('groupBy on a bare user metadata field groups by that field', async () => { + await brain.add({ + data: 'd1', + type: NounType.Document, + subtype: 'ns-law-group-bare', + metadata: { team: 'alpha' } + }) + await brain.add({ + data: 'd2', + type: NounType.Document, + subtype: 'ns-law-group-bare', + metadata: { team: 'alpha' } + }) + await brain.add({ + data: 'd3', + type: NounType.Document, + subtype: 'ns-law-group-bare', + metadata: { team: 'beta' } + }) + + brain.defineAggregate({ + name: 'ns_law_by_team_bare', + source: { type: NounType.Document, where: { subtype: 'ns-law-group-bare' } }, + groupBy: ['team'], + metrics: { count: { op: 'count' } } + }) + + const groups = await brain.queryAggregate('ns_law_by_team_bare') + const alphaGroup = groups.find((g) => Object.values(g.groupKey).includes('alpha')) + const betaGroup = groups.find((g) => Object.values(g.groupKey).includes('beta')) + expect(alphaGroup?.metrics.count).toBe(2) + expect(betaGroup?.metrics.count).toBe(1) + }) + + it('where on a bare user metadata field filters normally (score, not a system name)', async () => { + await brain.add({ + data: 'high score', + type: NounType.Person, + subtype: 'ns-law-score', + metadata: { score: 42 } + }) + await brain.add({ + data: 'low score', + type: NounType.Person, + subtype: 'ns-law-score', + metadata: { score: 7 } + }) + + const hit = await brain.find({ type: NounType.Person, subtype: 'ns-law-score', where: { score: 42 } }) + expect(hit).toHaveLength(1) + expect(hit[0].metadata?.score).toBe(42) + }) + + // ------------------------------------------------------------------- + // Rule 5 — typed refusals, naming both candidates. + // ------------------------------------------------------------------- + + it("bare orderBy 'createdAt' refuses when no such metadata field exists — names both candidates", async () => { + await brain.add({ + data: 'no metadata.createdAt here', + type: NounType.Person, + subtype: 'ns-law-refuse-createdAt', + metadata: { name: 'x' } + }) + + await expectRefusal( + brain.find({ + type: NounType.Person, + subtype: 'ns-law-refuse-createdAt', + orderBy: 'createdAt', + limit: 10 + }), + UnresolvableFieldError, + 'system.createdAt', + 'metadata.createdAt' + ) + }) + + // ------------------------------------------------------------------- + // Rule 3 — invisible plumbing refuses in either spelling; system. + // for a name that isn't in the ten-scalar map is unresolvable. + // ------------------------------------------------------------------- + + it('system.level refuses — level is invisible plumbing, never a system scalar', async () => { + await brain.add({ + data: 'has a level metadata field', + type: NounType.Person, + metadata: { level: 5 } + }) + await expectRefusal(brain.find({ orderBy: 'system.level', limit: 10 }), UnresolvableFieldError) + }) + + it('system.vector refuses — vector is invisible plumbing, never a system scalar', async () => { + await brain.add({ data: 'row', type: NounType.Person, metadata: { name: 'x' } }) + await expectRefusal(brain.find({ orderBy: 'system.vector', limit: 10 }), UnresolvableFieldError) + }) + + it('system.data refuses — data is a payload container, never a system scalar', async () => { + await brain.add({ data: 'row', type: NounType.Person, metadata: { name: 'x' } }) + await expectRefusal(brain.find({ orderBy: 'system.data', limit: 10 }), UnresolvableFieldError) + }) + + // ------------------------------------------------------------------- + // Rule 6 — the ordering contract. + // ------------------------------------------------------------------- + + async function addOrderingProbeRows(): Promise<{ ranked: string[]; missing: string }> { + const low = await brain.add({ + data: 'low score', + type: NounType.Person, + subtype: 'ns-law-ordering', + metadata: { score: 5 } + }) + const high = await brain.add({ + data: 'high score', + type: NounType.Person, + subtype: 'ns-law-ordering', + metadata: { score: 9 } + }) + const missing = await brain.add({ + data: 'no score field at all', + type: NounType.Person, + subtype: 'ns-law-ordering', + metadata: { name: 'no-score' } + }) + return { ranked: [low, high], missing } + } + + it('a row missing the orderBy field sorts LAST in desc — and is never dropped', async () => { + const { ranked, missing } = await addOrderingProbeRows() + const desc = await brain.find({ + type: NounType.Person, + subtype: 'ns-law-ordering', + orderBy: 'score', + order: 'desc', + limit: 100 + }) + expect(desc).toHaveLength(3) + expect(desc.map((r: any) => r.id)).toEqual([ranked[1], ranked[0], missing]) + }) + + it('a row missing the orderBy field sorts LAST in asc too — and is never dropped', async () => { + const { ranked, missing } = await addOrderingProbeRows() + const asc = await brain.find({ + type: NounType.Person, + subtype: 'ns-law-ordering', + orderBy: 'score', + order: 'asc', + limit: 100 + }) + expect(asc).toHaveLength(3) + expect(asc.map((r: any) => r.id)).toEqual([ranked[0], ranked[1], missing]) + }) + + it('ties on the orderBy field break by id ascending, in BOTH directions', async () => { + const tiedIds: string[] = [] + for (let i = 0; i < 4; i++) { + tiedIds.push( + await brain.add({ + data: `tied ${i}`, + type: NounType.Person, + subtype: 'ns-law-ties', + metadata: { score: 5 } + }) + ) + } + const expectedOrder = [...tiedIds].sort() + + const asc = await brain.find({ + type: NounType.Person, + subtype: 'ns-law-ties', + orderBy: 'score', + order: 'asc', + limit: 100 + }) + expect(asc.map((r: any) => r.id)).toEqual(expectedOrder) + + const desc = await brain.find({ + type: NounType.Person, + subtype: 'ns-law-ties', + orderBy: 'score', + order: 'desc', + limit: 100 + }) + // Same tie-break ordering regardless of the primary direction — the + // contract states one universal rule ("id ascending"), not "reverse of + // the primary order". + expect(desc.map((r: any) => r.id)).toEqual(expectedOrder) + }) + + // ------------------------------------------------------------------- + // Rule 5 (options face) — unimplemented find() options refuse loudly + // instead of being accepted and silently ignored. + // ------------------------------------------------------------------- + + it('find({ cursor }) refuses with UnsupportedFindOptionError', async () => { + await brain.add({ data: 'row', type: NounType.Person, metadata: { name: 'x' } }) + await expectRefusal(brain.find({ cursor: 'anything', limit: 10 }), UnsupportedFindOptionError) + }) + + it('find({ includeRelations }) refuses with UnsupportedFindOptionError', async () => { + await brain.add({ data: 'row', type: NounType.Person, metadata: { name: 'x' } }) + await expectRefusal(brain.find({ includeRelations: true, limit: 10 }), UnsupportedFindOptionError) + }) + + it('find({ writeOnly }) refuses with UnsupportedFindOptionError', async () => { + await brain.add({ data: 'row', type: NounType.Person, metadata: { name: 'x' } }) + await expectRefusal(brain.find({ writeOnly: true, limit: 10 }), UnsupportedFindOptionError) + }) +}) From 56deb2e8883f9c879caf3b4d8b5850461893d967 Mon Sep 17 00:00:00 2001 From: David Snelling Date: Mon, 3 Aug 2026 14:05:26 -0700 Subject: [PATCH 03/13] =?UTF-8?q?fix(namespace):=20the=20JS=20sorted=20fal?= =?UTF-8?q?lback=20honors=20the=20ruled=20ordering=20contract=20=E2=80=94?= =?UTF-8?q?=20nulls=20last=20in=20BOTH=20directions=20(was=20nulls-first?= =?UTF-8?q?=20on=20desc)=20+=20deterministic=20id-ascending=20tie-break?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/utils/metadataIndex.ts | 28 +++++++++++++++++++--------- 1 file changed, 19 insertions(+), 9 deletions(-) diff --git a/src/utils/metadataIndex.ts b/src/utils/metadataIndex.ts index fdb17c22..cf5d521e 100644 --- a/src/utils/metadataIndex.ts +++ b/src/utils/metadataIndex.ts @@ -2260,20 +2260,30 @@ export class MetadataIndexManager implements MetadataIndexProvider { } idValuePairs.sort((a, b) => { - if (a.value == null && b.value == null) return 0 - if (a.value == null) return order === 'asc' ? 1 : -1 - if (b.value == null) return order === 'asc' ? -1 : 1 - if (a.value === b.value) return 0 + // Ordering contract (cross-engine, ruled 2026-08-03): missing/null + // values sort LAST in BOTH directions — the direction flip never moves + // them to the front — and ties break by id ascending, so an ordered + // read is deterministic and identical on both engines. Rows are never + // dropped for lacking the field. + const aNull = a.value == null + const bNull = b.value == null + if (aNull || bNull) { + if (aNull && bNull) return a.id < b.id ? -1 : a.id > b.id ? 1 : 0 + return aNull ? 1 : -1 + } // Numbers compare numerically; everything else by code-point (UTF-8 byte) order. // This makes the JS fallback sort match cor's native column store exactly // (numeric i64/f64 vs code-point strings) and stay deterministic across // environments, unlike the `<` operator's UTF-16 ordering for strings. - let comparison: number - if (typeof a.value === 'number' && typeof b.value === 'number') { - comparison = a.value < b.value ? -1 : 1 - } else { - comparison = compareCodePoints(String(a.value), String(b.value)) + let comparison = 0 + if (a.value !== b.value) { + if (typeof a.value === 'number' && typeof b.value === 'number') { + comparison = a.value < b.value ? -1 : 1 + } else { + comparison = compareCodePoints(String(a.value), String(b.value)) + } } + if (comparison === 0) return a.id < b.id ? -1 : a.id > b.id ? 1 : 0 return order === 'asc' ? comparison : -comparison }) From 5502abcdd8f60e7484940cb00445c624a087df96 Mon Sep 17 00:00:00 2001 From: David Snelling Date: Mon, 3 Aug 2026 14:39:06 -0700 Subject: [PATCH 04/13] =?UTF-8?q?test(namespace):=20unit=20pins=20for=20th?= =?UTF-8?q?e=20pure=20law=20=E2=80=94=20the=20ruled=20maps=20verbatim=20(i?= =?UTF-8?q?ncl.=20the=20relation=20mirror,=20unpinnable=20via=20public=20A?= =?UTF-8?q?PI),=20plumbing=20refusals=20both=20kinds,=20did-you-mean=20tex?= =?UTF-8?q?t?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- tests/unit/db/fieldAddressing.test.ts | 141 ++++++++++++++++++++++++++ 1 file changed, 141 insertions(+) create mode 100644 tests/unit/db/fieldAddressing.test.ts diff --git a/tests/unit/db/fieldAddressing.test.ts b/tests/unit/db/fieldAddressing.test.ts new file mode 100644 index 00000000..f7ca1cbe --- /dev/null +++ b/tests/unit/db/fieldAddressing.test.ts @@ -0,0 +1,141 @@ +/** + * @module tests/unit/db/fieldAddressing + * @description Unit pins for the one field-addressing law (ruled 2026-08-03). + * These pin the PURE half of the law — parsing, the ruled maps, plumbing + * invisibility, refusal text — including the RELATION map, which cannot be + * pinned through the public query API today (related() carries no + * field-addressing options): the verb mirror is contract-tested here at the + * module level so the two engines cannot drift on it. + */ +import { describe, it, expect } from 'vitest' +import { + SYSTEM_ENTITY_SCALARS, + SYSTEM_RELATION_SCALARS, + PLUMBING_FIELDS, + parseFieldAddress, + buildUnresolvableMessage, + InvalidFieldAddressError +} from '../../../src/db/fieldAddressing.js' + +describe('field-addressing law — pure module pins', () => { + it('the entity system map is EXACTLY the ruled ten scalars', () => { + expect([...SYSTEM_ENTITY_SCALARS].sort()).toEqual( + [ + 'confidence', + 'createdAt', + 'createdBy', + 'id', + 'service', + 'subtype', + 'type', + 'updatedAt', + 'visibility', + 'weight' + ].sort() + ) + }) + + it('the relation system map is the ruled verb mirror', () => { + expect([...SYSTEM_RELATION_SCALARS].sort()).toEqual( + [ + 'verb', + 'sourceId', + 'targetId', + 'confidence', + 'createdAt', + 'createdBy', + 'service', + 'subtype', + 'updatedAt', + 'visibility', + 'weight' + ].sort() + ) + }) + + it('plumbing is exactly the ruled five, and none of it leaks into a system map', () => { + expect([...PLUMBING_FIELDS].sort()).toEqual( + ['_rev', 'connections', 'data', 'level', 'vector'].sort() + ) + for (const field of PLUMBING_FIELDS) { + expect(SYSTEM_ENTITY_SCALARS.has(field)).toBe(false) + expect(SYSTEM_RELATION_SCALARS.has(field)).toBe(false) + } + }) + + it('bare names address user metadata — even when the name matches a system scalar', () => { + expect(parseFieldAddress('level', 'entity')).toEqual({ + scope: 'metadata', + field: 'level', + raw: 'level' + }) + expect(parseFieldAddress('confidence', 'entity').scope).toBe('metadata') + expect(parseFieldAddress('createdAt', 'entity').scope).toBe('metadata') + expect(parseFieldAddress('verb', 'relation').scope).toBe('metadata') + }) + + it('metadata.-prefix is the explicit spelling of the bare form', () => { + expect(parseFieldAddress('metadata.level', 'entity')).toEqual({ + scope: 'metadata', + field: 'level', + raw: 'metadata.level' + }) + }) + + it('system.-prefix reaches exactly the map — entity and relation', () => { + for (const field of SYSTEM_ENTITY_SCALARS) { + expect(parseFieldAddress(`system.${field}`, 'entity')).toEqual({ + scope: 'system', + field, + raw: `system.${field}` + }) + } + for (const field of SYSTEM_RELATION_SCALARS) { + expect(parseFieldAddress(`system.${field}`, 'relation').scope).toBe('system') + } + // The structural relation members are NOT entity scalars. + expect(() => parseFieldAddress('system.verb', 'entity')).toThrow(InvalidFieldAddressError) + expect(() => parseFieldAddress('system.sourceId', 'entity')).toThrow(InvalidFieldAddressError) + }) + + it('plumbing refuses in the system spelling, on both record kinds', () => { + for (const field of PLUMBING_FIELDS) { + expect(() => parseFieldAddress(`system.${field}`, 'entity')).toThrow( + InvalidFieldAddressError + ) + expect(() => parseFieldAddress(`system.${field}`, 'relation')).toThrow( + InvalidFieldAddressError + ) + } + }) + + it('refusal text carries the whole valid map — the fix lives in the message', () => { + try { + parseFieldAddress('system.level', 'entity') + expect.unreachable('should have thrown') + } catch (e) { + const msg = (e as Error).message + for (const field of SYSTEM_ENTITY_SCALARS) { + expect(msg).toContain(`system.${field}`) + } + expect(msg).toContain('plumbing') + } + }) + + it('malformed addresses refuse: empty name, bare metadata. prefix', () => { + expect(() => parseFieldAddress('', 'entity')).toThrow(InvalidFieldAddressError) + expect(() => parseFieldAddress('metadata.', 'entity')).toThrow(InvalidFieldAddressError) + }) + + it('the did-you-mean names BOTH candidates for a system-colliding bare name', () => { + const msg = buildUnresolvableMessage('createdAt', 'entity') + expect(msg).toContain('system.createdAt') + expect(msg).toContain('metadata.createdAt') + }) + + it('a non-colliding unknown bare name gets the single-candidate refusal', () => { + const msg = buildUnresolvableMessage('scoore', 'entity') + expect(msg).not.toContain('system.scoore') + expect(msg).toContain('metadata.scoore') + }) +}) From fcb24ab627a63e69df0286ea77d1522df32ba2fc Mon Sep 17 00:00:00 2001 From: David Snelling Date: Mon, 3 Aug 2026 15:11:24 -0700 Subject: [PATCH 05/13] =?UTF-8?q?docs(namespace):=20the=20d.ts=20JSDoc=20w?= =?UTF-8?q?ave=20=E2=80=94=20the=20sealed=20field-addressing=20law=20on=20?= =?UTF-8?q?the=20full=20find=20+=20aggregation=20surface,=20present-tense,?= =?UTF-8?q?=20with=20the=20refusal=20semantics=20and=20migration=20note=20?= =?UTF-8?q?inline=20(comment-only;=20verified=20zero=20code=20lines=20chan?= =?UTF-8?q?ged)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/types/brainy.types.ts | 123 ++++++++++++++++++++++++++++++++------ 1 file changed, 106 insertions(+), 17 deletions(-) diff --git a/src/types/brainy.types.ts b/src/types/brainy.types.ts index 89be78b9..6c133cf0 100644 --- a/src/types/brainy.types.ts +++ b/src/types/brainy.types.ts @@ -498,6 +498,43 @@ export interface UpdateRelationParams { * - **Graph:** `connected` for relationship traversal (via GraphAdjacencyIndex) * * See also: [Query Operators](../../docs/QUERY_OPERATORS.md) for all `where` operators. + * + * @remarks + * **Field-addressing law.** Governs every query-surface field name — `where` + * and `orderBy` on this interface, plus `AggregateSource.where` and + * `AggregateDefinition.groupBy` in the aggregation engine: + * + * 1. A bare name (e.g. `'level'`, `'rank'`, `'score'`) always means the + * caller's own metadata field — it reads `entity.metadata.`. There + * is no fallback to an engine-internal field of the same name and no + * priority resolution between the two; metadata wins unconditionally. + * 2. `system.` reaches an engine scalar, explicitly, and only for + * these ten: `id`, `type`, `subtype`, `createdAt`, `updatedAt`, + * `confidence`, `weight`, `visibility`, `service`, `createdBy`. + * 3. `vector`, `connections`, `level` (the engine-internal node field — a + * different thing from a user metadata field also named `level`), + * `data`, and `_rev` are invisible plumbing: neither spelling can + * address them from a query surface. + * 4. `metadata.` is the explicit spelling of the bare form and means + * exactly the same thing as rule 1. + * 5. A name that matches none of the above — most often a bare name that + * collides with one of the ten system-scalar names in rule 2 — REFUSES + * with a typed {@link UnresolvableFieldError} naming both candidates, + * e.g. `no metadata field 'createdAt' — did you mean system.createdAt or + * metadata.createdAt?`. The same loud-refusal principle covers whole + * options: the previously accepted-and-silently-ignored `cursor`, + * `includeRelations`, and `writeOnly` now throw + * {@link UnsupportedFindOptionError} instead of doing nothing. + * 6. **Ordering contract** (identical on the pure-JS engine and the native + * accelerator): rows missing or `null` on the `orderBy` field sort LAST + * in BOTH `asc` and `desc` order and are never dropped from the result; + * ties break by `id` ascending. + * + * Migration note: a call site written against the old rule — e.g. + * `orderBy: 'createdAt'` or `where: { visibility: 'internal' }` meaning the + * engine scalar — now refuses instead of silently reading the wrong field. + * The thrown error names the exact fix (`system.createdAt`). A loud + * refusal with the fix in hand beats a silent behavior flip. */ export interface FindParams { // Vector Intelligence @@ -516,7 +553,18 @@ export interface FindParams { * `{ exists: true }`, `{ missing: true }`) use `where: { subtype: { …operators… } }`. */ subtype?: string | string[] - /** Metadata filters using BFO operators (e.g., `{ year: { greaterThan: 2020 } }`) */ + /** + * Metadata filters using BFO operators (e.g., `{ year: { greaterThan: 2020 } }`). + * Field names follow the field-addressing law — see the `@remarks` on + * {@link FindParams}: a bare key is always the caller's metadata field; + * an engine scalar needs the explicit `system.` form. + * + * @example + * ```typescript + * await brain.find({ where: { level: { greaterThan: 5 } } }) // metadata.level + * await brain.find({ where: { 'system.visibility': 'internal' } }) // engine scalar + * ``` + */ where?: Partial // Visibility @@ -548,29 +596,49 @@ export interface FindParams { // Control options limit?: number // Max results (default: 10) offset?: number // Skip N results + /** + * @deprecated Not implemented. Passing `cursor` throws + * {@link UnsupportedFindOptionError} — it used to be accepted and + * silently ignored, which masked that no cursor pagination ever ran. Use + * `offset` / `limit` until cursor pagination ships. + */ cursor?: string // Cursor-based pagination // Sorting /** - * Field to sort by. User metadata fields sort by their stored values — - * including natural names like `level`, `rank`, or `score` (an engine-internal - * field can never shadow your metadata; fixed 2026-08 after a production - * report). System timestamps (`createdAt`, `updatedAt`) sort by entity age. + * Field to sort by. Follows the field-addressing law (see the `@remarks` + * on {@link FindParams}): a bare name (`'level'`, `'rank'`, `'score'`, …) + * always sorts by that metadata field; the ten engine scalars sort only + * via the explicit `system.` form (e.g. `'system.createdAt'`); a + * name that resolves to neither throws {@link UnresolvableFieldError} + * naming the fix. * * Ordering contract (identical on the pure-JS engine and the native - * accelerator): entities missing the field sort LAST in both directions — - * they are never dropped from the result; ties break deterministically. + * accelerator): rows missing or `null` on this field sort LAST in BOTH + * `asc` and `desc` order and are never dropped from the result; ties + * break by `id` ascending. * - * NOTE — the field-addressing law is changing (announced 2026-08): bare - * names will mean user metadata ALWAYS, and system fields will be reached - * explicitly as `system.` (e.g. `system.createdAt`), with typed - * refusals for unresolvable names. Until that release, bare `createdAt` - * and friends keep resolving to the system fields as documented above. + * @example + * ```typescript + * await brain.find({ orderBy: 'level', order: 'desc' }) // metadata.level + * await brain.find({ orderBy: 'system.createdAt', order: 'desc' }) // engine scalar + * ``` */ orderBy?: string + /** + * Sort direction: `'asc'` (default) or `'desc'`. Per the ordering + * contract on `orderBy`, rows missing/`null` on the sorted field sort + * LAST in both directions — `order` never moves them to the front. + */ order?: 'asc' | 'desc' // Sort direction: 'asc' (default) or 'desc' // Advanced options + /** + * @deprecated Not implemented. Passing `includeRelations` throws + * {@link UnsupportedFindOptionError} — it used to be accepted and + * silently ignored, so no relationships were ever attached. Fetch + * relationships separately via `brain.related()`. + */ includeRelations?: boolean // Include entity relationships excludeVFS?: boolean // Exclude VFS entities from results (default: false - VFS included) service?: string // Multi-tenancy filter @@ -603,6 +671,11 @@ export interface FindParams { } // Performance options + /** + * @deprecated Not implemented. Passing `writeOnly` throws + * {@link UnsupportedFindOptionError} — it used to be accepted and + * silently ignored, so validation was never actually skipped. + */ writeOnly?: boolean // Skip validation for high-speed ingestion // Aggregation @@ -1352,7 +1425,10 @@ export type GroupByDimension = export interface AggregateSource { /** Filter by entity type(s) */ type?: NounType | NounType[] - /** Metadata filter (same syntax as find({ where })) */ + /** + * Metadata filter — same syntax and field-addressing law as find()'s + * `where` (see the `@remarks` on {@link FindParams}). + */ where?: Record /** Multi-tenancy service filter */ service?: string @@ -1366,7 +1442,11 @@ export interface AggregateDefinition { name: string /** Which entities contribute to this aggregate */ source: AggregateSource - /** Dimensions to group by */ + /** + * Dimensions to group by — field names follow the same field-addressing + * law as find()'s `where` / `orderBy` (see the `@remarks` on + * {@link FindParams}). + */ groupBy: GroupByDimension[] /** Named metrics to compute */ metrics: Record @@ -1425,16 +1505,25 @@ export interface AggregateGroupState { export interface AggregateQueryParams { /** Name of the aggregate to query */ name: string - /** Filter aggregate groups by their key values */ + /** + * Filter aggregate groups by their key values — same field-addressing + * law as find() (see the `@remarks` on {@link FindParams}). + */ where?: Record /** * Filter groups by their computed METRIC values (SQL HAVING). Same BFO operators as * `where`, but applied to the derived metric results plus `count`, e.g. * `{ revenue: { greaterThan: 1000 } }`. Evaluated per group (O(groups), independent of - * entity count), before sort/pagination. + * entity count), before sort/pagination. Metric names and `count` are looked up + * directly, not field-addressed; a group-KEY field used here follows the same + * field-addressing law as find() (see the `@remarks` on {@link FindParams}). */ having?: Record - /** Sort by metric name or group key field */ + /** + * Sort by metric name (a key from `metrics`, looked up directly) or by a + * group key field — a group key field follows the same field-addressing + * law as find()'s `orderBy` (see the `@remarks` on {@link FindParams}). + */ orderBy?: string /** Sort direction */ order?: 'asc' | 'desc' From 11c724bc865646f46d87a68933b7ea5a9f273f32 Mon Sep 17 00:00:00 2001 From: David Snelling Date: Mon, 3 Aug 2026 15:28:24 -0700 Subject: [PATCH 06/13] =?UTF-8?q?feat(namespace):=20the=20index=20speaks?= =?UTF-8?q?=20the=20frozen=20keys=20=E2=80=94=20record-frame=20scalars=20i?= =?UTF-8?q?ndex=20under=20literal=20'system.'=20(legacy=20'noun'=20?= =?UTF-8?q?spelling=20folds=20into=20system.type;=20plumbing=20never=20ind?= =?UTF-8?q?exed=20from=20a=20record=20frame),=20user=20fields=20stay=20bar?= =?UTF-8?q?e=20in=20every=20shape;=20filter=20+=20sorted=20paths=20route?= =?UTF-8?q?=20every=20address=20through=20parseFieldAddress;=20storage=20f?= =?UTF-8?q?allbacks=20read=20the=20addressed=20side=20of=20the=20record?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/utils/metadataIndex.ts | 144 +++++++++++++++++++++++++------------ 1 file changed, 99 insertions(+), 45 deletions(-) diff --git a/src/utils/metadataIndex.ts b/src/utils/metadataIndex.ts index cf5d521e..bfde5fe9 100644 --- a/src/utils/metadataIndex.ts +++ b/src/utils/metadataIndex.ts @@ -5,6 +5,7 @@ */ import { StorageAdapter, resolveEntityField, NounMetadata, VerbMetadata } from '../coreTypes.js' +import { SYSTEM_ENTITY_SCALARS, parseFieldAddress } from '../db/fieldAddressing.js' import { ColumnStore } from '../indexes/columnStore/ColumnStore.js' import type { MetadataIndexProvider } from '../plugin.js' import { MetadataIndexCache, MetadataIndexCacheConfig } from './metadataIndexCache.js' @@ -43,8 +44,8 @@ import { BrainyError } from '../errors/brainyError.js' * bucketed field is added (e.g. a compressed float), add it here too. */ const BUCKETED_INDEX_FIELDS: ReadonlySet = new Set([ - 'createdAt', - 'updatedAt' + 'system.createdAt', + 'system.updatedAt' ]) export interface MetadataIndexEntry { @@ -1218,12 +1219,56 @@ export class MetadataIndexManager implements MetadataIndexProvider { // the reserved entity-identity field, resolved specially by find().) const NEVER_INDEX = new Set(['vector', 'embedding', 'embeddings', 'connections', 'id']) - const extract = (obj: any, prefix = ''): void => { + // THE FROZEN INDEX KEY FORMAT (cross-engine, sealed 2026-08-03; the native + // accelerator keys identically — epoch 3 rebuilds every brain onto it): + // user fields index under BARE keys exactly as the caller wrote them; + // the ten system scalars index under literal 'system.' keys — the + // key IS the query address, so the two namespaces can never collide + // inside the index again. `origin` tracks which side of the record a key + // came from: 'record' = the entity/stored-record frame (system scalars, + // plumbing, and the metadata bag live here — the WRITE PATH's reserved- + // name remap guarantees a record-frame key matching a system name IS the + // system value); 'user' = inside the flattened metadata bag (everything + // is the user's, including natural names like `level` and `data`). + // Frame kinds: 'entity-record' = entityForIndexing shape (user fields + // nested under `metadata`; stray top-level keys are DROPPED, not guessed — + // epoch-3's rebuild-from-canonical normalizes historical shapes); + // 'flat-record' = the stored metadata-record shape (user fields FLAT + // beside the reserved ones — the write path's reserved-name remap + // guarantees a key matching a system name IS the system value, so + // non-system keys here are the user's and index bare); 'user' = inside + // the metadata bag (everything is the user's, including natural names + // like `level` and `data`). + type Frame = 'entity-record' | 'flat-record' | 'user' + const extract = (obj: any, prefix = '', frame: Frame = 'entity-record'): void => { for (const [key, value] of Object.entries(obj)) { - const fullKey = prefix ? `${prefix}.${key}` : key + let fullKey = prefix ? `${prefix}.${key}` : key - // Skip fields in never-index list (CRITICAL: prevents vector indexing bug + HNSW fields) - if (!prefix && NEVER_INDEX.has(key)) continue + if (!prefix && frame !== 'user') { + if (key === 'metadata' && typeof value === 'object' && value !== null && !Array.isArray(value)) { + extract(value, '', 'user') // the user's namespace: bare keys + continue + } + if (key === 'type' || key === 'noun') { + fullKey = 'system.type' // legacy 'noun' spelling folds into the frozen key + } else if (SYSTEM_ENTITY_SCALARS.has(key) && key !== 'id') { + fullKey = `system.${key}` + } else if ( + key === 'data' || key === '_rev' || key === 'level' || NEVER_INDEX.has(key) + ) { + continue // plumbing / identity / bulk payloads — never indexed from a record frame + } else if (frame === 'entity-record') { + continue // stray entity-frame key: dropped, not guessed + } + // flat-record fallthrough: a non-system, non-plumbing key IS a user + // field (flat beside the reserved ones) — indexes bare via fullKey. + } else if (!prefix && NEVER_INDEX.has(key)) { + // User frame: only the bulk-payload guards apply — natural names + // like `level` and `data` are real user fields here. (`id` as a + // user metadata field remains un-indexed this train — documented + // limitation; system.id resolves via the id mapper, never a column.) + continue + } // Skip purely numeric field names (array indices converted to object keys) // Legitimate field names should never be purely numeric @@ -1233,21 +1278,12 @@ export class MetadataIndexManager implements MetadataIndexProvider { // Skip fields based on user configuration if (!this.shouldIndexField(fullKey)) continue - // Special handling for metadata field at top level - // Flatten metadata fields to top-level (no prefix) for cleaner queries - // Standard fields are already at top-level, custom fields go in metadata - // By flattening here, queries can use { category: 'B' } instead of { 'metadata.category': 'B' } - if (key === 'metadata' && !prefix && typeof value === 'object' && !Array.isArray(value)) { - extract(value, '') // Flatten to top-level, no prefix - continue - } - // Skip large arrays (> 10 elements) - likely vectors or bulk data if (Array.isArray(value) && value.length > 10) continue if (value && typeof value === 'object' && !Array.isArray(value)) { - // Recurse into nested objects (but not arrays) - extract(value, fullKey) + // Recurse into nested objects (but not arrays), keeping the frame + extract(value, fullKey, frame) } else if (Array.isArray(value) && value.length <= 10) { // Small arrays: index as multi-value field (all with same field name) // Example: tags: ["javascript", "node"] → field="tags", value="javascript" + field="tags", value="node" @@ -1258,16 +1294,21 @@ export class MetadataIndexManager implements MetadataIndexProvider { } } } else { - // Primitive value: index it - // Map 'type' → 'noun' for backward compatibility - const indexField = (!prefix && key === 'type') ? 'noun' : fullKey - fields.push({ field: indexField, value }) + // Primitive value: index it under the frozen key computed above. + // (The legacy 'type'→'noun' remap is gone — 'noun' columns die at + // the epoch-3 rebuild; system.type is the one spelling.) + fields.push({ field: fullKey, value }) } } } if (data && typeof data === 'object') { - extract(data) + // Shape detection for the top frame: an object carrying a nested + // `metadata` bag is the entityForIndexing shape; anything else is the + // flat stored-record shape (user fields flat beside reserved ones). + const entityShaped = + 'metadata' in data && typeof data.metadata === 'object' && data.metadata !== null + extract(data, '', entityShaped ? 'entity-record' : 'flat-record') } // Extract words for hybrid text search @@ -1911,22 +1952,15 @@ export class MetadataIndexManager implements MetadataIndexProvider { // Skip logical operators if (rawField === 'allOf' || rawField === 'anyOf' || rawField === 'not') continue - // Metadata is FLATTENED at index time (metadata.entry.title indexes as - // entry.title), so a `metadata.`-prefixed where key is almost always - // the caller spelling the STORAGE shape rather than the index shape. - // Accept both spellings: when the key as spelled is unindexed but its - // stripped spelling is, query the stripped one. A literal nested - // custom key named `metadata` still wins when indexed as spelled - // (checked first), so that rare shape keeps working. - let field = rawField - if ( - rawField.startsWith('metadata.') && - this.columnStore && - !this.columnStore.hasField(rawField) && - this.columnStore.hasField(rawField.slice('metadata.'.length)) - ) { - field = rawField.slice('metadata.'.length) - } + // THE ONE ADDRESSING LAW (sealed 2026-08-03): every filter key routes + // through parseFieldAddress — bare and 'metadata.'-prefixed spellings + // address the user's fields (indexed under BARE keys), 'system.' + // addresses the ten engine scalars (indexed under their literal + // 'system.' keys). A malformed address (system., + // plumbing in the system spelling) throws typed BEFORE any index read — + // an accepted name either works or refuses. + const address = parseFieldAddress(rawField, 'entity') + const field = address.scope === 'system' ? `system.${address.field}` : address.field let fieldResults: string[] = [] @@ -2207,9 +2241,18 @@ export class MetadataIndexManager implements MetadataIndexProvider { order: 'asc' | 'desc' = 'asc', topK?: number ): Promise { + // THE ONE ADDRESSING LAW — the orderBy address routes through the same + // parse the filter path uses (the historical asymmetry where the filter + // path understood 'metadata.' but the sorted path never did is dead). + // Bare / 'metadata.' → the user's bare index key; 'system.' → the + // literal frozen key; malformed addresses throw typed before any read. + const orderAddress = parseFieldAddress(orderBy, 'entity') + const orderKey = + orderAddress.scope === 'system' ? `system.${orderAddress.field}` : orderAddress.field + // Column store path: O(K log S) sort via k-way merge across segments. // No per-entity storage reads, no precision loss from bucketing. - if (this.columnStore && this.columnStore.hasField(orderBy)) { + if (this.columnStore && this.columnStore.hasField(orderKey)) { // Get filtered IDs from existing roaring bitmap path const hasFilter = filter && Object.keys(filter).length > 0 const filteredIds = hasFilter ? await this.getIdsForFilter(filter) : [] @@ -2229,12 +2272,12 @@ export class MetadataIndexManager implements MetadataIndexProvider { // log K) heap, not a full sort materialization. const k = topK !== undefined ? Math.min(topK, filteredIds.length) : filteredIds.length sortedIntIds = await this.columnStore.filteredSortTopK( - filterBitmap, orderBy, order, k + filterBitmap, orderKey, order, k ) } else { // Unfiltered sort — column store handles the full entity set efficiently sortedIntIds = await this.columnStore.sortTopK( - orderBy, order, topK !== undefined ? Math.min(topK, this.idMapper.size) : this.idMapper.size + orderKey, order, topK !== undefined ? Math.min(topK, this.idMapper.size) : this.idMapper.size ) } @@ -2255,7 +2298,7 @@ export class MetadataIndexManager implements MetadataIndexProvider { const idValuePairs: Array<{ id: string, value: any }> = [] for (const id of filteredIds) { - const value = await this.getFieldValueForEntity(id, orderBy) + const value = await this.getFieldValueForEntity(id, orderKey) idValuePairs.push({ id, value }) } @@ -2320,10 +2363,17 @@ export class MetadataIndexManager implements MetadataIndexProvider { * @public (called from brainy.ts for sorted queries) */ async getFieldValueForEntity(entityId: string, field: string): Promise { - // Path 1: Bucketed fields need the actual value from storage. + // `field` arrives as a FROZEN INDEX KEY (bare = user metadata; + // 'system.' = engine scalar). Storage fallbacks read the matching + // side of the record — a system key reads the record scalar, a bare key + // reads the user's metadata bag; the two can never shadow each other. + const systemInner = field.startsWith('system.') ? field.slice('system.'.length) : null + + // Path 1: Bucketed fields need the actual (un-bucketed) value from storage. if (BUCKETED_INDEX_FIELDS.has(field)) { const noun = await this.storage.getNoun(entityId) - return noun ? resolveEntityField(noun, field) : undefined + if (!noun) return undefined + return (noun as unknown as Record)[systemInner as string] } // Path 3 precondition: entity must be in the id mapper for bitmap lookup. @@ -2340,7 +2390,11 @@ export class MetadataIndexManager implements MetadataIndexProvider { // yet indexed. resolveEntityField handles the shape contract. if (!sparseIndex) { const noun = await this.storage.getNoun(entityId) - return noun ? resolveEntityField(noun, field) : undefined + if (!noun) return undefined + if (systemInner !== null) { + return (noun as unknown as Record)[systemInner] + } + return (noun as { metadata?: Record }).metadata?.[field] } // Path 3: Search sparse index chunks for this entity's value. From 7a28a94639e4ce9777c3e4a11c032f665ab2ccb0 Mon Sep 17 00:00:00 2001 From: David Snelling Date: Mon, 3 Aug 2026 15:33:01 -0700 Subject: [PATCH 07/13] =?UTF-8?q?feat(namespace):=20find's=20own=20filter?= =?UTF-8?q?=20builders=20speak=20the=20frozen=20keys=20=E2=80=94=20params.?= =?UTF-8?q?type/subtype/service=20become=20system.*=20index=20keys=20at=20?= =?UTF-8?q?every=20construction=20site=20(three=20pipelines=20+=20the=20ca?= =?UTF-8?q?nonical=20buildMetadataFilter);=20the=20where.type=E2=86=92noun?= =?UTF-8?q?=20alias=20is=20dead=20(bare=20'type'=20belongs=20to=20the=20us?= =?UTF-8?q?er=20now)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/brainy.ts | 64 ++++++++++++++++++++++----------------------------- 1 file changed, 28 insertions(+), 36 deletions(-) diff --git a/src/brainy.ts b/src/brainy.ts index d8eca08b..a8df56bd 100644 --- a/src/brainy.ts +++ b/src/brainy.ts @@ -6148,20 +6148,18 @@ export class Brainy implements BrainyInterface { // Build filter for metadata index let filter: any = {} if (params.where) { + // Where keys pass through UNTOUCHED — the addressing law parses + // them at the index boundary. The old where.type→noun alias is + // dead: bare 'type' is the user's own field now. Object.assign(filter, params.where) - // Alias: where.type → where.noun (storage field name for entity type) - if ('type' in filter && !('noun' in filter)) { - filter.noun = filter.type - delete filter.type - } } - if (params.service) filter.service = params.service + if (params.service) filter['system.service'] = params.service // Subtype (top-level standard field — fast path, not metadata fallback). // Must be assigned BEFORE the type-array expansion below so the spread // into each anyOf branch carries it through. if (params.subtype !== undefined) { - filter.subtype = Array.isArray(params.subtype) + filter['system.subtype'] = Array.isArray(params.subtype) ? { oneOf: params.subtype } : params.subtype } @@ -6169,11 +6167,11 @@ export class Brainy implements BrainyInterface { if (params.type) { const types = Array.isArray(params.type) ? params.type : [params.type] if (types.length === 1) { - filter.noun = types[0] + filter['system.type'] = types[0] } else { filter = { anyOf: types.map(type => ({ - noun: type, + 'system.type': type, ...filter })) } @@ -11388,27 +11386,26 @@ export class Brainy implements BrainyInterface { if (params.where || params.subtype || params.service) { let filter: any = {} if (params.where) { + // Where keys pass through UNTOUCHED — the one addressing law + // parses them at the index boundary (bare = user metadata, + // system.* = engine scalars). The old where.type→noun alias is + // dead: a bare 'type' is the user's own field now. Object.assign(filter, params.where) - // Alias: where.type → where.noun (storage field name for entity type) - if ('type' in filter && !('noun' in filter)) { - filter.noun = filter.type - delete filter.type - } } - if (params.service) filter.service = params.service + if (params.service) filter['system.service'] = params.service if (params.subtype !== undefined) { - filter.subtype = Array.isArray(params.subtype) + filter['system.subtype'] = Array.isArray(params.subtype) ? { oneOf: params.subtype } : params.subtype } if (params.type) { const types = Array.isArray(params.type) ? params.type : [params.type] if (types.length === 1) { - filter.noun = types[0] + filter['system.type'] = types[0] } else { const baseFilter = { ...filter } filter = { - anyOf: types.map(type => ({ noun: type, ...baseFilter })) + anyOf: types.map(type => ({ 'system.type': type, ...baseFilter })) } } } @@ -11458,27 +11455,24 @@ export class Brainy implements BrainyInterface { // Use MetadataIndexManager for efficient filtered streaming let filterObj: any = {} if (filter.where) { + // Where keys pass through — the addressing law parses them at + // the index boundary; the type→noun alias is dead. Object.assign(filterObj, filter.where) - // Alias: where.type → where.noun (storage field name for entity type) - if ('type' in filterObj && !('noun' in filterObj)) { - filterObj.noun = filterObj.type - delete filterObj.type - } } - if (filter.service) filterObj.service = filter.service + if (filter.service) filterObj['system.service'] = filter.service if (filter.subtype !== undefined) { - filterObj.subtype = Array.isArray(filter.subtype) + filterObj['system.subtype'] = Array.isArray(filter.subtype) ? { oneOf: filter.subtype } : filter.subtype } if (filter.type) { const types = Array.isArray(filter.type) ? filter.type : [filter.type] if (types.length === 1) { - filterObj.noun = types[0] + filterObj['system.type'] = types[0] } else { const baseFilterObj = { ...filterObj } filterObj = { - anyOf: types.map(type => ({ noun: type, ...baseFilterObj })) + anyOf: types.map(type => ({ 'system.type': type, ...baseFilterObj })) } } } @@ -13605,14 +13599,12 @@ export class Brainy implements BrainyInterface { } let filter: any = {} if (params.where) { + // Where keys pass through UNTOUCHED — the one addressing law parses + // them at the index boundary (bare = user metadata, system.* = engine + // scalars, typed refusal otherwise). The old type→noun alias is dead. Object.assign(filter, params.where) - // Alias: where.type → where.noun (storage field name for entity type) - if ('type' in filter && !('noun' in filter)) { - filter.noun = filter.type - delete filter.type - } } - if (params.service) filter.service = params.service + if (params.service) filter['system.service'] = params.service if (params.excludeVFS === true) { filter.vfsType = { exists: false } filter.isVFSEntity = { ne: true } @@ -13620,14 +13612,14 @@ export class Brainy implements BrainyInterface { // Subtype (top-level standard field — fast path). Assigned BEFORE the type-array // expansion below so the spread into each anyOf branch carries it through. if (params.subtype !== undefined) { - filter.subtype = Array.isArray(params.subtype) ? { oneOf: params.subtype } : params.subtype + filter['system.subtype'] = Array.isArray(params.subtype) ? { oneOf: params.subtype } : params.subtype } if (params.type) { const types = Array.isArray(params.type) ? params.type : [params.type] if (types.length === 1) { - filter.noun = types[0] + filter['system.type'] = types[0] } else { - filter = { anyOf: types.map((type) => ({ noun: type, ...filter })) } + filter = { anyOf: types.map((type) => ({ 'system.type': type, ...filter })) } } } return filter From 4679c89458aa5faabcea931862b9052030f35120 Mon Sep 17 00:00:00 2001 From: David Snelling Date: Mon, 3 Aug 2026 15:37:04 -0700 Subject: [PATCH 08/13] =?UTF-8?q?fix(namespace):=20noun-record=20updates?= =?UTF-8?q?=20preserve=20legacy=20inline=20HNSW=20adjacency=20=E2=80=94=20?= =?UTF-8?q?the=20placeholder-adjacency=20write=20stamped=20out=20pre-codec?= =?UTF-8?q?=20records'=20stored=20connections=20(crash-window=20unreachabi?= =?UTF-8?q?lity);=20codec-era=20records=20were=20never=20at=20risk=20(empt?= =?UTF-8?q?y=20field=20is=20the=20blob=20marker);=20pin=20covers=20the=20l?= =?UTF-8?q?egacy=20shape?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../operations/StorageOperations.ts | 23 ++++++++- tests/integration/level-field-shadow.test.ts | 47 +++++++++++++++++++ 2 files changed, 68 insertions(+), 2 deletions(-) diff --git a/src/transaction/operations/StorageOperations.ts b/src/transaction/operations/StorageOperations.ts index 316f1ac0..9858219b 100644 --- a/src/transaction/operations/StorageOperations.ts +++ b/src/transaction/operations/StorageOperations.ts @@ -77,8 +77,27 @@ export class SaveNounOperation implements Operation { ? null : await this.storage.getNoun(this.noun.id) - // Save new noun - await this.storage.saveNoun(this.noun) + // PRESERVE stored graph state on updates. Callers stage this op with + // placeholder adjacency ({connections: empty, level: 0}) because the + // vector index owns those values and persists them at flush. Codec-era + // records (2.4.0+) carry an empty connections field by design (adjacency + // lives in a separate compressed blob — the placeholder is harmless), but + // LEGACY pre-codec records store adjacency INLINE: writing the + // placeholder over one stamped out its stored connections, leaving a + // crash window (until the next flush) where a reload found the node + // unreachable. Stale adjacency in that window is tolerable — HNSW + // self-corrects at the reindex flush; EMPTY adjacency is silent recall + // loss. The read above is already paid for rollback; preservation is free. + const toSave: HNSWNoun = + previousNoun && this.noun.connections.size === 0 + ? { + ...this.noun, + connections: previousNoun.connections || this.noun.connections, + level: previousNoun.level ?? this.noun.level + } + : this.noun + + await this.storage.saveNoun(toSave) // Return rollback action return async () => { diff --git a/tests/integration/level-field-shadow.test.ts b/tests/integration/level-field-shadow.test.ts index d50593ff..ab5ffb9a 100644 --- a/tests/integration/level-field-shadow.test.ts +++ b/tests/integration/level-field-shadow.test.ts @@ -145,3 +145,50 @@ describe('level field shadow — user metadata named level is a real field', () expect(EXPECTED_INDEX_EPOCH).toBe(2) }) }) + +describe('noun-record writes never stamp over stored graph state', () => { + let brain: Brainy + + beforeEach(async () => { + brain = new Brainy({ + requireSubtype: false, + storage: { type: 'memory' as const }, + embeddingFunction: stubEmbedding + }) + await brain.init() + }) + + afterEach(async () => { + await brain.close() + }) + + it('a data-changing update preserves LEGACY inline connections in the record', async () => { + // Codec-era records carry an EMPTY connections field by design (the + // adjacency lives in a separate compressed blob) — the clobber window + // exists only for legacy pre-codec records whose adjacency is inline. + // Simulate one: write the record with inline connections directly. + const id = await brain.add({ + data: 'legacy-shaped node', + type: NounType.Concept, + metadata: { n: 1 } + }) + const storage = (brain as any).storage + const rec = await storage.getNoun(id) + const legacy = { + ...rec, + connections: new Map([[0, new Set(['00000000-0000-4000-8000-00000000aaaa'])]]), + level: 1 + } + await storage.saveNoun(legacy) + const before = await storage.getNoun(id) + expect(before.connections.size).toBeGreaterThan(0) + + // A data-changing update stages SaveNounOperation with placeholder + // adjacency — the legacy inline connections must survive the write. + await brain.update({ id, data: 'completely re-embedded text' }) + + const after = await storage.getNoun(id) + expect(after.connections.size).toBeGreaterThan(0) + expect(after.level).toBe(1) + }) +}) From c2fb28a2f7c261dd055677b6042803e2afd8de3d Mon Sep 17 00:00:00 2001 From: David Snelling Date: Mon, 3 Aug 2026 15:51:14 -0700 Subject: [PATCH 09/13] =?UTF-8?q?feat(namespace):=20egress=20guard=20+=20v?= =?UTF-8?q?alidation=20speak=20the=20law=20=E2=80=94=20whereMatcher's=20re?= =?UTF-8?q?solver=20reads=20system.*=20from=20the=20record=20and=20bare=20?= =?UTF-8?q?names=20from=20the=20metadata=20bag=20only=20(the=20bare-system?= =?UTF-8?q?=20switch=20is=20dead);=20validateFindParams=20refuses=20cursor?= =?UTF-8?q?/includeRelations/writeOnly=20typed=20(accepted-and-ignored=20d?= =?UTF-8?q?ies=20as=20a=20class),=20validates=20order,=20and=20parses=20ev?= =?UTF-8?q?ery=20orderBy=20address?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/db/fieldAddressing.ts | 39 ++++++++++++++++++++ src/db/whereMatcher.ts | 69 +++++++++++++++++++----------------- src/utils/paramValidation.ts | 29 +++++++++++++-- 3 files changed, 101 insertions(+), 36 deletions(-) diff --git a/src/db/fieldAddressing.ts b/src/db/fieldAddressing.ts index 0ee09a05..cd63871c 100644 --- a/src/db/fieldAddressing.ts +++ b/src/db/fieldAddressing.ts @@ -244,3 +244,42 @@ export class InvalidFieldAddressError extends Error { this.kind = kind } } + +/** + * @description Refusal for a syntactically valid address that resolves to + * NOTHING — a bare name no user field carries. Carries the did-you-mean + * (both candidate spellings when the name collides with a system scalar) so + * the fix ships inside the error. Thrown by the query layer with index + * knowledge, never by the pure parser. + */ +export class UnresolvableFieldError extends Error { + public readonly raw: string + public readonly kind: FieldAddressKind + + constructor(raw: string, kind: FieldAddressKind) { + super(buildUnresolvableMessage(raw, kind)) + this.name = 'UnresolvableFieldError' + this.raw = raw + this.kind = kind + } +} + +/** + * @description Refusal for a find() option that is accepted by the type + * surface but NOT implemented — an accepted option must work or refuse; + * accepted-and-ignored died as a class (sealed 2026-08-03). Names the + * option and the honest state so nobody discovers a no-op by measurement. + */ +export class UnsupportedFindOptionError extends Error { + public readonly option: string + + constructor(option: string) { + super( + `find() option '${option}' is not implemented — it used to be silently ` + + `ignored, which read as working. Remove it from the call (or track the ` + + `feature request); it will be honored or refused, never swallowed.` + ) + this.name = 'UnsupportedFindOptionError' + this.option = option + } +} diff --git a/src/db/whereMatcher.ts b/src/db/whereMatcher.ts index c5469209..8dab02fd 100644 --- a/src/db/whereMatcher.ts +++ b/src/db/whereMatcher.ts @@ -61,41 +61,44 @@ export class UnsupportedWhereOperatorError extends Error { * @returns The field's value, or `undefined` when absent. */ export function resolveEntityField(entity: Entity, field: string): unknown { - switch (field) { - case 'noun': - case 'type': - return entity.type - case 'subtype': - return entity.subtype - case 'id': - return entity.id - case 'createdAt': - return entity.createdAt - case 'updatedAt': - return entity.updatedAt - case 'service': - return entity.service - case 'createdBy': - return entity.createdBy - case 'confidence': - return entity.confidence - case 'weight': - return entity.weight - case '_rev': - return entity._rev - case 'data': - return entity.data + // THE ONE ADDRESSING LAW (sealed 2026-08-03): `system.` reads the + // entity scalar; bare and `metadata.`-prefixed names read the user's + // metadata bag (dotted paths traverse INSIDE the bag). The old bare-name + // switch over system fields is dead — bare `createdAt` is the user's own + // field now; the engine scalar is `system.createdAt`. Plumbing (vector, + // connections, level, data, _rev) is invisible: no spelling reaches it. + if (field.startsWith('system.')) { + switch (field.slice('system.'.length)) { + case 'type': + return entity.type + case 'subtype': + return entity.subtype + case 'id': + return entity.id + case 'createdAt': + return entity.createdAt + case 'updatedAt': + return entity.updatedAt + case 'service': + return entity.service + case 'createdBy': + return entity.createdBy + case 'confidence': + return entity.confidence + case 'weight': + return entity.weight + case 'visibility': + return (entity as unknown as Record).visibility + } + // Out-of-map system spelling: parse refuses these upstream with a typed + // error; reaching here (internal callers only) reads as absent. + return undefined } - if (field.includes('.')) { - // Dotted path: resolve against the whole entity first (`metadata.x`), - // then against the metadata bag (`address.city` on nested metadata). - const fromEntity = resolvePath(entity as unknown as Record, field) - if (fromEntity !== undefined) return fromEntity - return resolvePath((entity.metadata ?? {}) as Record, field) - } - - return ((entity.metadata ?? {}) as Record)[field] + const path = field.startsWith('metadata.') ? field.slice('metadata.'.length) : field + const bag = (entity.metadata ?? {}) as Record + if (!path.includes('.')) return bag[path] + return resolvePath(bag, path) } /** Walk a dotted path through nested plain objects. */ diff --git a/src/utils/paramValidation.ts b/src/utils/paramValidation.ts index ca439524..fd018a04 100644 --- a/src/utils/paramValidation.ts +++ b/src/utils/paramValidation.ts @@ -17,6 +17,7 @@ import { findCallerLocation } from './callerLocation.js' // fallback branches that no supported runtime can reach. import * as os from 'node:os' import * as fs from 'node:fs' +import { parseFieldAddress, UnsupportedFindOptionError } from '../db/fieldAddressing.js' const getSystemMemory = (): number => { if (os) { @@ -466,9 +467,31 @@ export function validateFindParams(params: FindParams): void { throw new Error('cannot specify both query and vector - they are mutually exclusive') } - // Universal truth: can't use both cursor and offset pagination - if (params.cursor !== undefined && params.offset !== undefined) { - throw new Error('cannot use both cursor and offset pagination simultaneously') + // ACCEPTED-AND-IGNORED DIED AS A CLASS (sealed 2026-08-03): options the + // engine does not implement REFUSE with a typed error instead of silently + // doing nothing — a production consumer discovered a no-op by measurement + // once; never again. + if (params.cursor !== undefined) { + throw new UnsupportedFindOptionError('cursor') + } + if ((params as Record).includeRelations !== undefined) { + throw new UnsupportedFindOptionError('includeRelations') + } + if ((params as Record).writeOnly !== undefined) { + throw new UnsupportedFindOptionError('writeOnly') + } + + // THE ONE ADDRESSING LAW: the orderBy address must PARSE (bare/metadata. = + // user field, system. = the ruled map, anything else refuses typed + // with the valid map in the message) and order must be a real direction. + if (params.orderBy !== undefined) { + if (typeof params.orderBy !== 'string') { + throw new Error('orderBy must be a string field address') + } + parseFieldAddress(params.orderBy, 'entity') // throws InvalidFieldAddressError on a bad address + } + if (params.order !== undefined && params.order !== 'asc' && params.order !== 'desc') { + throw new Error(`order must be 'asc' or 'desc', got '${String(params.order)}'`) } // Auto-limit query length based on memory From 7492b6cb59362a88e3af8f739001a4e0926860d7 Mon Sep 17 00:00:00 2001 From: David Snelling Date: Mon, 3 Aug 2026 15:53:13 -0700 Subject: [PATCH 10/13] =?UTF-8?q?feat(namespace):=20aggregation=20reads=20?= =?UTF-8?q?under=20the=20law=20+=20epoch=203=20(the=20key-split=20rebuild)?= =?UTF-8?q?=20+=20THE=20ARMING=20COMMIT=20=E2=80=94=20the=20capability=20c?= =?UTF-8?q?onstant,=20the=20law=20module,=20and=20the=20typed=20refusals?= =?UTF-8?q?=20export=20from=20the=20package=20root;=20both=20engines'=20co?= =?UTF-8?q?nformance=20suites=20light=20on=20this=20signal?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/aggregation/AggregationIndex.ts | 31 ++++++++++++++----- src/brainy.ts | 11 +++++-- src/db/fieldAddressing.ts | 8 +++++ src/index.ts | 19 ++++++++++++ src/storage/brainFormat.ts | 15 +++++---- tests/integration/level-field-shadow.test.ts | 4 +-- tests/unit/brainy/migration-deference.test.ts | 7 +++-- 7 files changed, 73 insertions(+), 22 deletions(-) diff --git a/src/aggregation/AggregationIndex.ts b/src/aggregation/AggregationIndex.ts index f9382218..407b1fe0 100644 --- a/src/aggregation/AggregationIndex.ts +++ b/src/aggregation/AggregationIndex.ts @@ -14,7 +14,22 @@ */ import type { StorageAdapter, HNSWNounWithMetadata } from '../coreTypes.js' -import { resolveEntityField } from '../coreTypes.js' +import { parseFieldAddress, readEntityFieldAddress } from '../db/fieldAddressing.js' +import type { HNSWNounWithMetadata as AddressedEntity } from '../coreTypes.js' + +/** + * Read a user-supplied field name under the one addressing law (sealed + * 2026-08-03): bare / `metadata.` = the user's metadata field, `system.` = + * the ruled engine scalar, malformed = typed refusal. The aggregation engine + * NEVER resolves names any other way — the pre-law resolver made bare + * `subtype`/`confidence` read engine scalars, silently shadowing user fields. + */ +function readAddressed(e: unknown, name: string): unknown { + return readEntityFieldAddress( + e as AddressedEntity, + parseFieldAddress(name, 'entity') + ) +} import type { AggregateDefinition, AggregateGroupState, @@ -97,7 +112,7 @@ function matchesSource(entity: Record, source: AggregateDefinit const e = entity as unknown as HNSWNounWithMetadata const resolved: Record = {} for (const key of Object.keys(source.where)) { - resolved[key] = resolveEntityField(e, key) + resolved[key] = readAddressed(e, key) } if (!matchesMetadataFilter(resolved, source.where)) return false } @@ -129,11 +144,11 @@ function computeGroupKeys( for (const dim of groupBy) { if (typeof dim === 'string') { - const val = resolveEntityField(e, dim) + const val = readAddressed(e, dim) const v = val !== undefined && val !== null ? String(val) : '__null__' for (const k of keys) k[dim] = v } else if ('unnest' in dim) { - const val = resolveEntityField(e, dim.field) + const val = readAddressed(e, dim.field) const raw = Array.isArray(val) ? val : val !== undefined && val !== null ? [val] : [] // Distinct elements: an entity with duplicate tags counts once per distinct tag. const elems = Array.from(new Set(raw.map(x => String(x)))) @@ -145,7 +160,7 @@ function computeGroupKeys( keys = next } else { // Time-windowed field - const val = resolveEntityField(e, dim.field) + const val = readAddressed(e, dim.field) const v = typeof val === 'number' ? bucketTimestamp(val, dim.window) : '__null__' for (const k of keys) k[dim.field] = v } @@ -174,7 +189,7 @@ function computeGroupKey( * in metadata are both handled in one place. */ function getNumericField(entity: Record, field: string): number | undefined { - const val = resolveEntityField(entity as unknown as HNSWNounWithMetadata, field) + const val = readAddressed(entity as unknown as HNSWNounWithMetadata, field) if (typeof val === 'number' && !isNaN(val)) return val if (typeof val === 'string') { const num = parseFloat(val) @@ -990,7 +1005,7 @@ export class AggregationIndex { // distinctCount tracks distinct values of ANY type (strings, numbers, booleans), // keyed by their string form — NOT numeric-coerced, since its primary use is // categorical (distinct categories / users / tags), not numeric columns. - const raw = resolveEntityField(entity as unknown as HNSWNounWithMetadata, metricDef.field!) + const raw = readAddressed(entity as unknown as HNSWNounWithMetadata, metricDef.field!) if (raw !== undefined && raw !== null) { if (!state.valueCounts) state.valueCounts = {} const key = String(raw) @@ -1034,7 +1049,7 @@ export class AggregationIndex { state.count = Math.max(0, state.count - 1) state.sum = Math.max(0, state.sum - 1) } else if (metricDef.op === 'distinctCount') { - const raw = resolveEntityField(entity as unknown as HNSWNounWithMetadata, metricDef.field!) + const raw = readAddressed(entity as unknown as HNSWNounWithMetadata, metricDef.field!) if (raw !== undefined && raw !== null && state.valueCounts) { const key = String(raw) const c = state.valueCounts[key] diff --git a/src/brainy.ts b/src/brainy.ts index a8df56bd..3dd8ef93 100644 --- a/src/brainy.ts +++ b/src/brainy.ts @@ -5619,7 +5619,7 @@ export class Brainy implements BrainyInterface { this._aggregationIndex!.defineAggregate({ name: aggregateName, source: {}, - groupBy: perType ? [name, 'noun'] : [name], + groupBy: perType ? [name, 'system.type'] : [name], metrics: { count: { op: 'count' } } }) } @@ -5630,7 +5630,12 @@ export class Brainy implements BrainyInterface { * and `counts.byField()` agree on the convention. */ private fieldCountsAggregateName(name: string): string { - return `__fieldCounts__${name}` + // v2 suffix: the per-type dimension moved from the legacy 'noun' alias to + // 'system.type' under the addressing law — a NEW name makes the ensure + // block re-define and BACKFILL from canonical instead of silently serving + // the old-dim definition (whose 'noun' key now reads user metadata and + // would drift). The v1 rows are derived state, superseded not lost. + return `__fieldCounts_v2__${name}` } /** @@ -11852,7 +11857,7 @@ export class Brainy implements BrainyInterface { // don't have the tracked field at all (e.g. the VFS root) bucket under // '__null__' and would otherwise pollute the count map. if (value === undefined || value === null || value === '__null__') continue - if (options?.type !== undefined && row.groupKey?.['noun'] !== options.type) continue + if (options?.type !== undefined && row.groupKey?.['system.type'] !== options.type) continue const key = String(value) result[key] = (result[key] || 0) + (typeof row.metrics?.count === 'number' ? row.metrics.count : row.count) } diff --git a/src/db/fieldAddressing.ts b/src/db/fieldAddressing.ts index cd63871c..056ba3f2 100644 --- a/src/db/fieldAddressing.ts +++ b/src/db/fieldAddressing.ts @@ -283,3 +283,11 @@ export class UnsupportedFindOptionError extends Error { this.option = option } } + +/** + * @description The capability signal both engines' conformance suites arm on + * (never a version guess): its presence at the package root means the one + * field-addressing law is LIVE on every query surface — bare = user metadata, + * `system.*` = the ruled scalars, plumbing invisible, refusals typed. + */ +export const FIELD_ADDRESSING_CAPABILITY = 'field-addressing/v1' diff --git a/src/index.ts b/src/index.ts index ae8eef5c..3876a903 100644 --- a/src/index.ts +++ b/src/index.ts @@ -106,6 +106,25 @@ export type { // Export Aggregation Engine export { AggregationIndex, AggregateMaterializer, bucketTimestamp, parseBucketRange } from './aggregation/index.js' +// THE ONE FIELD-ADDRESSING LAW (sealed 2026-08-03) — the arming surface both +// engines' conformance suites detect: bare names = user metadata, system.* = +// the ten ruled scalars, plumbing invisible, refusals typed with the fix in +// the message. See docs/concepts/field-addressing.md. +export { + FIELD_ADDRESSING_CAPABILITY, + SYSTEM_ENTITY_SCALARS, + SYSTEM_RELATION_SCALARS, + PLUMBING_FIELDS, + parseFieldAddress, + readEntityFieldAddress, + readRelationFieldAddress, + buildUnresolvableMessage, + InvalidFieldAddressError, + UnresolvableFieldError, + UnsupportedFindOptionError +} from './db/fieldAddressing.js' +export type { FieldAddress, FieldAddressKind } from './db/fieldAddressing.js' + // Export Neural Import (AI data understanding) export { NeuralImport } from './neural/neuralImport.js' export type { diff --git a/src/storage/brainFormat.ts b/src/storage/brainFormat.ts index a1241fe0..6ef913d4 100644 --- a/src/storage/brainFormat.ts +++ b/src/storage/brainFormat.ts @@ -69,12 +69,15 @@ export const BRAIN_FORMAT_PATH = '_system/brain-format.json' * (the 8.0 GA baseline). An on-disk `indexEpoch` that differs from this — or an * absent marker — triggers a full derived-index rebuild on open. */ -// Epoch 2 (2026-08-03, paired with the native accelerator's same-day release): -// user metadata fields named `level` become indexable on both engines — the -// derived posting set changed, so every pre-fix brain must rebuild its -// metadata index from canonical at first open (poisoned multi-valued `level` -// columns heal through this rebuild; no bespoke heal path). -export const EXPECTED_INDEX_EPOCH = 2 +// Epoch 3 (2026-08-03, the namespace-law pair): the index key format split +// the two namespaces — user fields keep bare flattened keys, the ten system +// scalars moved to literal 'system.' keys (the legacy 'noun' column +// spelling died with them). Every brain rebuilds its derived indexes from +// canonical at first open onto the frozen keys. +// Epoch 2 (2026-08-03, same day, the interim pair): user metadata fields +// named `level` became indexable on both engines; poisoned multi-valued +// `level` columns healed through the rebuild. +export const EXPECTED_INDEX_EPOCH = 3 /** * @description The data-layer format string this build writes and runs as. diff --git a/tests/integration/level-field-shadow.test.ts b/tests/integration/level-field-shadow.test.ts index ab5ffb9a..cfe34c13 100644 --- a/tests/integration/level-field-shadow.test.ts +++ b/tests/integration/level-field-shadow.test.ts @@ -141,8 +141,8 @@ describe('level field shadow — user metadata named level is a real field', () expect(Array.isArray(after?.vector) && after!.vector!.length).toBe(384) }) - it('this build runs index epoch 2 (the paired level-indexability rebuild)', () => { - expect(EXPECTED_INDEX_EPOCH).toBe(2) + it('this build runs index epoch 3 (the namespace-law key split rebuild)', () => { + expect(EXPECTED_INDEX_EPOCH).toBe(3) }) }) diff --git a/tests/unit/brainy/migration-deference.test.ts b/tests/unit/brainy/migration-deference.test.ts index f03bba9c..5968c620 100644 --- a/tests/unit/brainy/migration-deference.test.ts +++ b/tests/unit/brainy/migration-deference.test.ts @@ -245,9 +245,10 @@ describe('rc.8 no-freeze migration deference (isMigrating / stampBrainFormat / b it('the brain-format marker module exports the compiled epoch + data-format constants', () => { // cor imports these from '@soulcraft/brainy/brain-format' (Hook 3) so both // sides share ONE source of truth — no duplicated constant to drift. - // Epoch 2: user metadata named `level` became indexable (the reserved-name - // shadow fix, 2026-08-03) — pre-fix brains rebuild derived indexes at open. - expect(EXPECTED_INDEX_EPOCH).toBe(2) + // Epoch 3: the namespace-law key split (bare user keys · literal + // 'system.' scalars, 2026-08-03) — every brain rebuilds onto the + // frozen keys at first open. (Epoch 2 same day: `level` indexability.) + expect(EXPECTED_INDEX_EPOCH).toBe(3) expect(CURRENT_DATA_FORMAT).toBe('8.0') }) }) From 8e962dabdaec6dabef88ebfce5d47afee463588e Mon Sep 17 00:00:00 2001 From: David Snelling Date: Mon, 3 Aug 2026 16:01:02 -0700 Subject: [PATCH 11/13] =?UTF-8?q?feat(namespace):=20conformance=20green=20?= =?UTF-8?q?19/19=20=E2=80=94=20data-aware=20did-you-mean=20on=20unindexed?= =?UTF-8?q?=20bare=20addresses,=20ordering=20contract=20on=20the=20column?= =?UTF-8?q?=20top-K=20path=20(never=20drop,=20nulls=20last,=20ties=20by=20?= =?UTF-8?q?id),=20shape-complete=20addressed=20reads=20(entity=20views=20A?= =?UTF-8?q?ND=20raw=20storage=20shapes,=20shadow-proof=20both=20scopes),?= =?UTF-8?q?=20per-key=20source=20matching=20for=20dotted=20addresses;=20re?= =?UTF-8?q?fusal=20classes=20unified=20under=20UnresolvableFieldError?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/aggregation/AggregationIndex.ts | 12 ++- src/db/fieldAddressing.ts | 81 +++++++++++++------- src/utils/metadataIndex.ts | 98 +++++++++++++++++-------- tests/conformance/namespace-law.test.ts | 4 +- 4 files changed, 134 insertions(+), 61 deletions(-) diff --git a/src/aggregation/AggregationIndex.ts b/src/aggregation/AggregationIndex.ts index 407b1fe0..ca44ac8b 100644 --- a/src/aggregation/AggregationIndex.ts +++ b/src/aggregation/AggregationIndex.ts @@ -110,11 +110,15 @@ function matchesSource(entity: Record, source: AggregateDefinit // live in the custom bag, so those filters could never match anything. if (source.where && Object.keys(source.where).length > 0) { const e = entity as unknown as HNSWNounWithMetadata - const resolved: Record = {} - for (const key of Object.keys(source.where)) { - resolved[key] = readAddressed(e, key) + for (const [key, condition] of Object.entries(source.where)) { + // Evaluate ONE field at a time under a neutral key: the address may be + // dotted ('system.subtype'), and the filter evaluator would otherwise + // walk dots as a nested path instead of treating the key as an address. + const value = readAddressed(e, key) + if (!matchesMetadataFilter({ v: value }, { v: condition } as Record)) { + return false + } } - if (!matchesMetadataFilter(resolved, source.where)) return false } return true diff --git a/src/db/fieldAddressing.ts b/src/db/fieldAddressing.ts index 056ba3f2..98c81e5f 100644 --- a/src/db/fieldAddressing.ts +++ b/src/db/fieldAddressing.ts @@ -167,10 +167,39 @@ export function readEntityFieldAddress( entity: HNSWNounWithMetadata, address: FieldAddress ): unknown { + const rec = entity as unknown as Record + const bag = + rec.metadata && typeof rec.metadata === 'object' + ? (rec.metadata as Record) + : null + if (address.scope === 'system') { - return (entity as unknown as Record)[address.field] + // Entity views carry system scalars top-level; raw storage shapes carry + // them inside the stored metadata record (where `type` is spelled `noun`). + // Read top-level first, then the record — never the user's namespace. + const top = rec[address.field] + if (top !== undefined) return top + if (bag) { + if (address.field === 'type') return bag.type ?? bag.noun + return bag[address.field] + } + return undefined } - return entity.metadata?.[address.field] + + // User scope. The write-path remap guarantees the user can never OWN a + // field named like a system scalar (those lift top-level at write), so a + // bare system name reads as ABSENT — reading the stored record's reserved + // key here would re-create the shadow this module exists to kill. Same for + // plumbing and the legacy 'noun' spelling. + if ( + SYSTEM_ENTITY_SCALARS.has(address.field) || + PLUMBING_FIELDS.has(address.field) || + address.field === 'noun' + ) { + return undefined + } + if (bag) return bag[address.field] + return rec[address.field] } /** @@ -222,29 +251,6 @@ export function buildUnresolvableMessage( ) } -/** - * @description Refusal for a malformed or out-of-map field ADDRESS — - * `system.` (including all plumbing), an empty - * name, or a bare `metadata.` prefix. The message carries the full valid - * system map so the fix never needs a docs lookup. - */ -export class InvalidFieldAddressError extends Error { - public readonly raw: string - public readonly kind: FieldAddressKind - - constructor(raw: string, kind: FieldAddressKind, systemMap: ReadonlySet) { - const valid = [...systemMap].map((f) => `system.${f}`).join(', ') - super( - `'${raw}' is not an addressable ${kind} field. Bare names address your own ` + - `metadata fields; engine fields are exactly: ${valid}. Engine plumbing ` + - `(vector, connections, level, data, _rev) is not part of the query surface.` - ) - this.name = 'InvalidFieldAddressError' - this.raw = raw - this.kind = kind - } -} - /** * @description Refusal for a syntactically valid address that resolves to * NOTHING — a bare name no user field carries. Carries the did-you-mean @@ -256,14 +262,35 @@ export class UnresolvableFieldError extends Error { public readonly raw: string public readonly kind: FieldAddressKind - constructor(raw: string, kind: FieldAddressKind) { - super(buildUnresolvableMessage(raw, kind)) + constructor(raw: string, kind: FieldAddressKind, messageOverride?: string) { + super(messageOverride ?? buildUnresolvableMessage(raw, kind)) this.name = 'UnresolvableFieldError' this.raw = raw this.kind = kind } } +/** + * @description Refusal for a malformed or out-of-map field ADDRESS — + * `system.` (including all plumbing), an empty + * name, or a bare `metadata.` prefix. The message carries the full valid + * system map so the fix never needs a docs lookup. + */ +export class InvalidFieldAddressError extends UnresolvableFieldError { + constructor(raw: string, kind: FieldAddressKind, systemMap: ReadonlySet) { + const valid = [...systemMap].map((f) => `system.${f}`).join(', ') + super( + raw, + kind, + `'${raw}' is not an addressable ${kind} field. Bare names address your own ` + + `metadata fields; engine fields are exactly: ${valid}. Engine plumbing ` + + `(vector, connections, level, data, _rev) is not part of the query surface.` + ) + this.name = 'InvalidFieldAddressError' + } +} + + /** * @description Refusal for a find() option that is accepted by the type * surface but NOT implemented — an accepted option must work or refuse; diff --git a/src/utils/metadataIndex.ts b/src/utils/metadataIndex.ts index bfde5fe9..6deeb811 100644 --- a/src/utils/metadataIndex.ts +++ b/src/utils/metadataIndex.ts @@ -5,7 +5,7 @@ */ import { StorageAdapter, resolveEntityField, NounMetadata, VerbMetadata } from '../coreTypes.js' -import { SYSTEM_ENTITY_SCALARS, parseFieldAddress } from '../db/fieldAddressing.js' +import { SYSTEM_ENTITY_SCALARS, parseFieldAddress, UnresolvableFieldError } from '../db/fieldAddressing.js' import { ColumnStore } from '../indexes/columnStore/ColumnStore.js' import type { MetadataIndexProvider } from '../plugin.js' import { MetadataIndexCache, MetadataIndexCacheConfig } from './metadataIndexCache.js' @@ -2250,6 +2250,18 @@ export class MetadataIndexManager implements MetadataIndexProvider { const orderKey = orderAddress.scope === 'system' ? `system.${orderAddress.field}` : orderAddress.field + // DATA-AWARE REFUSAL (the did-you-mean): a bare address no user field + // carries cannot mean anything as a sort key — and when the name collides + // with a system scalar the caller almost certainly meant system.. + // Refusing loudly with both candidates beats silently sorting nothing. + if ( + orderAddress.scope === 'metadata' && + !(this.columnStore && this.columnStore.hasField(orderKey)) && + !(await this.loadSparseIndex(orderKey)) + ) { + throw new UnresolvableFieldError(orderAddress.raw, 'entity') + } + // Column store path: O(K log S) sort via k-way merge across segments. // No per-entity storage reads, no precision loss from bucketing. if (this.columnStore && this.columnStore.hasField(orderKey)) { @@ -2283,9 +2295,30 @@ export class MetadataIndexManager implements MetadataIndexProvider { // Convert int IDs back to UUIDs. Number() narrowing is lossless — the // shipped EntityIdSpaceExceeded guard caps the JS mapper at u32. - return sortedIntIds + const sortedUuids = sortedIntIds .map(intId => this.idMapper.getUuid(Number(intId))) .filter((uuid): uuid is string => uuid !== undefined) + + // ORDERING CONTRACT (cross-engine, sealed): rows missing the field are + // NEVER dropped — they sort LAST in both directions — and ties break by + // id ascending. The column only contains rows that HAVE the field, so + // (1) re-sort the page deterministically (value, then id) with K cheap + // value reads, and (2) append the filtered rows the column omitted, + // id-ascending, filling any remaining page budget. + const page = await Promise.all( + sortedUuids.map(async id => ({ id, value: await this.getFieldValueForEntity(id, orderKey) })) + ) + page.sort((a, b) => this.compareAddressedValues(a.value, b.value, a.id, b.id, order)) + let result = page.map(p => p.id) + + if (hasFilter) { + const present = new Set(sortedUuids) + if (topK === undefined || result.length < topK) { + const missing = filteredIds.filter(id => !present.has(id)).sort() + result = result.concat(missing) + } + } + return topK !== undefined ? result.slice(0, topK) : result } // Fallback: sparse index path (for fields not yet in column store). @@ -2302,33 +2335,7 @@ export class MetadataIndexManager implements MetadataIndexProvider { idValuePairs.push({ id, value }) } - idValuePairs.sort((a, b) => { - // Ordering contract (cross-engine, ruled 2026-08-03): missing/null - // values sort LAST in BOTH directions — the direction flip never moves - // them to the front — and ties break by id ascending, so an ordered - // read is deterministic and identical on both engines. Rows are never - // dropped for lacking the field. - const aNull = a.value == null - const bNull = b.value == null - if (aNull || bNull) { - if (aNull && bNull) return a.id < b.id ? -1 : a.id > b.id ? 1 : 0 - return aNull ? 1 : -1 - } - // Numbers compare numerically; everything else by code-point (UTF-8 byte) order. - // This makes the JS fallback sort match cor's native column store exactly - // (numeric i64/f64 vs code-point strings) and stay deterministic across - // environments, unlike the `<` operator's UTF-16 ordering for strings. - let comparison = 0 - if (a.value !== b.value) { - if (typeof a.value === 'number' && typeof b.value === 'number') { - comparison = a.value < b.value ? -1 : 1 - } else { - comparison = compareCodePoints(String(a.value), String(b.value)) - } - } - if (comparison === 0) return a.id < b.id ? -1 : a.id > b.id ? 1 : 0 - return order === 'asc' ? comparison : -comparison - }) + idValuePairs.sort((a, b) => this.compareAddressedValues(a.value, b.value, a.id, b.id, order)) const sorted = idValuePairs.map(p => p.id) return topK !== undefined ? sorted.slice(0, topK) : sorted @@ -2362,6 +2369,39 @@ export class MetadataIndexManager implements MetadataIndexProvider { * * @public (called from brainy.ts for sorted queries) */ + /** + * The cross-engine ordering contract in one comparator (sealed 2026-08-03): + * missing/null values sort LAST in BOTH directions — the direction flip + * never moves them to the front — and ties break by id ascending, so an + * ordered read is deterministic and identical on both engines. Numbers + * compare numerically; everything else by code-point (UTF-8 byte) order, + * matching the native column store exactly. + */ + private compareAddressedValues( + aVal: any, + bVal: any, + aId: string, + bId: string, + order: 'asc' | 'desc' + ): number { + const aNull = aVal == null + const bNull = bVal == null + if (aNull || bNull) { + if (aNull && bNull) return aId < bId ? -1 : aId > bId ? 1 : 0 + return aNull ? 1 : -1 + } + let comparison = 0 + if (aVal !== bVal) { + if (typeof aVal === 'number' && typeof bVal === 'number') { + comparison = aVal < bVal ? -1 : 1 + } else { + comparison = compareCodePoints(String(aVal), String(bVal)) + } + } + if (comparison === 0) return aId < bId ? -1 : aId > bId ? 1 : 0 + return order === 'asc' ? comparison : -comparison + } + async getFieldValueForEntity(entityId: string, field: string): Promise { // `field` arrives as a FROZEN INDEX KEY (bare = user metadata; // 'system.' = engine scalar). Storage fallbacks read the matching diff --git a/tests/conformance/namespace-law.test.ts b/tests/conformance/namespace-law.test.ts index 91227587..0231685a 100644 --- a/tests/conformance/namespace-law.test.ts +++ b/tests/conformance/namespace-law.test.ts @@ -294,7 +294,9 @@ describe.skipIf(!lawActive)('namespace law — bare/system/metadata field addres brain.defineAggregate({ name: 'ns_law_by_team_bare', - source: { type: NounType.Document, where: { subtype: 'ns-law-group-bare' } }, + // system.subtype — bare 'subtype' would address user metadata under the + // law (the exact migration every fleet consumer's aggregates make). + source: { type: NounType.Document, where: { 'system.subtype': 'ns-law-group-bare' } }, groupBy: ['team'], metrics: { count: { op: 'count' } } }) From 48a6130a50251be464dda201ee85d40758efb63e Mon Sep 17 00:00:00 2001 From: David Snelling Date: Mon, 3 Aug 2026 16:07:27 -0700 Subject: [PATCH 12/13] =?UTF-8?q?feat(namespace):=20write-door=20forgery?= =?UTF-8?q?=20refusal=20(user=20metadata=20keys=20may=20never=20start=20's?= =?UTF-8?q?ystem.')=20+=20refusal=20messages=20name=20both=20spellings=20i?= =?UTF-8?q?n=20every=20branch=20(the=20non-colliding=20case=20marks=20syst?= =?UTF-8?q?em.=20honestly=20as=20NOT=20valid)=20=E2=80=94=20cross-engin?= =?UTF-8?q?e=20message=20pin=20alignment?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/db/fieldAddressing.ts | 3 ++- src/utils/paramValidation.ts | 22 ++++++++++++++++++++++ tests/unit/db/fieldAddressing.test.ts | 8 ++++++-- 3 files changed, 30 insertions(+), 3 deletions(-) diff --git a/src/db/fieldAddressing.ts b/src/db/fieldAddressing.ts index 98c81e5f..ae83ea18 100644 --- a/src/db/fieldAddressing.ts +++ b/src/db/fieldAddressing.ts @@ -247,7 +247,8 @@ export function buildUnresolvableMessage( return ( `no metadata field '${raw}' on this store — nothing carries it, so an ordered or ` + `filtered read against it cannot mean anything. Spell it metadata.${raw} once the ` + - `field exists, or check the field name.` + `field exists, or check the field name (system.${raw} is NOT valid — '${raw}' is ` + + `not one of the engine's system scalars).` ) } diff --git a/src/utils/paramValidation.ts b/src/utils/paramValidation.ts index fd018a04..749849b3 100644 --- a/src/utils/paramValidation.ts +++ b/src/utils/paramValidation.ts @@ -518,7 +518,28 @@ export function validateFindParams(params: FindParams): void { /** * Validate add parameters */ + +/** + * The namespace cannot be forged: a USER metadata key literally spelled + * 'system.' would collide with the engine's explicit address + * namespace at read time — refuse it at the write door, loudly, with the + * fix in the message (sealed 2026-08-03). + */ +function rejectForgedSystemKeys(metadata: Record | undefined, site: string): void { + if (!metadata) return + for (const key of Object.keys(metadata)) { + if (key.startsWith('system.')) { + throw new Error( + `${site}: metadata key '${key}' is not allowed — the 'system.' prefix is the ` + + `engine's explicit address namespace and cannot be used as a user field name. ` + + `Rename the field (e.g. '${key.slice('system.'.length)}').` + ) + } + } +} + export function validateAddParams(params: AddParams): void { + rejectForgedSystemKeys(params.metadata as Record | undefined, 'add()') // Universal truth: must have data or vector if (!params.data && !params.vector) { throw new Error( @@ -559,6 +580,7 @@ export function validateAddParams(params: AddParams): void { * Validate update parameters */ export function validateUpdateParams(params: UpdateParams): void { + rejectForgedSystemKeys(params.metadata as Record | undefined, 'update()') // Universal truth: must have an ID if (!params.id) { throw new Error('id is required for update') diff --git a/tests/unit/db/fieldAddressing.test.ts b/tests/unit/db/fieldAddressing.test.ts index f7ca1cbe..04110992 100644 --- a/tests/unit/db/fieldAddressing.test.ts +++ b/tests/unit/db/fieldAddressing.test.ts @@ -133,9 +133,13 @@ describe('field-addressing law — pure module pins', () => { expect(msg).toContain('metadata.createdAt') }) - it('a non-colliding unknown bare name gets the single-candidate refusal', () => { + it('a non-colliding unknown bare name names both spellings — system. explicitly as NOT valid', () => { + // Cross-engine pin (cor's suite greps for both spellings in every + // refusal): the metadata candidate is the fix; the system spelling is + // named but HONESTLY marked invalid, never offered as a candidate. const msg = buildUnresolvableMessage('scoore', 'entity') - expect(msg).not.toContain('system.scoore') expect(msg).toContain('metadata.scoore') + expect(msg).toContain('system.scoore') + expect(msg).toContain('NOT valid') }) }) From 24bf6cdbc58f329c0946166f9ab22d628494c290 Mon Sep 17 00:00:00 2001 From: David Snelling Date: Mon, 3 Aug 2026 16:59:13 -0700 Subject: [PATCH 13/13] =?UTF-8?q?feat(namespace):=20NO=20SPECIAL=20NAMES?= =?UTF-8?q?=20+=20storage=20fidelity=20=E2=80=94=20the=20ruled=20completio?= =?UTF-8?q?n=20of=20the=20field-addressing=20law?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The write side of the law, ruled 2026-08-03: data is either in main space where developers can use anything, or it is in system.*. - The reserved-name write door DIES: add/update/relate/updateRelation metadata bags accept EVERY name (confidence, type, id, data, level, content, ...) as ordinary user fields — indexed, filterable, sortable, aggregatable, identical to any other field. The remap/enforce/warn machinery, the reservedFieldPolicy config (now a typed init refusal), and the compile-time metadata key bans are all removed. The one write refusal left: keys spelled 'system.*' (namespace forgery), now enforced on all four write doors. - STORED RECORDS GO NESTED (v2): engine fields top-level, the user bag nested verbatim under 'metadata', sealed by a format stamp — by-name storage discrimination is unsound once colliders are admitted. Legacy flat records stay readable forever through the shape-aware splitters (sound for them: the old door refused colliders). Time travel rides the same split (generation store snapshots whole records). - Name-based index exclusions DIE: user frame indexes every name; the excludeFields/indexedFields knobs and their silent-[] holes are gone; bulk-payload protection is value-shape only, uniform across names. - Consumer-sweep findings fixed in the same wave: per-type counts read the frozen 'system.type' column (addToIndex sort, affinity tracking, cold-count rehydration, VFS type bitmaps — legacy 'noun' fallback for pre-rebuild reads); resolveHiddenIds addresses 'system.visibility' (bare 'visibility' was a silent no-op under the law — VFS/system entities leaked into default reads). - Fidelity fallout fixed in the owning layers: readEntityFieldAddress reads the bag first (colliders were absent-shadowed by its own guard) and never serves system addresses from the bag; blob history refs read the bag shape-aware; migration transforms now receive ONE normalized view (engine fields + nested bag) regardless of stored era, and stray flat-habit keys refuse with the fix in the message. - THE REOPEN-COLLIDER CONFORMANCE CASE (required before any RC counts as gates-green): all ten collider names + plumbing names written as user fields, verified verbatim + queryable across live reads, flush+reopen, a forced epoch rebuild, and asOf time travel; relation mirror; forgery refusals; legacy flat-record compat. 8/8 green. Gates: unit 1901/1901 (exit 0) · integration 758 (exit 0) · conformance 27/27 (exit 0) · consumer test sweep migrated (10 files). --- docs/concepts/field-addressing.md | 42 +- src/brainy.ts | 697 +++++------------- src/db/db.ts | 67 +- src/db/fieldAddressing.ts | 28 +- src/import/ImportCoordinator.ts | 62 +- src/index.ts | 7 +- src/migration/MigrationRunner.ts | 95 ++- src/migration/types.ts | 14 +- src/neural/neuralImport.ts | 19 +- src/storage/baseStorage.ts | 13 +- src/types/brainy.types.ts | 73 +- src/types/reservedFields.ts | 264 +++++-- src/utils/metadataIndex.ts | 223 +++--- src/utils/paramValidation.ts | 2 + tests/conformance/collider-fidelity.test.ts | 307 ++++++++ .../advanced-apis-regression.test.ts | 6 +- .../aggregate-reserved-fields.test.ts | 13 +- .../all-apis-comprehensive.test.ts | 4 +- tests/integration/fact-log-dual-write.test.ts | 10 +- tests/integration/lens-consistency.test.ts | 21 +- tests/integration/migration.test.ts | 82 ++- tests/integration/orderby-sort-bug.test.ts | 10 +- .../metadata-index-cleanup.unit.test.ts | 15 +- tests/unit/brainy/find-orderby-pagek.test.ts | 3 +- .../unit/brainy/reserved-field-policy.test.ts | 251 ------- .../update-reserved-metadata-remap.test.ts | 403 ---------- tests/unit/brainy/visibility.test.ts | 77 +- tests/unit/db/whereMatcher.test.ts | 39 +- tests/unit/test-suite-coverage-guard.test.ts | 13 +- tests/unit/types/nestedBagRecord.test.ts | 127 ++++ .../types/reserved-metadata-keys.test-d.ts | 265 ------- tests/unit/utils/paramValidation.test.ts | 8 +- 32 files changed, 1355 insertions(+), 1905 deletions(-) create mode 100644 tests/conformance/collider-fidelity.test.ts delete mode 100644 tests/unit/brainy/reserved-field-policy.test.ts delete mode 100644 tests/unit/brainy/update-reserved-metadata-remap.test.ts create mode 100644 tests/unit/types/nestedBagRecord.test.ts delete mode 100644 tests/unit/types/reserved-metadata-keys.test-d.ts diff --git a/docs/concepts/field-addressing.md b/docs/concepts/field-addressing.md index 863f7474..dcae1057 100644 --- a/docs/concepts/field-addressing.md +++ b/docs/concepts/field-addressing.md @@ -114,6 +114,43 @@ Reach for the explicit spelling when it reads more clearly next to a `system.` field in the same query — for example, sorting by your own `score` while filtering on `system.confidence`. +## No special names — the write side + +The same law governs writes: + +> **Data is either in main space, where developers can use anything, or it +> is in `system.*`.** + +There are **no reserved metadata names**. A field called `confidence`, +`type`, `id`, `data`, `content`, or anything else inside your `metadata` bag +is an ordinary user field: it is stored verbatim, indexed, filterable, +sortable, aggregatable, and it survives restarts, index rebuilds, and +time-travel (`asOf`) reads exactly as written — even when an engine scalar +shares its spelling. The engine's values are written only through their +dedicated params (`confidence`, `weight`, `subtype`, `visibility`, …) and +read at `system.`; your bag can never touch them and they can never +shadow your bag. + +```typescript +const id = await brain.add({ + data: 'Ada Lovelace', + type: NounType.Person, + confidence: 0.9, // the ENGINE scalar + metadata: { confidence: 'self-rated' } // YOUR field, same spelling — both live +}) + +await brain.find({ where: { confidence: 'self-rated' } }) // finds it (yours) +await brain.find({ where: { 'system.confidence': 0.9 } }) // finds it (engine's) +``` + +The one spelling a write refuses is a metadata key that literally starts +with `system.` — the explicit address namespace cannot be forged as a user +field name. That refusal is typed and names the fix. + +Value **shape** rules still apply uniformly to every name (they are not name +carve-outs): arrays longer than 10 elements are not turned into posting-list +scalars, and very long values are indexed by hash. + ## Refusal semantics A name that resolves to neither your metadata nor a system scalar is a typed @@ -190,7 +227,6 @@ against the new rule. ## Where to go next -- [Consistency Model](./consistency-model.md) — the separate (and - longer-standing) contract for *reserved* fields: which names may never - appear inside a `metadata` bag at write time, distinct from this page's +- [Consistency Model](./consistency-model.md) — visibility tiers, revision + counters, and the rest of the read/write contract this page's read-time addressing rule. diff --git a/src/brainy.ts b/src/brainy.ts index 3dd8ef93..600e4474 100644 --- a/src/brainy.ts +++ b/src/brainy.ts @@ -148,7 +148,9 @@ import { import { NounType, VerbType, TypeUtils } from './types/graphTypes.js' import { splitNounMetadataRecord, - splitVerbMetadataRecord + splitVerbMetadataRecord, + buildNounMetadataRecord, + buildVerbMetadataRecord } from './types/reservedFields.js' import { BrainyInterface } from './types/brainyInterface.js' import type { IntegrationHub, IntegrationHubConfig } from './integrations/core/IntegrationHub.js' @@ -746,6 +748,21 @@ export class Brainy implements BrainyInterface { private lazyRebuildPromise: Promise | null = null constructor(config?: BrainyConfig) { + // The reserved-field write policy died with the field-addressing law: + // every metadata name is the user's now (engine scalars write via their + // dedicated params and read at `system.*`), so there is nothing left for + // the policy to govern. A config still passing it refuses loudly rather + // than being silently ignored. + if (config && 'reservedFieldPolicy' in (config as Record)) { + throw new Error( + `reservedFieldPolicy was removed by the field-addressing law: metadata field ` + + `names are never reserved anymore — every name in the metadata bag is the ` + + `user's and works like any other field. Set engine scalars via their ` + + `dedicated params (confidence, weight, subtype, …) and query them as ` + + `system.. Remove the reservedFieldPolicy option.` + ) + } + // Normalize configuration with defaults this.config = this.normalizeConfig(config) @@ -2018,12 +2035,6 @@ export class Brainy implements BrainyInterface { // Zero-config validation (static import for performance) validateAddParams(params) - // Reserved fields arriving via the metadata bag (untyped callers — the - // compile-time guard stops TypeScript callers) are normalized to their - // canonical top-level location BEFORE any enforcement runs, so a - // remapped subtype participates in subtype-pairing enforcement and the - // indexed metadata bag carries only custom fields. - params = this.remapReservedAddMetadata(params) // Tracked-field vocabulary enforcement (Layer 2). Walks both bags so a // tracked field declared at top level (e.g. 'subtype') and one declared in @@ -2095,28 +2106,33 @@ export class Brainy implements BrainyInterface { ) } - // Prepare metadata for storage - // data is stored opaquely in the 'data' field - NOT spread into top-level metadata. - // Only metadata fields are queryable via find({ where }). - const storageMetadata = { - ...params.metadata, - // Preserve the caller's original (non-UUID) id when normalized, so reads - // can surface it. A real UUID passes through with no _originalId. - ...(originalId !== undefined && { [ORIGINAL_ID_KEY]: originalId }), - data: params.data, - noun: params.type, - ...(params.subtype !== undefined && { subtype: params.subtype }), - // visibility: stored only when not 'public' (absent === public, keeps records lean) - ...(params.visibility !== undefined && - params.visibility !== 'public' && { visibility: params.visibility }), - service: params.service, - createdAt: Date.now(), - updatedAt: Date.now(), - _rev: 1, - ...(params.confidence !== undefined && { confidence: params.confidence }), - ...(params.weight !== undefined && { weight: params.weight }), - ...(params.createdBy && { createdBy: params.createdBy }) - } + // Prepare metadata for storage: a v2 nested-bag record — engine fields + // top-level, the user's bag nested VERBATIM (any name, including engine + // spellings like `confidence` or `type`, is the user's and survives + // faithfully; the field-addressing law). + const storageMetadata = buildNounMetadataRecord( + { + data: params.data, + noun: params.type, + ...(params.subtype !== undefined && { subtype: params.subtype }), + // visibility: stored only when not 'public' (absent === public, keeps records lean) + ...(params.visibility !== undefined && + params.visibility !== 'public' && { visibility: params.visibility }), + service: params.service, + createdAt: Date.now(), + updatedAt: Date.now(), + _rev: 1, + ...(params.confidence !== undefined && { confidence: params.confidence }), + ...(params.weight !== undefined && { weight: params.weight }), + ...(params.createdBy && { createdBy: params.createdBy }) + }, + { + ...params.metadata, + // Preserve the caller's original (non-UUID) id when normalized, so reads + // can surface it. A real UUID passes through with no _originalId. + ...(originalId !== undefined && { [ORIGINAL_ID_KEY]: originalId }) + } + ) // Build entity structure for indexing (NEW - with top-level fields) // Optional fields must use conditional spreading to match storageMetadata exactly. @@ -2627,320 +2643,6 @@ export class Brainy implements BrainyInterface { return entity } - /** One-shot registry for reserved-field warnings (per process, per method+field). */ - private static warnedReservedFields = new Set() - - /** - * @description Resolve the human-readable "correct write path" guidance for a - * reserved field on a given write method. Single source of truth shared by the - * `'throw'` (Error message) and `'warn'` (one-shot warning) paths so the two - * never drift. The trio `confidence` / `weight` / `subtype` and the - * add()/relate()-time fields `service` / `createdBy` / `visibility` map to a - * dedicated param; everything else is system-managed. - * @param method - The public write method the bag arrived through. - * @param field - The reserved field name found in the metadata bag. - * @returns Guidance naming the correct way to set the field. - */ - private reservedWritePath( - method: 'add' | 'update' | 'relate' | 'updateRelation', - field: string - ): string { - const typeParam = "the top-level 'type' param" - switch (field) { - case 'noun': - case 'verb': - return typeParam - case 'data': - return "the top-level 'data' param" - case 'confidence': - return "the 'confidence' param" - case 'weight': - return "the 'weight' param" - case 'subtype': - return "the 'subtype' param" - case 'visibility': - return "the 'visibility' param ('public' | 'internal')" - case 'service': - return method === 'add' - ? "the 'service' param of add()" - : method === 'relate' - ? "the 'service' param of relate()" - : 'nothing — service is fixed at create time' - case 'createdBy': - return method === 'add' - ? "the 'createdBy' param of add()" - : 'nothing — createdBy is system-managed' - case 'createdAt': - return 'nothing — creation time is set automatically' - case 'updatedAt': - return 'nothing — set automatically on every write' - case '_rev': - return method === 'update' - ? "the 'ifRev' param for optimistic concurrency" - : 'nothing — revisions are system-managed' - default: - return 'a dedicated top-level param' - } - } - - /** - * @description Enforce {@link BrainyConfig.reservedFieldPolicy} for reserved - * fields found inside a metadata bag. Called by every write-path remap once - * the bag has been split and at least one reserved key is present. - * - * - `'throw'` (default): throw a clear Error naming every offending key and - * its correct write path. The caller never reaches the remap. - * - `'warn'`: emit a ONE-SHOT (per method+field, per process) warning for - * EVERY reserved key found — both the user-mutable fields that are about to - * be remapped and the system-managed fields that are about to be dropped — - * then fall through to the legacy remap. - * - `'remap'`: silent legacy remap, no warning. - * - * @param method - The public write method the bag arrived through. - * @param reserved - The reserved half of the split metadata bag (non-empty). - * @param reservedListName - `'RESERVED_ENTITY_FIELDS'` or - * `'RESERVED_RELATION_FIELDS'` — named in the thrown Error for discoverability. - * @returns `true` when the caller should proceed with the legacy remap - * (`'warn'` / `'remap'`); `'throw'` never returns (it throws first). - * @throws {Error} When the policy is `'throw'` and any reserved key is present. - */ - private enforceReservedPolicy( - method: 'add' | 'update' | 'relate' | 'updateRelation', - reserved: Partial>, - reservedListName: 'RESERVED_ENTITY_FIELDS' | 'RESERVED_RELATION_FIELDS' - ): boolean { - const policy = this.config.reservedFieldPolicy ?? 'throw' - const keys = Object.keys(reserved) - if (keys.length === 0) return true - - if (policy === 'throw') { - const detail = keys - .map((k) => { - const path = this.reservedWritePath(method, k) - // System-managed fields resolve to a "nothing — …" sentinel; phrase - // those as "is system-managed" rather than "pass it as the nothing". - return path.startsWith('nothing') - ? `metadata.${k} is a reserved field (${path.replace(/^nothing\s*—\s*/, '')}) and cannot be set through ${method}()` - : `metadata.${k} is a reserved field — pass it as ${path} to ${method}()` - }) - .join('; ') - throw new Error( - `${detail} (reserved: see ${reservedListName}). ` + - `Set reservedFieldPolicy:'remap' to opt into legacy remapping, ` + - `or reservedFieldPolicy:'warn' to remap with a warning.` - ) - } - - if (policy === 'warn') { - // One-shot warning for EVERY reserved key (today only system-managed ones - // warn — this closes that gap so user-mutable remaps are visible too). - for (const k of keys) { - this.warnReservedRemapped(method, k, this.reservedWritePath(method, k)) - } - } - - // 'warn' and 'remap' both fall through to the legacy remap. - return true - } - - /** - * @description One-shot (per method+field, per process) warning that a - * reserved field arrived inside a metadata bag under the `'warn'` policy. The - * wording is neutral on "remapped vs dropped" — `reservedWritePath()` already - * tells the caller where the value goes (a dedicated param, or "nothing"). - * @param method - The public write method the bag arrived through. - * @param field - The reserved field name found in the bag. - * @param rightPath - Guidance naming the correct write path. - */ - private warnReservedRemapped(method: string, field: string, rightPath: string): void { - const key = `${method}:${field}` - if (Brainy.warnedReservedFields.has(key)) return - Brainy.warnedReservedFields.add(key) - // System-managed fields resolve to a "nothing — …" sentinel; phrase the - // guidance so it reads cleanly in both the remapped and dropped cases. - const guidance = rightPath.startsWith('nothing') - ? `it is ${rightPath.replace(/^nothing\s*—\s*/, '')} and was dropped` - : `set it via ${rightPath} instead` - prodLog.warn( - `[brainy] ${method}(): '${field}' is a reserved field and was found inside the ` + - `metadata bag — ${guidance}. (Legacy remap applied because ` + - `reservedFieldPolicy is 'warn'. This warning is shown once per field per process.)` - ) - } - - /** - * @description Normalize an `add()` params object with respect to - * Brainy-reserved fields arriving inside `metadata` (untyped callers only — - * the compile-time guard on `AddParams.metadata` stops TypeScript callers). - * Governed by {@link BrainyConfig.reservedFieldPolicy} (default `'throw'`): - * `'throw'` rejects the write naming the offending key(s); `'warn'`/`'remap'` - * fall through to the legacy remap, where fields with a dedicated `add()` - * param (`confidence`, `weight`, `subtype`, `visibility`, `service`, - * `createdBy`) are remapped to that param unless the caller also passed it - * explicitly (top-level wins) and system-managed fields (`noun`, `data`, - * `createdAt`, `updatedAt`, `_rev`) are dropped. A remapped `subtype` flows - * through subtype-pairing enforcement exactly like a top-level one. - * @param params - The caller's add params (not mutated). - * @returns Params with reserved fields normalized out of `metadata`. - * @throws {Error} When `reservedFieldPolicy` is `'throw'` and the bag carries a reserved key. - */ - private remapReservedAddMetadata(params: AddParams): AddParams { - const bag = params.metadata as Record | undefined - if (!bag || typeof bag !== 'object') return params - const { reserved, custom } = splitNounMetadataRecord(bag) - if (Object.keys(reserved).length === 0) return params - - // Policy gate: 'throw' (default) throws here; 'warn' warns once per key then - // remaps; 'remap' silently remaps. (Throw never returns.) - this.enforceReservedPolicy('add', reserved, 'RESERVED_ENTITY_FIELDS') - - const createdBy = reserved.createdBy as { augmentation?: unknown; version?: unknown } | undefined - const createdByValid = - typeof createdBy === 'object' && - createdBy !== null && - typeof createdBy.augmentation === 'string' && - typeof createdBy.version === 'string' - - return { - ...params, - metadata: custom as AddParams['metadata'], - ...(params.confidence === undefined && - typeof reserved.confidence === 'number' && { confidence: reserved.confidence }), - ...(params.weight === undefined && - typeof reserved.weight === 'number' && { weight: reserved.weight }), - ...(params.subtype === undefined && - typeof reserved.subtype === 'string' && { subtype: reserved.subtype }), - ...(params.visibility === undefined && - (reserved.visibility === 'public' || reserved.visibility === 'internal') && { - visibility: reserved.visibility as 'public' | 'internal' - }), - ...(params.service === undefined && - typeof reserved.service === 'string' && { service: reserved.service }), - ...(params.createdBy === undefined && - createdByValid && { createdBy: createdBy as { augmentation: string; version: string } }) - } - } - - /** - * @description Normalize an `update()` params object with respect to - * Brainy-reserved fields arriving inside the metadata patch — the `update()` - * mirror of {@link remapReservedAddMetadata}, closing the historical trap - * where `add({metadata:{confidence}})` lifted the field but - * `update({metadata:{confidence}})` silently dropped it (the patch value - * survived the merge and was then clobbered by the preserve-existing - * spread; a production consumer's confidence-evolution writes no-oped until - * read back). Governed by {@link BrainyConfig.reservedFieldPolicy} (default - * `'throw'`): `'throw'` rejects the write; `'warn'`/`'remap'` remap - * user-mutable fields (`confidence`, `weight`, `subtype`) to their dedicated - * param unless the caller also passed it (top-level wins) and drop everything - * else (`noun`, `data`, `createdAt`, `updatedAt`, `service`, `createdBy`, - * `_rev`) as system-managed or fixed at `add()` time. - * @param params - The caller's update params (not mutated). - * @returns Params with reserved fields normalized out of `metadata`. - * @throws {Error} When `reservedFieldPolicy` is `'throw'` and the bag carries a reserved key. - */ - private remapReservedUpdateMetadata(params: UpdateParams): UpdateParams { - const bag = params.metadata as Record | undefined - if (!bag || typeof bag !== 'object') return params - const { reserved, custom } = splitNounMetadataRecord(bag) - if (Object.keys(reserved).length === 0) return params - - // Policy gate: 'throw' (default) throws; 'warn' warns once per key then - // remaps; 'remap' silently remaps. - this.enforceReservedPolicy('update', reserved, 'RESERVED_ENTITY_FIELDS') - - return { - ...params, - metadata: custom as UpdateParams['metadata'], - ...(params.confidence === undefined && - typeof reserved.confidence === 'number' && { confidence: reserved.confidence }), - ...(params.weight === undefined && - typeof reserved.weight === 'number' && { weight: reserved.weight }), - ...(params.subtype === undefined && - typeof reserved.subtype === 'string' && { subtype: reserved.subtype }) - } - } - - /** - * @description Normalize a `relate()` params object with respect to - * Brainy-reserved fields arriving inside `metadata` — the relationship - * mirror of {@link remapReservedAddMetadata}. Governed by - * {@link BrainyConfig.reservedFieldPolicy} (default `'throw'`): `'throw'` - * rejects the write; `'warn'`/`'remap'` remap fields with a dedicated - * `relate()` param (`confidence`, `weight`, `subtype`, `visibility`, - * `service`) to that param (top-level wins) and drop system-managed fields - * (`verb`, `data`, `createdAt`, `updatedAt`, `createdBy`, `_rev`). - * @param params - The caller's relate params (not mutated). - * @returns Params with reserved fields normalized out of `metadata`. - * @throws {Error} When `reservedFieldPolicy` is `'throw'` and the bag carries a reserved key. - */ - private remapReservedRelateMetadata(params: RelateParams): RelateParams { - const bag = params.metadata as Record | undefined - if (!bag || typeof bag !== 'object') return params - const { reserved, custom } = splitVerbMetadataRecord(bag) - if (Object.keys(reserved).length === 0) return params - - // Policy gate: 'throw' (default) throws; 'warn' warns once per key then - // remaps; 'remap' silently remaps. - this.enforceReservedPolicy('relate', reserved, 'RESERVED_RELATION_FIELDS') - - return { - ...params, - metadata: custom as RelateParams['metadata'], - ...(params.confidence === undefined && - typeof reserved.confidence === 'number' && { confidence: reserved.confidence }), - ...(params.weight === undefined && - typeof reserved.weight === 'number' && { weight: reserved.weight }), - ...(params.subtype === undefined && - typeof reserved.subtype === 'string' && { subtype: reserved.subtype }), - ...(params.visibility === undefined && - (reserved.visibility === 'public' || reserved.visibility === 'internal') && { - visibility: reserved.visibility as 'public' | 'internal' - }), - ...(params.service === undefined && - typeof reserved.service === 'string' && { service: reserved.service }) - } - } - - /** - * @description Normalize an `updateRelation()` params object with respect - * to Brainy-reserved fields arriving inside the metadata patch — the - * relationship mirror of {@link remapReservedUpdateMetadata}. Governed by - * {@link BrainyConfig.reservedFieldPolicy} (default `'throw'`): `'throw'` - * rejects the write; `'warn'`/`'remap'` remap user-mutable fields - * (`confidence`, `weight`, `subtype`, `visibility`) to their dedicated param - * (top-level wins) and drop everything else. - * @param params - The caller's update-relation params (not mutated). - * @returns Params with reserved fields normalized out of `metadata`. - * @throws {Error} When `reservedFieldPolicy` is `'throw'` and the bag carries a reserved key. - */ - private remapReservedUpdateRelationMetadata( - params: UpdateRelationParams - ): UpdateRelationParams { - const bag = params.metadata as Record | undefined - if (!bag || typeof bag !== 'object') return params - const { reserved, custom } = splitVerbMetadataRecord(bag) - if (Object.keys(reserved).length === 0) return params - - // Policy gate: 'throw' (default) throws; 'warn' warns once per key then - // remaps; 'remap' silently remaps. - this.enforceReservedPolicy('updateRelation', reserved, 'RESERVED_RELATION_FIELDS') - - return { - ...params, - metadata: custom as UpdateRelationParams['metadata'], - ...(params.confidence === undefined && - typeof reserved.confidence === 'number' && { confidence: reserved.confidence }), - ...(params.weight === undefined && - typeof reserved.weight === 'number' && { weight: reserved.weight }), - ...(params.subtype === undefined && - typeof reserved.subtype === 'string' && { subtype: reserved.subtype }), - ...(params.visibility === undefined && - (reserved.visibility === 'public' || reserved.visibility === 'internal') && { - visibility: reserved.visibility as 'public' | 'internal' - }) - } - } /** * Update an existing entity @@ -3006,12 +2708,6 @@ export class Brainy implements BrainyInterface { // Reserved fields arriving via the metadata patch are remapped to their // canonical top-level location, mirroring add()'s lift. Without this the // patch value survived the merge but was then clobbered by the - // preserve-existing spreads below — a silent no-op consumers could only - // detect by reading values back. User-mutable fields (confidence, - // weight, subtype) remap unless the same field was also passed top-level - // (top-level wins); system-managed fields are dropped with a one-shot - // warning naming the right path. - params = this.remapReservedUpdateMetadata(params) // Tracked-field vocabulary enforcement (Layer 2). Same as add() — the // metadata bag carries fields registered via trackField(), and subtype is @@ -3078,31 +2774,33 @@ export class Brainy implements BrainyInterface { ? { ...existing.metadata, ...params.metadata } : params.metadata || existing.metadata - // Prepare updated metadata object - // data is stored opaquely in the 'data' field - NOT spread into top-level metadata. - const updatedMetadata = { - ...newMetadata, - data: params.data !== undefined ? params.data : existing.data, - noun: params.type || existing.type, - service: existing.service, - createdAt: existing.createdAt, - updatedAt: Date.now(), - _rev: currentRev + 1, - // Update confidence and weight if provided, otherwise preserve existing - ...(params.confidence !== undefined && { confidence: params.confidence }), - ...(params.weight !== undefined && { weight: params.weight }), - ...(params.confidence === undefined && existing.confidence !== undefined && { confidence: existing.confidence }), - ...(params.weight === undefined && existing.weight !== undefined && { weight: existing.weight }), - // Update subtype if provided, otherwise preserve existing - ...(params.subtype !== undefined && { subtype: params.subtype }), - ...(params.subtype === undefined && existing.subtype !== undefined && { subtype: existing.subtype }), - // Visibility: take the new value if provided, else preserve existing. Stored only - // when the effective value is not 'public' (absent === public, keeps records lean). - // A change to 'public' therefore drops the field entirely. - ...(((params.visibility ?? existing.visibility) ?? 'public') !== 'public' && { - visibility: params.visibility ?? existing.visibility - }) - } + // Prepare the updated v2 nested-bag record: engine fields top-level, + // the merged user bag nested verbatim (collider names stay the user's). + const updatedMetadata = buildNounMetadataRecord( + { + data: params.data !== undefined ? params.data : existing.data, + noun: params.type || existing.type, + service: existing.service, + createdAt: existing.createdAt, + updatedAt: Date.now(), + _rev: currentRev + 1, + // Update confidence and weight if provided, otherwise preserve existing + ...(params.confidence !== undefined && { confidence: params.confidence }), + ...(params.weight !== undefined && { weight: params.weight }), + ...(params.confidence === undefined && existing.confidence !== undefined && { confidence: existing.confidence }), + ...(params.weight === undefined && existing.weight !== undefined && { weight: existing.weight }), + // Update subtype if provided, otherwise preserve existing + ...(params.subtype !== undefined && { subtype: params.subtype }), + ...(params.subtype === undefined && existing.subtype !== undefined && { subtype: existing.subtype }), + // Visibility: take the new value if provided, else preserve existing. Stored only + // when the effective value is not 'public' (absent === public, keeps records lean). + // A change to 'public' therefore drops the field entirely. + ...(((params.visibility ?? existing.visibility) ?? 'public') !== 'public' && { + visibility: params.visibility ?? existing.visibility + }) + }, + newMetadata as Record + ) // Build entity structure for metadata index (with top-level fields). // No `level`: engine plumbing never enters the indexing view (it @@ -4043,9 +3741,6 @@ export class Brainy implements BrainyInterface { // engine-minted UUID — relation ids are never caller-supplied here.) params = { ...params, from: resolveEntityId(params.from), to: resolveEntityId(params.to) } - // Reserved fields arriving via the metadata bag are normalized to their - // canonical top-level params before enforcement — mirror of add()'s lift. - params = this.remapReservedRelateMetadata(params) // Subtype pairing enforcement (Layer 3 — 7.30.0). Per-type rules registered // via brain.requireSubtype() compose with the brain-wide strict-mode flag. @@ -4097,25 +3792,28 @@ export class Brainy implements BrainyInterface { (v, i) => (v + toEntity.vector[i]) / 2 ) - // Prepare verb metadata - // User metadata spread FIRST, then system fields ALWAYS win (prevents collision) + // Prepare verb metadata: a v2 nested-bag record — engine fields + // top-level, the user's edge bag nested verbatim (any name is the + // user's; the field-addressing law). // One timestamp for both createdAt and updatedAt so a never-updated edge reports a // stable updatedAt (=== createdAt) instead of a fresh Date.now() fabricated per read. const relateTs = Date.now() - const verbMetadata = { - ...(params.metadata || {}), - verb: params.type, - ...(params.subtype !== undefined && { subtype: params.subtype }), - // visibility: stored only when not 'public' (absent === public, keeps records lean) - ...(params.visibility !== undefined && - params.visibility !== 'public' && { visibility: params.visibility }), - weight: params.weight ?? 1.0, - ...(params.confidence !== undefined && { confidence: params.confidence }), - ...(params.service !== undefined && { service: params.service }), - createdAt: relateTs, - updatedAt: relateTs, - ...(params.data !== undefined && { data: params.data }) - } + const verbMetadata = buildVerbMetadataRecord( + { + verb: params.type, + ...(params.subtype !== undefined && { subtype: params.subtype }), + // visibility: stored only when not 'public' (absent === public, keeps records lean) + ...(params.visibility !== undefined && + params.visibility !== 'public' && { visibility: params.visibility }), + weight: params.weight ?? 1.0, + ...(params.confidence !== undefined && { confidence: params.confidence }), + ...(params.service !== undefined && { service: params.service }), + createdAt: relateTs, + updatedAt: relateTs, + ...(params.data !== undefined && { data: params.data }) + }, + (params.metadata as Record) || {} + ) // Save to storage (vector and metadata separately) const verb: GraphVerb = { @@ -4347,9 +4045,6 @@ export class Brainy implements BrainyInterface { validateUpdateRelationParams(params) - // Reserved fields arriving via the metadata patch are remapped to their - // canonical top-level params — mirror of update()'s normalization. - params = this.remapReservedUpdateRelationMetadata(params) const existing = await this.storage.getVerb(params.id) if (!existing) { @@ -4378,32 +4073,36 @@ export class Brainy implements BrainyInterface { ? { ...(existingRec.metadata || {}), ...(params.metadata || {}) } : params.metadata || existingRec.metadata - // Build updated stored metadata. System fields ALWAYS win — same shape as relate(). - const updatedMetadata = { - ...newMetadata, - verb: newVerbType, - ...(params.subtype !== undefined - ? { subtype: params.subtype } - : existingRec.subtype !== undefined && { subtype: existingRec.subtype }), - // Visibility: new value if provided, else preserve existing; stored only when the - // effective value is not 'public' (a change to 'public' drops the field). - ...(((params.visibility ?? existingRec.visibility) ?? 'public') !== 'public' && { - visibility: params.visibility ?? existingRec.visibility - }), - weight: params.weight ?? existingRec.weight ?? 1.0, - ...(params.confidence !== undefined - ? { confidence: params.confidence } - : existingRec.confidence !== undefined && { confidence: existingRec.confidence }), - // service/createdBy are fixed at relate() time — always carried forward - // (omitting them here silently erased them on every updateRelation()). - ...(existingRec.service !== undefined && { service: existingRec.service }), - ...(existingRec.createdBy !== undefined && { createdBy: existingRec.createdBy }), - createdAt: existingRec.createdAt, - updatedAt: Date.now(), - ...(params.data !== undefined - ? { data: params.data } - : existingRec.data !== undefined && { data: existingRec.data }) - } + // Build the updated stored record: v2 nested-bag — engine fields + // top-level, the merged user bag nested verbatim (mirror of update()). + const updatedWeight = params.weight ?? existingRec.weight ?? 1.0 + const updatedData = + params.data !== undefined ? params.data : existingRec.data + const updatedMetadata = buildVerbMetadataRecord( + { + verb: newVerbType, + ...(params.subtype !== undefined + ? { subtype: params.subtype } + : existingRec.subtype !== undefined && { subtype: existingRec.subtype }), + // Visibility: new value if provided, else preserve existing; stored only when the + // effective value is not 'public' (a change to 'public' drops the field). + ...(((params.visibility ?? existingRec.visibility) ?? 'public') !== 'public' && { + visibility: params.visibility ?? existingRec.visibility + }), + weight: updatedWeight, + ...(params.confidence !== undefined + ? { confidence: params.confidence } + : existingRec.confidence !== undefined && { confidence: existingRec.confidence }), + // service/createdBy are fixed at relate() time — always carried forward + // (omitting them here silently erased them on every updateRelation()). + ...(existingRec.service !== undefined && { service: existingRec.service }), + ...(existingRec.createdBy !== undefined && { createdBy: existingRec.createdBy }), + createdAt: existingRec.createdAt, + updatedAt: Date.now(), + ...(updatedData !== undefined && { data: updatedData }) + }, + newMetadata as Record + ) // Build the verb view used by the graph index — top-level fields mirror relate()'s. const verbForIndex: GraphVerb = { @@ -4419,9 +4118,9 @@ export class Brainy implements BrainyInterface { ...(((params.visibility ?? existingRec.visibility) ?? 'public') !== 'public' && { visibility: params.visibility ?? existingRec.visibility }), - weight: updatedMetadata.weight, + weight: updatedWeight, metadata: newMetadata, - data: updatedMetadata.data, + data: updatedData, createdAt: existingRec.createdAt } @@ -6027,8 +5726,12 @@ export class Brainy implements BrainyInterface { ): Promise> { const excluded = this.excludedVisibilityTiers(params) if (!excluded) return new Set() + // 'system.visibility' — the engine scalar's frozen address. A bare + // 'visibility' key would address the USER's metadata bag under the + // field-addressing law and silently hide nothing (VFS/system entities + // would leak into every default read). const ids = await this.metadataIndex.getIdsForFilter({ - visibility: excluded.length === 1 ? excluded[0] : { oneOf: excluded } + 'system.visibility': excluded.length === 1 ? excluded[0] : { oneOf: excluded } }) return new Set(ids) } @@ -9282,10 +8985,7 @@ export class Brainy implements BrainyInterface { ): Promise { const { op: _discriminator, ...rawParams } = op validateAddParams(rawParams as AddParams) - // Same reserved-field normalization as add() — the metadata bag is - // cleaned BEFORE enforcement so a remapped subtype participates in - // subtype-pairing enforcement and only custom fields reach the index. - const params = this.remapReservedAddMetadata(rawParams as AddParams) + const params = rawParams as AddParams this.enforceTrackedFieldValues(params.metadata as Record | undefined, 'metadata') this.enforceTrackedFieldValues({ subtype: params.subtype } as Record, 'top-level') this.enforceSubtypeOnAdd('add', params.type, params.subtype, params.metadata) @@ -9360,25 +9060,31 @@ export class Brainy implements BrainyInterface { plan.createdNouns.add(id) const now = Date.now() - const storageMetadata = { - ...params.metadata, - // Preserve the caller's original (non-UUID) id when normalized — mirror - // of add(). A real UUID passes through with no _originalId. - ...(originalId !== undefined && { [ORIGINAL_ID_KEY]: originalId }), - data: params.data, - noun: params.type, - ...(params.subtype !== undefined && { subtype: params.subtype }), - // visibility: stored only when not 'public' (absent === public, keeps records lean) - ...(params.visibility !== undefined && - params.visibility !== 'public' && { visibility: params.visibility }), - service: params.service, - createdAt: now, - updatedAt: now, - _rev: 1, - ...(params.confidence !== undefined && { confidence: params.confidence }), - ...(params.weight !== undefined && { weight: params.weight }), - ...(params.createdBy && { createdBy: params.createdBy }) - } + // v2 nested-bag record — mirror of add(): engine fields top-level, the + // user's bag nested verbatim (collider names stay the user's). + const storageMetadata = buildNounMetadataRecord( + { + data: params.data, + noun: params.type, + ...(params.subtype !== undefined && { subtype: params.subtype }), + // visibility: stored only when not 'public' (absent === public, keeps records lean) + ...(params.visibility !== undefined && + params.visibility !== 'public' && { visibility: params.visibility }), + service: params.service, + createdAt: now, + updatedAt: now, + _rev: 1, + ...(params.confidence !== undefined && { confidence: params.confidence }), + ...(params.weight !== undefined && { weight: params.weight }), + ...(params.createdBy && { createdBy: params.createdBy }) + }, + { + ...params.metadata, + // Preserve the caller's original (non-UUID) id when normalized — mirror + // of add(). A real UUID passes through with no _originalId. + ...(originalId !== undefined && { [ORIGINAL_ID_KEY]: originalId }) + } + ) const entityForIndexing = { id, vector, @@ -9441,10 +9147,7 @@ export class Brainy implements BrainyInterface { ): Promise { const { op: _discriminator, ...rawParams } = op validateUpdateParams(rawParams as UpdateParams) - // Same reserved-field normalization as update() — user-mutable fields - // remap to their dedicated param (top-level wins), system-managed fields - // drop with a one-shot warning. - const params = this.remapReservedUpdateMetadata(rawParams as UpdateParams) + const params = rawParams as UpdateParams // Id normalization (8.0) — mirror of update(): a natural key resolves to the // canonical UUID add() stored. A real UUID passes through. params.id = resolveEntityId(params.id) @@ -9496,29 +9199,33 @@ export class Brainy implements BrainyInterface { ? { ...existing.metadata, ...params.metadata } : params.metadata || existing.metadata const now = Date.now() - const updatedMetadata = { - ...newMetadata, - data: params.data !== undefined ? params.data : existing.data, - noun: params.type || existing.type, - service: existing.service, - createdAt: existing.createdAt, - updatedAt: now, - _rev: currentRev + 1, - ...(params.confidence !== undefined && { confidence: params.confidence }), - ...(params.weight !== undefined && { weight: params.weight }), - ...(params.confidence === undefined && - existing.confidence !== undefined && { confidence: existing.confidence }), - ...(params.weight === undefined && - existing.weight !== undefined && { weight: existing.weight }), - ...(params.subtype !== undefined && { subtype: params.subtype }), - ...(params.subtype === undefined && - existing.subtype !== undefined && { subtype: existing.subtype }), - // Visibility: new value if provided, else preserve existing; stored only when the - // effective value is not 'public' (a change to 'public' drops the field). - ...(((params.visibility ?? existing.visibility) ?? 'public') !== 'public' && { - visibility: params.visibility ?? existing.visibility - }) - } + // v2 nested-bag record — mirror of update(): engine fields top-level, + // the merged user bag nested verbatim. + const updatedMetadata = buildNounMetadataRecord( + { + data: params.data !== undefined ? params.data : existing.data, + noun: params.type || existing.type, + service: existing.service, + createdAt: existing.createdAt, + updatedAt: now, + _rev: currentRev + 1, + ...(params.confidence !== undefined && { confidence: params.confidence }), + ...(params.weight !== undefined && { weight: params.weight }), + ...(params.confidence === undefined && + existing.confidence !== undefined && { confidence: existing.confidence }), + ...(params.weight === undefined && + existing.weight !== undefined && { weight: existing.weight }), + ...(params.subtype !== undefined && { subtype: params.subtype }), + ...(params.subtype === undefined && + existing.subtype !== undefined && { subtype: existing.subtype }), + // Visibility: new value if provided, else preserve existing; stored only when the + // effective value is not 'public' (a change to 'public' drops the field). + ...(((params.visibility ?? existing.visibility) ?? 'public') !== 'public' && { + visibility: params.visibility ?? existing.visibility + }) + }, + newMetadata as Record + ) // Register for the authoritative under-mutex CAS re-verify + rev re-stamp // (see PlannedTransact.casUpdates). The staged UpdateNounMetadataOperation @@ -9739,8 +9446,7 @@ export class Brainy implements BrainyInterface { ): Promise { const { op: _discriminator, ...rawParams } = op validateRelateParams(rawParams as RelateParams) - // Same reserved-field normalization as relate(). - const params = this.remapReservedRelateMetadata(rawParams as RelateParams) + const params = rawParams as RelateParams // Id normalization (8.0) — mirror of relate(): resolve BOTH endpoints to the // canonical UUID add() stored, so a relate op may reference either side by // natural key. Real UUIDs pass through. (Relationship ids are engine-minted.) @@ -9790,19 +9496,23 @@ export class Brainy implements BrainyInterface { const id = uuidv4() const relationVector = fromEntity.vector.map((v, i) => (v + toEntity.vector[i]) / 2) const now = Date.now() - const verbMetadata = { - ...(params.metadata || {}), - verb: params.type, - ...(params.subtype !== undefined && { subtype: params.subtype }), - // visibility: stored only when not 'public' (absent === public, keeps records lean) - ...(params.visibility !== undefined && - params.visibility !== 'public' && { visibility: params.visibility }), - weight: params.weight ?? 1.0, - ...(params.confidence !== undefined && { confidence: params.confidence }), - ...(params.service !== undefined && { service: params.service }), - createdAt: now, - ...(params.data !== undefined && { data: params.data }) - } + // v2 nested-bag record — mirror of relate(): engine fields top-level, + // the user's edge bag nested verbatim. + const verbMetadata = buildVerbMetadataRecord( + { + verb: params.type, + ...(params.subtype !== undefined && { subtype: params.subtype }), + // visibility: stored only when not 'public' (absent === public, keeps records lean) + ...(params.visibility !== undefined && + params.visibility !== 'public' && { visibility: params.visibility }), + weight: params.weight ?? 1.0, + ...(params.confidence !== undefined && { confidence: params.confidence }), + ...(params.service !== undefined && { service: params.service }), + createdAt: now, + ...(params.data !== undefined && { data: params.data }) + }, + (params.metadata as Record) || {} + ) const verb: GraphVerb = { id, vector: relationVector, @@ -15115,12 +14825,7 @@ export class Brainy implements BrainyInterface { requireSubtype: config?.requireSubtype ?? true, // Multi-process safety mode: config?.mode ?? 'writer', - force: config?.force ?? false, - // Reserved-field-in-metadata-bag policy (8.0 — no silent failures). - // Default 'throw': an untyped caller that smuggles a reserved key past - // the compile guard gets a loud Error naming the correct write path. - // 'warn' = remap + one-shot warning per key; 'remap' = legacy silent remap. - reservedFieldPolicy: config?.reservedFieldPolicy ?? 'throw' + force: config?.force ?? false } } diff --git a/src/db/db.ts b/src/db/db.ts index c5cbad8b..68428a7c 100644 --- a/src/db/db.ts +++ b/src/db/db.ts @@ -59,10 +59,6 @@ import type { import type { StorageAdapter } from '../coreTypes.js' import { exportGraph } from './portableGraph.js' import type { ExportSelector, ExportOptions, PortableGraph } from './portableGraph.js' -import { - splitNounMetadataRecord, - splitVerbMetadataRecord -} from '../types/reservedFields.js' import { v4 as uuidv4 } from '../universal/uuid.js' import { coerceNewEntityId, resolveEntityId, ORIGINAL_ID_KEY } from '../utils/idNormalization.js' import { EntityNotFoundError } from '../errors/notFound.js' @@ -705,23 +701,15 @@ export class Db { for (const op of ops) { switch (op.op) { case 'add': { - // Reserved-field normalization — mirror of the brain.transact() - // write path: user-settable fields lift to their dedicated field - // (top-level wins), system-managed fields drop, and the entity's - // metadata bag carries ONLY custom fields. Speculative views skip - // the one-shot warnings — committing the same ops through - // `brain.transact()` warns on the real write path. - const { reserved, custom } = splitNounMetadataRecord( - op.metadata as Record | undefined - ) - const confidence = - op.confidence ?? (typeof reserved.confidence === 'number' ? reserved.confidence : undefined) - const weight = - op.weight ?? (typeof reserved.weight === 'number' ? reserved.weight : undefined) - const subtype = - op.subtype ?? (typeof reserved.subtype === 'string' ? reserved.subtype : undefined) - const service = - op.service ?? (typeof reserved.service === 'string' ? reserved.service : undefined) + // Field-addressing law: the metadata bag is the user's, VERBATIM — + // no reserved-name lift, no drops. Engine scalars come ONLY from + // their dedicated op fields; a bag field named `confidence` is an + // ordinary user field, exactly as on the committed write path. + const custom = { ...(op.metadata as Record | undefined) } + const confidence = op.confidence + const weight = op.weight + const subtype = op.subtype + const service = op.service // Id normalization (8.0) — mirror of the committed transact() add // path: a natural key coerces to a STABLE UUID (v5), preserving the @@ -759,16 +747,12 @@ export class Db { `with(): entity ${updateId} not found at generation ${this.gen}` ) } - // Same reserved-field normalization as the committed update path. - const { reserved, custom } = splitNounMetadataRecord( - op.metadata as Record | undefined - ) - const confidence = - op.confidence ?? (typeof reserved.confidence === 'number' ? reserved.confidence : undefined) - const weight = - op.weight ?? (typeof reserved.weight === 'number' ? reserved.weight : undefined) - const subtype = - op.subtype ?? (typeof reserved.subtype === 'string' ? reserved.subtype : undefined) + // Field-addressing law — mirror of the add case: the patch bag is + // the user's verbatim; engine scalars only from dedicated op fields. + const custom = { ...(op.metadata as Record | undefined) } + const confidence = op.confidence + const weight = op.weight + const subtype = op.subtype const mergedMetadata = op.merge !== false ? ({ ...(base.metadata as object), ...custom } as T) @@ -830,19 +814,14 @@ export class Db { } if (duplicate) break - // Reserved-field normalization — relationship mirror of the add - // op above (and of the committed relate() path). - const { reserved, custom } = splitVerbMetadataRecord( - op.metadata as Record | undefined - ) - const confidence = - op.confidence ?? (typeof reserved.confidence === 'number' ? reserved.confidence : undefined) - const weight = - op.weight ?? (typeof reserved.weight === 'number' ? reserved.weight : undefined) - const subtype = - op.subtype ?? (typeof reserved.subtype === 'string' ? reserved.subtype : undefined) - const service = - op.service ?? (typeof reserved.service === 'string' ? reserved.service : undefined) + // Field-addressing law — relationship mirror of the add case: the + // edge bag is the user's verbatim; engine scalars only from + // dedicated op fields. + const custom = { ...(op.metadata as Record | undefined) } + const confidence = op.confidence + const weight = op.weight + const subtype = op.subtype + const service = op.service const id = uuidv4() overlay.verbs.set(id, { diff --git a/src/db/fieldAddressing.ts b/src/db/fieldAddressing.ts index ae83ea18..21689319 100644 --- a/src/db/fieldAddressing.ts +++ b/src/db/fieldAddressing.ts @@ -174,23 +174,26 @@ export function readEntityFieldAddress( : null if (address.scope === 'system') { - // Entity views carry system scalars top-level; raw storage shapes carry - // them inside the stored metadata record (where `type` is spelled `noun`). - // Read top-level first, then the record — never the user's namespace. + // System scalars live at the record's top level, NEVER in the user's + // bag — a user field named `confidence` must be unreachable from + // system.confidence (and vice versa). Entity views carry the scalars + // top-level directly; record-derived views spell the type `noun`. const top = rec[address.field] if (top !== undefined) return top - if (bag) { - if (address.field === 'type') return bag.type ?? bag.noun - return bag[address.field] - } + if (address.field === 'type') return rec.noun return undefined } - // User scope. The write-path remap guarantees the user can never OWN a - // field named like a system scalar (those lift top-level at write), so a - // bare system name reads as ABSENT — reading the stored record's reserved - // key here would re-create the shadow this module exists to kill. Same for - // plumbing and the legacy 'noun' spelling. + // User scope: the bag IS the user's namespace, authoritative — EVERY name + // reads from it, engine spellings included (`bag.confidence` is the user's + // confidence field under the field-addressing law). + if (bag) return bag[address.field] + + // No bag at all: a LEGACY flat record (pre-nested-bag storage). Its keys + // matching system/plumbing names are the ENGINE's — the pre-law write door + // refused user colliders — so a bare system name reads as ABSENT rather + // than resurrecting the shadow this module exists to kill. Same for the + // legacy 'noun' spelling. if ( SYSTEM_ENTITY_SCALARS.has(address.field) || PLUMBING_FIELDS.has(address.field) || @@ -198,7 +201,6 @@ export function readEntityFieldAddress( ) { return undefined } - if (bag) return bag[address.field] return rec[address.field] } diff --git a/src/import/ImportCoordinator.ts b/src/import/ImportCoordinator.ts index 1e1316b7..dd145045 100644 --- a/src/import/ImportCoordinator.ts +++ b/src/import/ImportCoordinator.ts @@ -22,7 +22,6 @@ import { SmartYAMLImporter } from '../importers/SmartYAMLImporter.js' import { SmartDOCXImporter } from '../importers/SmartDOCXImporter.js' import { VFSStructureGenerator } from '../importers/VFSStructureGenerator.js' import { NounType, VerbType } from '../types/graphTypes.js' -import { splitNounMetadataRecord, splitVerbMetadataRecord } from '../types/reservedFields.js' import { v4 as uuidv4 } from '../universal/uuid.js' import * as fs from 'fs' import * as path from 'path' @@ -871,35 +870,18 @@ export class ImportCoordinator { } /** - * Strip Brainy-reserved entity keys out of an extractor-supplied metadata bag. - * - * Extractors (and consumer `customMetadata`) can carry reserved keys - * (`confidence`, `subtype`, `weight`, …) inside `metadata`. Brainy 8.0's - * default `reservedFieldPolicy` is `'throw'`, so spreading such a bag into - * `add({ metadata })` would reject the whole import. The import pipeline owns - * the correct write path: user-mutable reserved values are passed as dedicated - * `AddParams` params (see the call sites), so here we simply drop the reserved - * half of the bag and keep only the custom fields that belong in `metadata`. - * + * Normalize an extractor/consumer metadata bag for spreading — the + * field-addressing law: the bag is the user's, VERBATIM. No name is + * reserved anymore ('confidence', 'subtype', 'type', … in a source bag + * import as ordinary user fields); the old reserved-key strip was data + * loss under the law and is gone. A forged 'system.'-prefixed key still + * refuses loudly at the write door (`rejectForgedSystemKeys`). * @param bag - The extractor/consumer metadata bag (may be undefined). - * @returns The custom-only metadata (reserved keys removed). + * @returns The bag itself, or `{}` for non-object inputs. */ - private stripReservedFromBag(bag: Record | undefined | null): Record { + private bagVerbatim(bag: Record | undefined | null): Record { if (!bag || typeof bag !== 'object') return {} - return splitNounMetadataRecord(bag).custom - } - - /** - * Relationship mirror of {@link stripReservedFromBag} — strips reserved verb - * keys (`verb`, `confidence`, `weight`, `subtype`, …) out of an edge metadata - * bag so it carries only custom fields. Reserved values that have a dedicated - * `RelateParams` param are passed there by the call site instead. - * @param bag - The extractor/consumer edge metadata bag (may be undefined). - * @returns The custom-only edge metadata (reserved keys removed). - */ - private stripReservedFromRelationBag(bag: Record | undefined | null): Record { - if (!bag || typeof bag !== 'object') return {} - return splitVerbMetadataRecord(bag).custom + return bag } /** @@ -1017,7 +999,7 @@ export class ImportCoordinator { importedAt: trackingContext.importedAt, importFormat: trackingContext.importFormat, importSource: trackingContext.importSource, - ...this.stripReservedFromBag(trackingContext.customMetadata) + ...this.bagVerbatim(trackingContext.customMetadata) }) } }) @@ -1045,13 +1027,11 @@ export class ImportCoordinator { data: entity.description || entity.name, type: entity.type, subtype: entity.subtype ?? options.defaultSubtype ?? 'imported', - // `confidence` is a reserved field — pass it as the dedicated param, - // never inside the metadata bag (8.0 reservedFieldPolicy defaults to 'throw'). + // Engine confidence rides its dedicated param; the bag below is + // the user's verbatim (no name is reserved — field-addressing law). confidence: entity.confidence, metadata: { - // Extractor/consumer bags may smuggle reserved keys — strip them so - // the bag carries only custom fields. - ...this.stripReservedFromBag(entity.metadata), + ...this.bagVerbatim(entity.metadata), name: entity.name, vfsPath: vfsFile?.path, importedFrom: 'import-coordinator', @@ -1064,7 +1044,7 @@ export class ImportCoordinator { importSource: trackingContext.importSource, sourceRow: row.rowNumber, sourceSheet: row.sheet, - ...this.stripReservedFromBag(trackingContext.customMetadata) + ...this.bagVerbatim(trackingContext.customMetadata) }) } } @@ -1145,7 +1125,7 @@ export class ImportCoordinator { importIds: [trackingContext.importId], projectId: trackingContext.projectId, importFormat: trackingContext.importFormat, - ...this.stripReservedFromRelationBag(trackingContext.customMetadata) + ...this.bagVerbatim(trackingContext.customMetadata) }) } } @@ -1180,7 +1160,7 @@ export class ImportCoordinator { confidence: entity.confidence, metadata: { // Strip any reserved keys an extractor smuggled into the bag. - ...this.stripReservedFromBag(entity.metadata), + ...this.bagVerbatim(entity.metadata), name: entity.name, vfsPath: vfsFile?.path, importedFrom: 'import-coordinator', @@ -1194,7 +1174,7 @@ export class ImportCoordinator { importSource: trackingContext.importSource, sourceRow: row.rowNumber, sourceSheet: row.sheet, - ...this.stripReservedFromBag(trackingContext.customMetadata) + ...this.bagVerbatim(trackingContext.customMetadata) }) } }) @@ -1234,7 +1214,7 @@ export class ImportCoordinator { importIds: [trackingContext.importId], projectId: trackingContext.projectId, importFormat: trackingContext.importFormat, - ...this.stripReservedFromRelationBag(trackingContext.customMetadata) + ...this.bagVerbatim(trackingContext.customMetadata) }) } }) @@ -1289,7 +1269,7 @@ export class ImportCoordinator { projectId: trackingContext.projectId, importedAt: trackingContext.importedAt, importFormat: trackingContext.importFormat, - ...this.stripReservedFromBag(trackingContext.customMetadata) + ...this.bagVerbatim(trackingContext.customMetadata) }) } }) @@ -1319,7 +1299,7 @@ export class ImportCoordinator { projectId: trackingContext.projectId, importedAt: trackingContext.importedAt, importFormat: trackingContext.importFormat, - ...this.stripReservedFromRelationBag(trackingContext.customMetadata) + ...this.bagVerbatim(trackingContext.customMetadata) }) } }) @@ -1422,7 +1402,7 @@ export class ImportCoordinator { ...(typeof (rel as any).confidence === 'number' && { confidence: (rel as any).confidence }), ...(typeof (rel as any).weight === 'number' && { weight: (rel as any).weight }), metadata: { - ...this.stripReservedFromRelationBag(rel.metadata), + ...this.bagVerbatim(rel.metadata), relationshipType: 'semantic', // Distinguish from VFS/provenance inferredType: verbType !== rel.type, // Track if type was enhanced originalType: rel.type diff --git a/src/index.ts b/src/index.ts index 3876a903..3186a6a7 100644 --- a/src/index.ts +++ b/src/index.ts @@ -89,7 +89,12 @@ export { RESERVED_ENTITY_FIELDS, RESERVED_RELATION_FIELDS, splitNounMetadataRecord, - splitVerbMetadataRecord + splitVerbMetadataRecord, + buildNounMetadataRecord, + buildVerbMetadataRecord, + isNestedBagRecord, + METADATA_RECORD_FORMAT_KEY, + NESTED_BAG_FORMAT } from './types/reservedFields.js' export type { ReservedEntityField, diff --git a/src/migration/MigrationRunner.ts b/src/migration/MigrationRunner.ts index d2e251a2..6a8a34bd 100644 --- a/src/migration/MigrationRunner.ts +++ b/src/migration/MigrationRunner.ts @@ -9,6 +9,67 @@ import type { BaseStorage } from '../storage/baseStorage.js' import type { NounMetadata, VerbMetadata } from '../coreTypes.js' import type { Migration, MigrationState, MigrationPreview, MigrationResult, MigrateOptions, MigrationError } from './types.js' import { MIGRATIONS } from './migrations.js' +import { + splitNounMetadataRecord, + splitVerbMetadataRecord, + buildNounMetadataRecord, + buildVerbMetadataRecord, + RESERVED_ENTITY_FIELDS, + RESERVED_RELATION_FIELDS +} from '../types/reservedFields.js' + +const RESERVED_NOUN_SET: ReadonlySet = new Set(RESERVED_ENTITY_FIELDS) +const RESERVED_VERB_SET: ReadonlySet = new Set(RESERVED_RELATION_FIELDS) + +/** + * Normalize a stored record (either era: legacy flat OR v2 nested-bag) into + * THE transform view — the one shape every migration transform receives: + * engine fields top-level, the user's metadata bag nested under `metadata`. + * Transforms never see the storage era; a migration written today works on + * a brain of any age. + */ +function toTransformView( + record: Record, + kind: 'noun' | 'verb' +): Record { + const { reserved, custom } = + kind === 'noun' ? splitNounMetadataRecord(record) : splitVerbMetadataRecord(record) + return { ...reserved, metadata: { ...custom } } +} + +/** + * Convert a transform's returned view back into a stamped v2 stored record. + * LOUD CONTRACT: user fields belong inside `.metadata` — a stray top-level + * key that is not an engine field is a migration bug under the + * field-addressing law (pre-law transforms wrote user fields flat), and it + * refuses with the fix in the message rather than silently dropping or + * silently storing it as an engine key. + */ +function fromTransformView( + view: Record, + kind: 'noun' | 'verb' +): Record { + const reservedSet = kind === 'noun' ? RESERVED_NOUN_SET : RESERVED_VERB_SET + const engine: Record = {} + for (const [key, value] of Object.entries(view)) { + if (key === 'metadata') continue + if (!reservedSet.has(key)) { + throw new Error( + `migration transform returned a top-level key '${key}' that is not an ` + + `engine field — under the field-addressing law user fields live inside ` + + `.metadata (return { ...view, metadata: { ...view.metadata, ${key}: … } }).` + ) + } + engine[key] = value + } + const bag = + view.metadata && typeof view.metadata === 'object' && !Array.isArray(view.metadata) + ? (view.metadata as Record) + : {} + return kind === 'noun' + ? buildNounMetadataRecord(engine, bag) + : buildVerbMetadataRecord(engine, bag) +} const MIGRATION_STATE_KEY = '__migration_state__' const PREVIEW_SAMPLE_SIZE = 5 @@ -125,14 +186,16 @@ export class MigrationRunner { const entityMeta = metadataBatch.get(entity.id) if (!entityMeta) continue - const metadata = entityMeta as Record - const result = this.applyTransforms(metadata, nounMigrations) + // Transforms see THE view (engine fields + nested user bag), + // never the raw storage era. + const view = toTransformView(entityMeta as Record, 'noun') + const result = this.applyTransforms(view, nounMigrations) if (result !== null) { affectedEntities++ if (sampleChanges.length < PREVIEW_SAMPLE_SIZE) { sampleChanges.push({ id: entity.id, - before: { ...metadata }, + before: view, after: result }) } @@ -157,14 +220,14 @@ export class MigrationRunner { const verbMeta = await this.storage.getVerbMetadata(verb.id) if (!verbMeta) continue - const metadata = verbMeta as Record - const result = this.applyTransforms(metadata, verbMigrations) + const view = toTransformView(verbMeta as Record, 'verb') + const result = this.applyTransforms(view, verbMigrations) if (result !== null) { affectedEntities++ if (sampleChanges.length < PREVIEW_SAMPLE_SIZE) { sampleChanges.push({ id: verb.id, - before: { ...metadata }, + before: view, after: result }) } @@ -289,9 +352,16 @@ export class MigrationRunner { if (!entityMeta) continue try { - const transformed = migration.transform(entityMeta as Record) + const transformed = migration.transform( + toTransformView(entityMeta as Record, 'noun') + ) if (transformed !== null) { - await this.storage.saveNounMetadata(entity.id, transformed as NounMetadata) + // Re-stamp as a v2 record (also upgrades legacy records touched + // by a migration onto the nested-bag shape). + await this.storage.saveNounMetadata( + entity.id, + fromTransformView(transformed, 'noun') as NounMetadata + ) modified++ } } catch (err) { @@ -357,9 +427,14 @@ export class MigrationRunner { if (!metadata) continue try { - const transformed = migration.transform(metadata as Record) + const transformed = migration.transform( + toTransformView(metadata as Record, 'verb') + ) if (transformed !== null) { - await this.storage.saveVerbMetadata(verb.id, transformed as VerbMetadata) + await this.storage.saveVerbMetadata( + verb.id, + fromTransformView(transformed, 'verb') as VerbMetadata + ) modified++ } } catch (err) { diff --git a/src/migration/types.ts b/src/migration/types.ts index 2dcc1d1a..a63e40b9 100644 --- a/src/migration/types.ts +++ b/src/migration/types.ts @@ -14,7 +14,19 @@ export interface Migration { description: string /** Which entity types this migration applies to */ applies: 'nouns' | 'verbs' | 'both' - /** Return transformed metadata, or null if no change needed */ + /** + * Return the transformed record view, or null if no change needed. + * + * THE VIEW CONTRACT (field-addressing law): the transform receives ONE + * normalized shape regardless of how old the stored record is — engine + * fields top-level (`noun`/`verb`, `subtype`, `confidence`, `weight`, + * timestamps, `_rev`, …) and the USER's metadata bag nested under + * `metadata` (where every name is the user's, engine spellings included). + * Return the same shape: user-field changes go inside `.metadata`; a + * stray non-engine top-level key in the returned object refuses loudly + * (it is the pre-law flat habit, and silently guessing its namespace + * would corrupt data). + */ transform: (metadata: Record) => Record | null } diff --git a/src/neural/neuralImport.ts b/src/neural/neuralImport.ts index c8d19eb5..ed240a3f 100644 --- a/src/neural/neuralImport.ts +++ b/src/neural/neuralImport.ts @@ -7,7 +7,6 @@ import { Brainy } from '../brainy.js' import { NounType, VerbType } from '../types/graphTypes.js' -import { splitNounMetadataRecord, splitVerbMetadataRecord } from '../types/reservedFields.js' import * as fs from '../universal/fs.js' import * as path from '../universal/path.js' // @ts-ignore @@ -803,12 +802,14 @@ export class NeuralImport { data: this.extractMainText(entity.originalData), type: entity.nounType as NounType, subtype: entity.subtype ?? options.defaultSubtype ?? 'extracted', - // `confidence` is a reserved field — dedicated param, not metadata - // (8.0 reservedFieldPolicy defaults to 'throw'). + // Engine confidence rides its dedicated param; the source object + // imports as the user's bag VERBATIM — no name is reserved + // (field-addressing law). confidence: entity.confidence, metadata: { - // Strip any reserved keys the source data smuggled into the bag. - ...splitNounMetadataRecord(entity.originalData).custom, + ...(typeof entity.originalData === 'object' && entity.originalData !== null + ? entity.originalData + : {}), id: entity.suggestedId } }) @@ -822,11 +823,13 @@ export class NeuralImport { type: relationship.verbType as VerbType, subtype: relationship.subtype ?? options.defaultSubtype ?? 'extracted', weight: relationship.weight, - confidence: relationship.confidence, // reserved field — dedicated param, not metadata + confidence: relationship.confidence, // engine confidence — dedicated param metadata: { context: relationship.context, - // Strip any reserved keys smuggled into the edge metadata bag. - ...splitVerbMetadataRecord(relationship.metadata).custom + // The edge bag imports verbatim — no name is reserved. + ...(typeof relationship.metadata === 'object' && relationship.metadata !== null + ? relationship.metadata + : {}) } }) } diff --git a/src/storage/baseStorage.ts b/src/storage/baseStorage.ts index 1d3e245d..b78b4a49 100644 --- a/src/storage/baseStorage.ts +++ b/src/storage/baseStorage.ts @@ -36,7 +36,8 @@ import { BrainyError, ProtectedArtifactError, DerivedArtifactMissingError } from import { MetadataWriteBuffer } from '../utils/metadataWriteBuffer.js' import { splitNounMetadataRecord, - splitVerbMetadataRecord + splitVerbMetadataRecord, + isNestedBagRecord } from '../types/reservedFields.js' /** @@ -1013,8 +1014,14 @@ export abstract class BaseStorage extends BaseStorageAdapter { const hashes: string[] = [] for (const record of records) { if (record.kind !== 'noun') continue - const storage = (record.metadata as { storage?: { type?: string; hash?: unknown } } | null) - ?.storage + // The VFS blob pointer (`storage: {type:'blob', hash}`) is a USER-bag + // field: in a v2 nested-bag record it lives inside `metadata`, in a + // legacy flat record it sits at the top level — read shape-aware. + const raw = record.metadata as Record | null + const bag = isNestedBagRecord(raw) + ? (raw!.metadata as Record) + : raw + const storage = (bag as { storage?: { type?: string; hash?: unknown } } | null)?.storage if (storage?.type === 'blob' && typeof storage.hash === 'string') { hashes.push(storage.hash) } diff --git a/src/types/brainy.types.ts b/src/types/brainy.types.ts index 6c133cf0..6c1f0ffd 100644 --- a/src/types/brainy.types.ts +++ b/src/types/brainy.types.ts @@ -320,15 +320,18 @@ export interface AddParams { */ visibility?: 'public' | 'internal' /** - * Structured queryable fields — indexed by MetadataIndex, used in `where` filters. + * Structured queryable fields — indexed by MetadataIndex, used in `where` + * filters, `orderBy`, and aggregation. * - * Reserved entity fields (`RESERVED_ENTITY_FIELDS` — `noun`, `subtype`, `visibility`, - * `createdAt`, `updatedAt`, `confidence`, `weight`, `service`, `data`, `createdBy`, - * `_rev`) may NOT appear here — they have dedicated top-level params and the type makes - * a literal reserved key a compile error. Untyped (JavaScript) callers that pass one - * anyway are normalized at write time: user-settable fields remap to their top-level - * param (top-level wins when both are supplied), system-managed fields are dropped with - * a one-shot warning. + * THE FIELD-ADDRESSING LAW: every name here is YOURS. There are no + * reserved metadata names — `confidence`, `type`, `id`, `level`, `data`, + * `content`, … are ordinary user fields that index, filter, sort, and + * aggregate like any other, and survive faithfully across restarts and + * rebuilds. Engine scalars are set only via their dedicated params + * (`confidence`, `weight`, `subtype`, …) and are queried explicitly as + * `system.` (`where: { 'system.confidence': … }`). The ONE illegal + * spelling is a key starting `'system.'` — the engine's explicit address + * namespace cannot be forged; such a write refuses with a typed error. */ metadata?: EntityMetadataInput /** Custom entity ID. When omitted, a time-ordered UUID v7 is generated; a supplied natural-key string is normalized to a stable UUID v5. */ @@ -386,12 +389,11 @@ export interface UpdateParams { */ visibility?: EntityVisibility /** - * Metadata fields to merge (or replace when `merge: false`). Reserved entity - * fields (`RESERVED_ENTITY_FIELDS`) may NOT appear here — `confidence` / - * `weight` / `subtype` / `visibility` have dedicated params on this call, and the rest - * are system-managed. A literal reserved key is a compile error; untyped callers - * are normalized at write time (remap user-settable, drop system-managed - * with a one-shot warning). + * Metadata fields to merge (or replace when `merge: false`). Every name is + * the user's (the field-addressing law) — a patch field named `confidence` + * updates YOUR field of that name, never the engine scalar (use the + * dedicated `confidence` param for that). Keys spelled `'system.…'` refuse + * with a typed error (namespace forgery). */ metadata?: EntityMetadataPatch merge?: boolean // Merge or replace metadata (default: true) @@ -444,11 +446,11 @@ export interface RelateParams { /** Content for the relationship (optional — overrides auto-computed vector) */ data?: any /** - * Structured queryable fields on the edge. Reserved relationship fields - * (`RESERVED_RELATION_FIELDS` — `verb`, `subtype`, `visibility`, `createdAt`, - * `updatedAt`, `confidence`, `weight`, `service`, `data`, `createdBy`, `_rev`) may NOT - * appear here — they have dedicated params. A literal reserved key is a - * compile error; untyped callers are normalized at write time. + * Structured queryable fields on the edge. Every name is the user's (the + * field-addressing law) — `verb`, `confidence`, `weight`, … in this bag are + * ordinary user fields; engine scalars ride their dedicated params and are + * addressed as `system.`. Keys spelled `'system.…'` refuse with a + * typed error (namespace forgery). */ metadata?: RelationMetadataInput /** Create reverse edge too (default: false) */ @@ -478,10 +480,9 @@ export interface UpdateRelationParams { confidence?: number // New confidence (0-1) data?: any // New content /** - * Metadata fields to merge (or replace when `merge: false`). Reserved - * relationship fields (`RESERVED_RELATION_FIELDS`) may NOT appear here — - * a literal reserved key is a compile error; untyped callers are - * normalized at write time. + * Metadata fields to merge (or replace when `merge: false`). Every name is + * the user's (the field-addressing law); engine scalars ride their + * dedicated params. Keys spelled `'system.…'` refuse with a typed error. */ metadata?: RelationMetadataPatch merge?: boolean // Merge or replace metadata @@ -2027,32 +2028,6 @@ export interface BrainyConfig { */ force?: boolean - /** - * How write paths react when an untyped (JavaScript) caller smuggles a - * Brainy-reserved field (`RESERVED_ENTITY_FIELDS` / `RESERVED_RELATION_FIELDS` - * — `confidence`, `weight`, `subtype`, `visibility`, `service`, `createdBy`, - * `noun`/`verb`, `data`, `createdAt`, `updatedAt`, `_rev`) **inside the - * `metadata` bag** of `add()` / `update()` / `relate()` / `updateRelation()` - * (and their `transact()` / `with()` mirrors). TypeScript callers can't write - * these shapes at all — the compile-time guard on the metadata param types - * (`NoReservedEntityKeys` / `NoReservedRelationKeys`) rejects a literal - * reserved key — so this policy only governs untyped callers that slip one - * past the compiler. - * - * - `'throw'` (**default, 8.0**): a reserved key in the bag throws a clear - * `Error` naming the offending key(s) and the correct write path. No silent - * remap, no data loss, no surprise. This is the 8.0 "no silent failures" - * contract. - * - `'warn'`: legacy remapping with a loud, one-shot (per key, per process) - * warning for EVERY reserved key found — user-mutable fields are remapped to - * their dedicated top-level param (top-level wins when both are supplied), - * system-managed fields are dropped. Use while migrating untyped call sites. - * - `'remap'`: the pre-8.0 silent remapping, no warning. Last-resort - * compatibility hatch for code that intentionally relies on the bag path. - * - * @default 'throw' - */ - reservedFieldPolicy?: 'throw' | 'warn' | 'remap' } // ============= Neural API Types ============= diff --git a/src/types/reservedFields.ts b/src/types/reservedFields.ts index a0606e1d..15b585c5 100644 --- a/src/types/reservedFields.ts +++ b/src/types/reservedFields.ts @@ -1,35 +1,54 @@ /** * @module types/reservedFields - * @description The canonical reserved-field contract — ONE place that defines - * which keys belong to Brainy (top-level entity/relationship fields) and may - * therefore never live inside a `metadata` bag. + * @description The stored-record layout contract — ONE place that defines + * which keys of a persisted metadata record belong to the ENGINE (top-level + * entity/relationship fields) and how the USER's metadata bag is kept apart + * from them, faithfully, across flush / reopen / rebuild / time travel. * - * Three layers enforce the contract, all driven by the constants below: + * THE FIELD-ADDRESSING LAW (ruled 2026-08-03, VENUE-BRAINY-ORDERBY-NOOP): + * data is either in main space — where developers can use ANY name, and it + * all works with every database function — or it is in `system.*`. There are + * NO reserved user-facing metadata names anymore: `confidence`, `type`, + * `level`, `data`, `id`, `content` … inside a metadata bag are ordinary user + * fields. The only refused write is a user metadata key literally starting + * with `'system.'` (namespace forgery — see `rejectForgedSystemKeys`). * - * 1. **Compile time** — `AddParams.metadata`, `UpdateParams.metadata`, - * `RelateParams.metadata` and `UpdateRelationParams.metadata` are typed so - * a literal reserved key is a TypeScript error (see - * {@link EntityMetadataInput} / {@link RelationMetadataInput}). - * 2. **Write time** — for untyped (JavaScript) callers that smuggle a - * reserved key past the compiler anyway, every write path normalizes the - * bag: user-mutable fields are remapped to their dedicated top-level - * param (top-level wins when both are supplied) and system-managed fields - * are dropped with a one-shot warning naming the correct write path. - * 3. **Read time** — every read path splits the stored flat record through - * {@link splitNounMetadataRecord} / {@link splitVerbMetadataRecord}, so a - * reserved field is surfaced ONLY at top level and `entity.metadata` / - * `relation.metadata` contain ONLY custom fields, always — live reads, - * batch reads, and historical (`asOf`) reads alike. + * That law makes name-based storage discrimination unsound for NEW records + * (a user field named `confidence` may now legally sit beside the engine's + * confidence scalar), so persisted metadata records carry the user bag + * NESTED, shape-discriminated by a format stamp: * - * Documented for consumers in `docs/concepts/consistency-model.md` - * ("Reserved fields"). + * - **v2 (nested-bag)** — `{ …engine fields…, [METADATA_RECORD_FORMAT_KEY]: + * NESTED_BAG_FORMAT, metadata: { …user bag, verbatim… } }`. Built ONLY by + * {@link buildNounMetadataRecord} / {@link buildVerbMetadataRecord}; the + * engine half and the user bag can never collide because they never share + * a level. + * - **legacy (flat)** — engine fields and user fields mixed at one level, + * discriminated BY NAME through the RESERVED_* lists. Sound for legacy + * records precisely because the pre-law write door REFUSED user metadata + * carrying those names — a flat key matching a reserved name IS the + * engine's value in any record the old door admitted. + * + * {@link splitNounMetadataRecord} / {@link splitVerbMetadataRecord} read + * BOTH shapes (stamp first, name split as the legacy fallback) and are the + * single read-side choke point for live, batch, AND historical (`asOf`) + * reads — the generation store snapshots whole records, so time travel + * rides the same split. + * + * The RESERVED_* lists therefore no longer describe a user-facing ban — they + * describe the ENGINE HALF of the stored record layout (and drive the legacy + * split). The write-door remap machinery and the compile-time metadata key + * bans that used to enforce the old contract are gone. */ /** - * @description Entity (noun) field names reserved by Brainy. These keys are - * stored in the flat per-entity metadata record alongside custom fields, but - * they belong to Brainy: every read path extracts them to top-level - * `Entity` fields, and no write path accepts them inside `metadata`. + * @description Entity (noun) field names owned by the ENGINE in a stored + * metadata record. In v2 (nested-bag) records these are the legal TOP-LEVEL + * keys beside the nested `metadata` bag; in legacy flat records they drive + * the by-name split. They are NOT a user-facing ban list: since the + * field-addressing law, a user metadata field may carry any of these names + * and remains the user's — it lives inside the nested bag, never at the + * record's top level. * * | Key | Canonical write path | * |-----|----------------------| @@ -119,68 +138,54 @@ export type ReservedRelationField = (typeof RESERVED_RELATION_FIELDS)[number] type IsAny = 0 extends 1 & T ? true : false /** - * @description Compile-time tripwire: marks every reserved entity key as - * `never` so an object literal carrying one fails to type-check. Keys that - * `T` itself declares (including via an index signature, where - * `keyof T = string`) are exempted — a consumer who *explicitly* types a - * reserved key into their metadata shape keeps a working (if unwise) type, - * and index-signature metadata types remain assignable. + * @deprecated The compile-time reserved-key ban died with the + * field-addressing law: every name is legal user metadata now. Kept as an + * empty (no-op) guard so external type references keep compiling; it bans + * nothing. */ -export type NoReservedEntityKeys = { - readonly [K in ReservedEntityField as K extends keyof T ? never : K]?: never -} +export type NoReservedEntityKeys = unknown /** - * @description Relationship mirror of {@link NoReservedEntityKeys}. + * @deprecated Relationship mirror of {@link NoReservedEntityKeys} — no-op + * for the same reason. */ -export type NoReservedRelationKeys = { - readonly [K in ReservedRelationField as K extends keyof T ? never : K]?: never -} - -/** - * @description The metadata bag shape for untyped brains (`T = any`): an - * open index signature (any custom key, any value — exactly the pre-8.0 - * latitude) intersected with the reserved-key guard, whose declared - * `?: never` properties take precedence over the index signature so a - * literal reserved key is still a compile error. - */ -type OpenBag = { [key: string]: any } & Guard +export type NoReservedRelationKeys = unknown /** * @description The type of `AddParams.metadata`: the consumer's metadata - * shape `T` with reserved entity keys forbidden at compile time. For untyped - * brains (`T = any`) the bag stays open ({@link OpenBag}), so arbitrary - * custom fields remain legal while literal reserved keys still error. + * shape `T`, open. Under the field-addressing law EVERY key is a legal user + * field (engine scalars are written only via their dedicated params and read + * at `system.*`), so no name is banned at compile time. The one illegal + * spelling — a key starting `'system.'` — cannot be expressed as a mapped + * type ban and is refused at runtime (`rejectForgedSystemKeys`). */ export type EntityMetadataInput = IsAny extends true - ? OpenBag> - : T & NoReservedEntityKeys + ? { [key: string]: any } + : T /** * @description The type of `UpdateParams.metadata`: a partial patch of the - * consumer's metadata shape with reserved entity keys forbidden at compile - * time. Same `T = any` handling as {@link EntityMetadataInput}. + * consumer's metadata shape. Same openness as {@link EntityMetadataInput}. */ export type EntityMetadataPatch = IsAny extends true - ? OpenBag> - : Partial & NoReservedEntityKeys + ? { [key: string]: any } + : Partial /** * @description The type of `RelateParams.metadata`: the consumer's edge - * metadata shape with reserved relationship keys forbidden at compile time. + * metadata shape, open — the relation mirror of {@link EntityMetadataInput}. */ export type RelationMetadataInput = IsAny extends true - ? OpenBag> - : T & NoReservedRelationKeys + ? { [key: string]: any } + : T /** * @description The type of `UpdateRelationParams.metadata`: a partial patch - * of the consumer's edge metadata shape with reserved relationship keys - * forbidden at compile time. + * of the consumer's edge metadata shape, open. */ export type RelationMetadataPatch = IsAny extends true - ? OpenBag> - : Partial & NoReservedRelationKeys + ? { [key: string]: any } + : Partial /** * @description Result of splitting a stored flat metadata record into its @@ -196,6 +201,103 @@ export interface SplitMetadataRecord { const RESERVED_ENTITY_SET: ReadonlySet = new Set(RESERVED_ENTITY_FIELDS) const RESERVED_RELATION_SET: ReadonlySet = new Set(RESERVED_RELATION_FIELDS) +/** + * @description The format-stamp key of a persisted metadata record. Its + * presence with the exact value {@link NESTED_BAG_FORMAT} marks a v2 + * (nested-bag) record; its absence marks a legacy flat record. The stamp is + * what makes the shape check collision-proof against legacy user data: a + * pre-law record COULD carry a user field named `metadata` (the name was + * never reserved), but it cannot also carry this engine-written stamp. + */ +export const METADATA_RECORD_FORMAT_KEY = '_fmt' + +/** + * @description The nested-bag record format stamp (v2, the field-addressing + * law's storage shape, 2026-08-03): engine fields at top level, the user's + * metadata bag NESTED verbatim under `metadata`. Cross-engine: the native + * provider discriminates record shapes by the same stamp. + */ +export const NESTED_BAG_FORMAT = 2 + +/** + * @description `true` when a persisted record carries the v2 nested-bag + * stamp (and a structurally valid nested bag). + */ +export function isNestedBagRecord( + record: Record | null | undefined +): boolean { + return ( + record !== null && + record !== undefined && + typeof record === 'object' && + record[METADATA_RECORD_FORMAT_KEY] === NESTED_BAG_FORMAT && + typeof record.metadata === 'object' && + record.metadata !== null && + !Array.isArray(record.metadata) + ) +} + +/** + * @description Build a v2 (nested-bag) entity metadata record — THE only + * sanctioned way to construct a persisted noun metadata record. The engine + * half goes top-level; the user bag nests verbatim under `metadata`; the + * format stamp seals the shape. Because the two halves never share a level, + * a user field named `confidence` (or any other engine spelling) survives + * flush / reopen / rebuild / time travel exactly as written. + * @param engineFields - The engine-owned half (keys from + * {@link RESERVED_ENTITY_FIELDS} — `noun`, timestamps, `_rev`, …). + * @param userBag - The consumer's metadata bag, stored verbatim. + * @returns The stamped v2 record. + */ +export function buildNounMetadataRecord( + engineFields: Partial>, + userBag: Record | undefined +): Record { + return { + ...engineFields, + [METADATA_RECORD_FORMAT_KEY]: NESTED_BAG_FORMAT, + metadata: { ...(userBag ?? {}) } + } +} + +/** + * @description Build a v2 (nested-bag) relationship metadata record — the + * verb mirror of {@link buildNounMetadataRecord}. + * @param engineFields - The engine-owned half (keys from + * {@link RESERVED_RELATION_FIELDS} — `verb`, `weight`, timestamps, …). + * @param userBag - The consumer's edge metadata bag, stored verbatim. + * @returns The stamped v2 record. + */ +export function buildVerbMetadataRecord( + engineFields: Partial>, + userBag: Record | undefined +): Record { + return { + ...engineFields, + [METADATA_RECORD_FORMAT_KEY]: NESTED_BAG_FORMAT, + metadata: { ...(userBag ?? {}) } + } +} + +/** + * @description Shape-first split of a v2 record: the engine half is the top + * level filtered through the reserved list (belt — the builders only ever + * write reserved names there), the user bag is `record.metadata` verbatim. + */ +function splitNestedRecord( + record: Record, + reservedSet: ReadonlySet +): SplitMetadataRecord { + const reserved: Record = {} + for (const [key, value] of Object.entries(record)) { + if (reservedSet.has(key)) reserved[key] = value + } + return { + reserved: reserved as Partial>, + custom: { ...(record.metadata as Record) } + } +} + /** * @description Shared splitter — partitions a record's keys against a * reserved-name set. `null`/`undefined` records split to two empty objects. @@ -222,33 +324,45 @@ function splitRecord( } /** - * @description Split a stored entity (noun) flat metadata record into - * reserved fields and custom metadata — THE canonical read-side split. Every - * entity read path (live `get()`, batch reads, paginated listings, and - * historical `asOf()` materialization) goes through this function, so the - * reserved list can never drift between read paths. - * @param record - The stored flat metadata record. - * @returns `reserved` (Brainy-owned fields) and `custom` (the consumer's metadata bag). + * @description Split a stored entity (noun) metadata record into engine + * fields and the user's metadata bag — THE canonical read-side split, shape + * aware. v2 (nested-bag) records split by SHAPE: engine half top-level, bag + * = `record.metadata` verbatim (user collider names survive faithfully). + * Legacy flat records split BY NAME through the reserved list — sound for + * them because the pre-law write door refused user metadata carrying those + * names. Every entity read path (live `get()`, batch reads, paginated + * listings, and historical `asOf()` materialization — the generation store + * snapshots whole records) goes through this function, so the two shapes + * can never drift between read paths. + * @param record - The stored metadata record (either shape). + * @returns `reserved` (engine-owned fields) and `custom` (the consumer's metadata bag). * @example * const { reserved, custom } = splitNounMetadataRecord(stored) * // reserved.noun → entity.type, reserved.confidence → entity.confidence, … - * // custom → entity.metadata (custom fields only, always) + * // custom → entity.metadata (the user's fields only, always — ANY names) */ export function splitNounMetadataRecord( record: Record | null | undefined ): SplitMetadataRecord { + if (isNestedBagRecord(record)) { + return splitNestedRecord(record as Record, RESERVED_ENTITY_SET) + } return splitRecord(record, RESERVED_ENTITY_SET) } /** - * @description Split a stored relationship (verb) flat metadata record into - * reserved fields and custom metadata — the verb mirror of - * {@link splitNounMetadataRecord}, used by every relationship read path. - * @param record - The stored flat metadata record. - * @returns `reserved` (Brainy-owned fields) and `custom` (the consumer's metadata bag). + * @description Split a stored relationship (verb) metadata record into + * engine fields and the user's edge metadata bag — the verb mirror of + * {@link splitNounMetadataRecord}, shape aware, used by every relationship + * read path. + * @param record - The stored metadata record (either shape). + * @returns `reserved` (engine-owned fields) and `custom` (the consumer's metadata bag). */ export function splitVerbMetadataRecord( record: Record | null | undefined ): SplitMetadataRecord { + if (isNestedBagRecord(record)) { + return splitNestedRecord(record as Record, RESERVED_RELATION_SET) + } return splitRecord(record, RESERVED_RELATION_SET) } diff --git a/src/utils/metadataIndex.ts b/src/utils/metadataIndex.ts index 6deeb811..f010560d 100644 --- a/src/utils/metadataIndex.ts +++ b/src/utils/metadataIndex.ts @@ -73,8 +73,11 @@ export interface MetadataIndexConfig { maxIndexSize?: number // Max number of entries per field value (default: 10000) rebuildThreshold?: number // Rebuild if index is this % stale (default: 0.1) autoOptimize?: boolean // Auto-cleanup unused entries (default: true) - indexedFields?: string[] // Only index these fields (default: all) - excludeFields?: string[] // Never index these fields + // NOTE: the name-based indexedFields/excludeFields knobs died with the + // field-addressing law ("no special names"): EVERY user field indexes, + // whatever its name. Bulk-payload protection is value-SHAPE based and + // uniform across all names (large arrays never become posting scalars; + // long values index hashed) — shape is not a name carve-out. } export interface MetadataIndexOptions { @@ -185,31 +188,12 @@ export class MetadataIndexManager implements MetadataIndexProvider { this.config = { maxIndexSize: config.maxIndexSize ?? 10000, rebuildThreshold: config.rebuildThreshold ?? 0.1, - autoOptimize: config.autoOptimize ?? true, - indexedFields: config.indexedFields ?? [], - excludeFields: config.excludeFields ?? [ - // ONLY exclude truly un-indexable fields (binary data, large content) - // Timestamps are NOW indexed with automatic bucketing (prevents pollution) - - // Vectors and embeddings (binary data, already have HNSW indexes) - 'embedding', - 'vector', - 'embeddings', - 'vectors', - - // Large content fields (too large for metadata indexing) - 'content', - 'data', - 'originalData', - '_data', - - // Primary keys (use direct lookups instead) - 'id' - - // NOTE: 'accessed', 'modified', 'createdAt', etc. are NO LONGER excluded! - // They are now indexed with automatic 1-minute bucketing to prevent file pollution - // This enables range queries like: modified > yesterday - ] + autoOptimize: config.autoOptimize ?? true + // No name-based exclude/allow lists — the field-addressing law: every + // user field indexes, whatever its name ('content', 'data', 'id', + // 'vector', … included). Bulk payloads are kept out by uniform value- + // SHAPE rules in extractIndexableFields (arrays >10 never become + // posting scalars; >100-char values index hashed), never by name. } // Initialize metadata cache with similar config to search cache @@ -301,7 +285,7 @@ export class MetadataIndexManager implements MetadataIndexProvider { } // Warm the cache with common fields (lazy loading optimization) - // This loads the 'noun' sparse index which is needed for type counts + // This loads the type column ('system.type') needed for type counts await this.warmCache() // Load type counts AFTER warmCache (sparse index is now cached) @@ -350,8 +334,9 @@ export class MetadataIndexManager implements MetadataIndexProvider { * Target: >80% cache hit rate for typical workloads */ async warmCache(): Promise { - // Common fields used in most queries - const commonFields = ['noun', 'type', 'service', 'createdAt'] + // Common columns used in most queries — the frozen system keys, plus + // legacy spellings for a pre-epoch-3 brain read before its rebuild runs. + const commonFields = ['system.type', 'system.service', 'system.createdAt', 'noun'] prodLog.debug(`🔥 Warming metadata cache with common fields: ${commonFields.join(', ')}`) @@ -537,9 +522,11 @@ export class MetadataIndexManager implements MetadataIndexProvider { } /** - * Lazy load entity counts from the 'noun' field sparse index (O(n) where n = number of types) + * Lazy load entity counts from the type column (O(n) where n = number of + * types). The frozen key is 'system.type' (epoch 3); the legacy 'noun' + * column is read as a fallback for a pre-epoch-3 brain observed before its + * rebuild has run (e.g. a reader-mode open against an old writer). * FIX: Previously read from stats.nounCount which was SERVICE-keyed, not TYPE-keyed - * Now computes counts from the sparse index which has the correct type information */ private async lazyLoadCounts(): Promise { try { @@ -549,23 +536,31 @@ export class MetadataIndexManager implements MetadataIndexProvider { this.entityCountsByTypeFixed.fill(0) this.verbCountsByTypeFixed.fill(0) - // PRIMARY (8.0+): rehydrate per-type counts from the column store's 'noun' - // field — the authoritative on-disk source after a cold reopen. + // PRIMARY (8.0+): rehydrate per-type counts from the column store's + // type column — the authoritative on-disk source after a cold reopen. + // Frozen key first ('system.type', epoch 3), legacy 'noun' as the + // pre-rebuild fallback. // // The chunked sparse-index WRITE path was removed in 7.20.0 (commit - // 11be039): new workspaces persist the 'noun' field ONLY to the column - // store, never to a `__sparse_index__noun` blob. So the legacy sparse - // path below finds nothing and leaves every count at 0 — which is exactly - // why counts.byType/byTypeEnum/topTypes/allNounTypeCounts all read empty + // 11be039): new workspaces persist the type column ONLY to the column + // store, never to a sparse-index blob. So the legacy sparse path below + // finds nothing and leaves every count at 0 — which is exactly why + // counts.byType/byTypeEnum/topTypes/allNounTypeCounts all read empty // after close()+reopen while find()/getNounCount() (different sources) // stay correct. The column store's per-value cardinality matches the warm // `updateTypeFieldAffinity` counts EXACTLY because both are driven from the // same `addToIndex` field set, in lockstep, with no visibility gate on // either — so this rehydration reproduces the warm values precisely. - if (this.columnStore && this.columnStore.getIndexedFields().includes('noun')) { - const nounValues = await this.columnStore.getFilterValues('noun') + const indexedCols = this.columnStore ? this.columnStore.getIndexedFields() : [] + const typeCol = indexedCols.includes('system.type') + ? 'system.type' + : indexedCols.includes('noun') + ? 'noun' + : null + if (this.columnStore && typeCol) { + const nounValues = await this.columnStore.getFilterValues(typeCol) for (const value of nounValues) { - const bitmap = await this.columnStore.filter('noun', value) + const bitmap = await this.columnStore.filter(typeCol, value) if (bitmap.size > 0) { // Use the stored value directly as the key (the legacy sparse path // did the same): it is already the normalized type string that @@ -580,16 +575,17 @@ export class MetadataIndexManager implements MetadataIndexProvider { } // LEGACY FALLBACK (pre-7.20.0 workspaces still on the chunked sparse index). - const nounSparseIndex = await this.loadSparseIndex('noun') + const sparseCol = (await this.loadSparseIndex('system.type')) ? 'system.type' : 'noun' + const nounSparseIndex = await this.loadSparseIndex(sparseCol) if (!nounSparseIndex) { - // No column-store 'noun' field and no sparse index yet — counts will be + // No column-store type column and no sparse index yet — counts will be // populated as entities are added. return } // Iterate through all chunks and sum up bitmap sizes by type for (const chunkId of nounSparseIndex.getAllChunkIds()) { - const chunk = await this.chunkManager.loadChunk('noun', chunkId) + const chunk = await this.chunkManager.loadChunk(sparseCol, chunkId) if (chunk) { for (const [type, bitmap] of chunk.entries) { const currentCount = this.totalEntitiesByType.get(type) || 0 @@ -1179,66 +1175,46 @@ export class MetadataIndexManager implements MetadataIndexProvider { return `__HASH_${Math.abs(hash).toString(36)}` } - /** - * Check if field should be indexed - */ - private shouldIndexField(field: string): boolean { - if (this.config.excludeFields.includes(field)) return false - if (this.config.indexedFields.length > 0) { - return this.config.indexedFields.includes(field) - } - return true - } - /** * Extract indexable field-value pairs from entity or metadata * - * Now handles BOTH entity structure (with top-level fields) AND plain metadata - * - Extracts from top-level fields (confidence, weight, timestamps, type, service, etc.) - * - Also extracts from nested metadata field (custom user fields) - * - Skips HNSW-specific fields (vector, connections, level, id) - * - Maps 'type' → 'noun' for backward compatibility with existing indexes - * - * BUG FIX: Exclude vector embeddings and large arrays from indexing - * BUG FIX: Also exclude purely numeric field names (array indices) - * - Vector fields (384+ dimensions) were creating 825K chunk files for 1,144 entities - * - Arrays converted to objects with numeric keys were still being indexed + * Handles BOTH entity structure (with top-level fields) AND record shapes + * - Record-frame system scalars index under literal 'system.' keys + * - The user's metadata bag indexes under bare keys — EVERY name (the + * field-addressing law: no special names; 'level', 'data', 'id', + * 'content', 'vector' in a bag are ordinary user fields) + * - Record-frame plumbing (vector, connections, level, data, _rev, id) + * never indexes — that is namespace routing, not a name carve-out + * - Value-SHAPE rules apply uniformly to all names: arrays >10 never + * become posting scalars; purely numeric key names (array indices) + * skip; >100-char values index hashed (normalizeValue) */ private extractIndexableFields(data: any): Array<{ field: string, value: any }> { const fields: Array<{ field: string, value: any }> = [] - // Fields that should NEVER be indexed: bulk structural payloads that would - // blow up the index (the 384-dim vector, embeddings, the adjacency list). - // These are also caught by the array-size guard below, but naming them is - // belt-and-suspenders. NOTE: `level` was previously here (an HNSW node's - // layer) but it never actually reaches this path — every caller passes a - // metadata bag or Entity record, neither of which carries the node's - // `level` — so its only effect was to silently drop a legitimate USER - // metadata field named `level` (log level, skill level, access level…), - // making `where: { level: … }` return nothing. Removed. (`id` stays: it is - // the reserved entity-identity field, resolved specially by find().) - const NEVER_INDEX = new Set(['vector', 'embedding', 'embeddings', 'connections', 'id']) + // RECORD-FRAME-ONLY plumbing guard: on an entity/stored-record frame + // these keys are the engine's structural payloads (the 384-dim vector, + // embeddings, the adjacency list, the identity field) and never index. + // This set is NEVER applied inside the user's metadata bag — under the + // field-addressing law every user name indexes; a real vector-sized + // value in a bag is kept out by the uniform array-size shape guard, not + // by its name. + const RECORD_PLUMBING = new Set(['vector', 'embedding', 'embeddings', 'connections', 'id']) // THE FROZEN INDEX KEY FORMAT (cross-engine, sealed 2026-08-03; the native // accelerator keys identically — epoch 3 rebuilds every brain onto it): // user fields index under BARE keys exactly as the caller wrote them; // the ten system scalars index under literal 'system.' keys — the // key IS the query address, so the two namespaces can never collide - // inside the index again. `origin` tracks which side of the record a key - // came from: 'record' = the entity/stored-record frame (system scalars, - // plumbing, and the metadata bag live here — the WRITE PATH's reserved- - // name remap guarantees a record-frame key matching a system name IS the - // system value); 'user' = inside the flattened metadata bag (everything - // is the user's, including natural names like `level` and `data`). - // Frame kinds: 'entity-record' = entityForIndexing shape (user fields - // nested under `metadata`; stray top-level keys are DROPPED, not guessed — - // epoch-3's rebuild-from-canonical normalizes historical shapes); - // 'flat-record' = the stored metadata-record shape (user fields FLAT - // beside the reserved ones — the write path's reserved-name remap - // guarantees a key matching a system name IS the system value, so - // non-system keys here are the user's and index bare); 'user' = inside - // the metadata bag (everything is the user's, including natural names - // like `level` and `data`). + // inside the index again. + // Frame kinds: 'entity-record' = entityForIndexing shape / v2 nested-bag + // stored record (user fields nested under `metadata`; stray top-level + // keys are DROPPED, not guessed); 'flat-record' = the LEGACY stored + // metadata-record shape (user fields flat beside the engine's — sound to + // split by name because the pre-law write door refused user metadata + // carrying engine names, so a flat key matching a system name IS the + // system value); 'user' = inside the metadata bag, where EVERY key is + // the user's and indexes bare — collider names included. type Frame = 'entity-record' | 'flat-record' | 'user' const extract = (obj: any, prefix = '', frame: Frame = 'entity-record'): void => { for (const [key, value] of Object.entries(obj)) { @@ -1254,30 +1230,25 @@ export class MetadataIndexManager implements MetadataIndexProvider { } else if (SYSTEM_ENTITY_SCALARS.has(key) && key !== 'id') { fullKey = `system.${key}` } else if ( - key === 'data' || key === '_rev' || key === 'level' || NEVER_INDEX.has(key) + key === 'data' || key === '_rev' || key === 'level' || key === '_fmt' || + RECORD_PLUMBING.has(key) ) { - continue // plumbing / identity / bulk payloads — never indexed from a record frame + continue // plumbing / identity / format stamp — never indexed from a record frame } else if (frame === 'entity-record') { continue // stray entity-frame key: dropped, not guessed } // flat-record fallthrough: a non-system, non-plumbing key IS a user - // field (flat beside the reserved ones) — indexes bare via fullKey. - } else if (!prefix && NEVER_INDEX.has(key)) { - // User frame: only the bulk-payload guards apply — natural names - // like `level` and `data` are real user fields here. (`id` as a - // user metadata field remains un-indexed this train — documented - // limitation; system.id resolves via the id mapper, never a column.) - continue + // field (flat beside the engine's, legacy shape) — indexes bare. } + // User frame: NO name-based skips — every user field indexes, whatever + // its name (the field-addressing law). Only the uniform value-shape + // guards below apply. // Skip purely numeric field names (array indices converted to object keys) // Legitimate field names should never be purely numeric // This catches vectors stored as objects: {0: 0.1, 1: 0.2, ...} if (/^\d+$/.test(key)) continue - // Skip fields based on user configuration - if (!this.shouldIndexField(fullKey)) continue - // Skip large arrays (> 10 elements) - likely vectors or bulk data if (Array.isArray(value) && value.length > 10) continue @@ -1510,10 +1481,11 @@ export class MetadataIndexManager implements MetadataIndexProvider { prodLog.debug(`Entity ${id} has ${wordFields.length} indexed words (large document)`) } - // Sort fields to process 'noun' field first for type-field affinity tracking + // Sort fields to process the type column first for type-field affinity + // tracking ('system.type' is the frozen key; 'noun' died at epoch 3). fields.sort((a, b) => { - if (a.field === 'noun') return -1 - if (b.field === 'noun') return 1 + if (a.field === 'system.type') return -1 + if (b.field === 'system.type') return 1 return 0 }) @@ -2861,6 +2833,17 @@ export class MetadataIndexManager implements MetadataIndexProvider { // VFS Statistics Methods (uses existing Roaring bitmap infrastructure) // ============================================================================ + /** + * Read the type column's bitmap for one type value — frozen key first + * ('system.type', epoch 3), legacy 'noun' as the pre-rebuild fallback. + */ + private async getTypeBitmap(type: string): Promise { + return ( + (await this.getBitmapFromChunks('system.type', type)) ?? + (await this.getBitmapFromChunks('noun', type)) + ) + } + /** * Get VFS entity count for a specific type using Roaring bitmap intersection * Uses hardware-accelerated SIMD operations (AVX2/SSE4.2) @@ -2869,7 +2852,7 @@ export class MetadataIndexManager implements MetadataIndexProvider { */ async getVFSEntityCountByType(type: string): Promise { const vfsBitmap = await this.getBitmapFromChunks('isVFSEntity', true) - const typeBitmap = await this.getBitmapFromChunks('noun', type) + const typeBitmap = await this.getTypeBitmap(type) if (!vfsBitmap || !typeBitmap) return 0 @@ -2892,7 +2875,7 @@ export class MetadataIndexManager implements MetadataIndexProvider { // Iterate through all known types and compute VFS count via intersection for (const type of this.totalEntitiesByType.keys()) { - const typeBitmap = await this.getBitmapFromChunks('noun', type) + const typeBitmap = await this.getTypeBitmap(type) if (typeBitmap) { const intersection = RoaringBitmap32.and(vfsBitmap, typeBitmap) if (intersection.size > 0) { @@ -3486,18 +3469,21 @@ export class MetadataIndexManager implements MetadataIndexProvider { * Tracks which fields commonly appear with which entity types */ private updateTypeFieldAffinity(entityId: string, field: string, value: any, operation: 'add' | 'remove', metadata?: any): void { - // Only track affinity for non-system fields (but allow 'noun' for type detection) - if (this.config.excludeFields.includes(field) && field !== 'noun') return + // Only track affinity for user fields (plus the type column itself, + // which drives detection). Engine columns carry the literal 'system.' + // prefix under the frozen key format. + if (field.startsWith('system.') && field !== 'system.type') return - // For the 'noun' field, the value IS the entity type + // For the type column ('system.type'), the value IS the entity type let entityType: string | null = null - if (field === 'noun') { + if (field === 'system.type') { // This is the type definition itself entityType = this.normalizeValue(value, field) // Pass field for bucketing! - } else if (metadata && metadata.noun) { - // Extract entity type from metadata - entityType = this.normalizeValue(metadata.noun, 'noun') + } else if (metadata && (metadata.noun ?? metadata.type)) { + // Extract entity type from the source shape: stored records carry it + // under 'noun', entity-for-indexing views under 'type'. + entityType = this.normalizeValue(metadata.noun ?? metadata.type, 'system.type') } else { // No type information available, skip affinity tracking return @@ -3520,8 +3506,9 @@ export class MetadataIndexManager implements MetadataIndexProvider { const currentCount = typeFields.get(field) || 0 typeFields.set(field, currentCount + 1) - // Update total entities of this type (only count once per entity) - if (field === 'noun') { + // Update total entities of this type (only count once per entity — + // the type column appears exactly once per entity) + if (field === 'system.type') { const newCount = this.totalEntitiesByType.get(entityType)! + 1 this.totalEntitiesByType.set(entityType, newCount) @@ -3544,7 +3531,7 @@ export class MetadataIndexManager implements MetadataIndexProvider { } // Update total entities of this type - if (field === 'noun') { + if (field === 'system.type') { const total = this.totalEntitiesByType.get(entityType)! if (total > 1) { const newCount = total - 1 diff --git a/src/utils/paramValidation.ts b/src/utils/paramValidation.ts index 749849b3..359413d7 100644 --- a/src/utils/paramValidation.ts +++ b/src/utils/paramValidation.ts @@ -618,6 +618,7 @@ export function validateUpdateParams(params: UpdateParams): void { * Validate relate parameters */ export function validateRelateParams(params: RelateParams): void { + rejectForgedSystemKeys(params.metadata as Record | undefined, 'relate()') // 8.0 verb-id contract (L.7): verb ids are UUIDs, generated by brainy. // RelateParams has no `id` field — an untyped caller passing one would // previously have it silently ignored (a generated UUID was used instead). @@ -666,6 +667,7 @@ export function validateRelateParams(params: RelateParams): void { * accepts type/subtype/weight/confidence/data/metadata changes. */ export function validateUpdateRelationParams(params: UpdateRelationParams): void { + rejectForgedSystemKeys(params.metadata as Record | undefined, 'updateRelation()') if (!params.id) { throw new Error('id is required for updateRelation') } diff --git a/tests/conformance/collider-fidelity.test.ts b/tests/conformance/collider-fidelity.test.ts new file mode 100644 index 00000000..61c9413d --- /dev/null +++ b/tests/conformance/collider-fidelity.test.ts @@ -0,0 +1,307 @@ +/** + * @module tests/conformance/collider-fidelity + * @description THE REOPEN-COLLIDER CONFORMANCE CASE (required cross-engine + * before any RC counts as gates-green — ruled 2026-08-03). The + * field-addressing law's fidelity half: user metadata may carry ANY name — + * including every engine spelling (`confidence`, `type`, `id`, `createdAt`, + * …) and every plumbing name (`level`, `data`, `vector`, `_rev`) — and the + * value survives, verbatim and reachable, across the FULL lifecycle: live + * reads, where/orderBy, flush, close+reopen, a forced epoch rebuild, and + * time travel. The engine scalars stay separately reachable at `system.*` + * the whole way. No halfway states. + * + * Self-arming like the namespace-law suite: skips loudly until the arming + * exports are present, so the suite can sit on a branch ahead of the build. + */ +import { describe, it, expect, beforeAll, afterAll } from 'vitest' +import { mkdtempSync, rmSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import * as brainyExports from '../../src/index.js' +import { Brainy, NounType, VerbType } from '../../src/index.js' +import { + BRAIN_FORMAT_PATH, + EXPECTED_INDEX_EPOCH +} from '../../src/storage/brainFormat.js' + +const ARMED = 'UnresolvableFieldError' in brainyExports +const suite = ARMED ? describe : describe.skip +if (!ARMED) { + // eslint-disable-next-line no-console + console.warn( + '[collider-fidelity] SKIPPING: package root does not export the ' + + 'field-addressing law surface yet (UnresolvableFieldError absent).' + ) +} + +/** Every entity system scalar name written as a USER metadata field, with + * unmistakable user values, plus the plumbing names and naturals. */ +const COLLIDER_BAG = { + // the ten entity system scalars, as user fields + id: 'user-id', + type: 'user-type', + subtype: 'user-subtype', + createdAt: 'user-createdAt', + updatedAt: 'user-updatedAt', + confidence: 'user-confidence', + weight: 'user-weight', + visibility: 'user-visibility', + service: 'user-service', + createdBy: 'user-createdBy', + // plumbing names, as user fields + level: 7, + data: 'user-data', + vector: 'user-vector', + _rev: 'user-rev', + // naturals previously silently un-indexed by name + content: 'user-content', + // a plain control field + plain: 'control' +} as const + + +suite('collider fidelity — the reopen-collider case (both suites, ruled)', () => { + let dir: string + let brain: Brainy + let colliderId: string + + const open = async (): Promise => { + const b = new Brainy({ + storage: { type: 'filesystem', path: dir }, + requireSubtype: false + }) + await b.init() + return b + } + + /** The full read battery — run at every lifecycle boundary. */ + const verifyColliderTruth = async (label: string): Promise => { + // 1. get(): the bag comes back verbatim; engine scalars stay engine. + const entity = await brain.get(colliderId) + expect(entity, `${label}: entity readable`).toBeTruthy() + for (const [k, v] of Object.entries(COLLIDER_BAG)) { + expect( + (entity!.metadata as Record)[k], + `${label}: bag.${k} verbatim` + ).toEqual(v) + } + expect(entity!.type, `${label}: engine type intact`).toBe(NounType.Document) + expect(entity!.confidence, `${label}: engine confidence intact`).toBe(0.25) + + // 2. where on collider names (bare = the user's field, always). + for (const [k, v] of [ + ['confidence', 'user-confidence'], + ['type', 'user-type'], + ['id', 'user-id'], + ['content', 'user-content'], + ['data', 'user-data'], + ['level', 7] + ] as const) { + const rows = await brain.find({ where: { [k]: v }, limit: 10 }) + expect( + rows.map((r) => r.id), + `${label}: where {${k}} finds the collider row` + ).toContain(colliderId) + } + + // 3. system.* keeps reading the ENGINE values. + const byEngine = await brain.find({ + where: { 'system.confidence': 0.25 }, + limit: 10 + }) + expect( + byEngine.map((r) => r.id), + `${label}: system.confidence reads the engine scalar` + ).toContain(colliderId) + const byUserSpelledSystem = await brain.find({ + where: { 'system.confidence': 'user-confidence' }, + limit: 10 + }) + expect( + byUserSpelledSystem.map((r) => r.id), + `${label}: the user's value is NOT reachable via system.*` + ).not.toContain(colliderId) + + // 4. orderBy a collider name orders by the USER values. + const ordered = await brain.find({ + type: NounType.Document, + orderBy: 'level', + order: 'desc', + limit: 10 + }) + expect(ordered.length, `${label}: ordered read complete`).toBe(3) + expect( + (ordered[0].metadata as Record).plain, + `${label}: user level orders desc (7 first)` + ).toBe('control') + } + + beforeAll(async () => { + dir = mkdtempSync(join(tmpdir(), 'brainy-collider-')) + brain = await open() + + colliderId = await brain.add({ + data: 'the collider probe document', + type: NounType.Document, + confidence: 0.25, + metadata: { ...COLLIDER_BAG } + }) + // two ordering companions with smaller user `level`s + await brain.add({ + data: 'ordering companion low', + type: NounType.Document, + metadata: { level: 3, plain: 'low' } + }) + await brain.add({ + data: 'ordering companion mid', + type: NounType.Document, + metadata: { level: 5, plain: 'mid' } + }) + }, 120000) + + afterAll(async () => { + await brain.close().catch(() => {}) + rmSync(dir, { recursive: true, force: true }) + }) + + it('LIVE: colliders are the user’s, verbatim and fully queryable', async () => { + await verifyColliderTruth('live') + }) + + it('REOPEN: the restart boundary loses nothing', async () => { + await brain.flush() + await brain.close() + brain = await open() + await verifyColliderTruth('reopen') + }) + + it('REBUILD: a forced epoch rebuild re-indexes the colliders from canonical', async () => { + await brain.close() + // Simulate epoch drift: a missing marker forces the full derived-index + // rebuild at open — the exact path every pre-law brain takes once. + rmSync(join(dir, BRAIN_FORMAT_PATH), { force: true }) + brain = await open() + await verifyColliderTruth('rebuild') + // And the rebuild re-stamps the current epoch. + const marker = await ( + brain as unknown as { + storage: { readRawObject(p: string): Promise<{ indexEpoch?: number } | null> } + } + ).storage.readRawObject(BRAIN_FORMAT_PATH) + expect(marker?.indexEpoch).toBe(EXPECTED_INDEX_EPOCH) + }) + + it('TIME TRAVEL: asOf reads historical collider values faithfully', async () => { + const gen = brain.generation() + await brain.update({ id: colliderId, metadata: { confidence: 'user-confidence-v2' } }) + const now = await brain.get(colliderId) + expect((now!.metadata as Record).confidence).toBe('user-confidence-v2') + + const past = await brain.asOf(gen) + try { + const then = await past.get(colliderId) + expect( + (then!.metadata as Record).confidence, + 'asOf reads the pre-update USER value' + ).toBe('user-confidence') + } finally { + await past.release() + } + // engine scalar untouched throughout + expect(now!.confidence).toBe(0.25) + }) + + it('RELATION MIRROR: edge collider bags survive write → read → reopen', async () => { + const a = await brain.add({ data: 'edge endpoint a', type: NounType.Person, metadata: { plain: 'a' } }) + const b = await brain.add({ data: 'edge endpoint b', type: NounType.Person, metadata: { plain: 'b' } }) + const edgeBag = { + verb: 'user-verb', + confidence: 'user-edge-confidence', + weight: 'user-edge-weight', + subtype: 'user-edge-subtype', + createdAt: 'user-edge-createdAt', + service: 'user-edge-service' + } + const relId = await brain.relate({ + from: a, + to: b, + type: VerbType.RelatedTo, + confidence: 0.5, + metadata: { ...edgeBag } + }) + + const check = async (label: string): Promise => { + const rels = await brain.related({ from: a, type: VerbType.RelatedTo }) + const rel = rels.find((r) => r.id === relId) + expect(rel, `${label}: relation readable`).toBeTruthy() + for (const [k, v] of Object.entries(edgeBag)) { + expect( + (rel!.metadata as Record)[k], + `${label}: edge bag.${k} verbatim` + ).toEqual(v) + } + expect(rel!.confidence, `${label}: engine edge confidence intact`).toBe(0.5) + expect(rel!.type, `${label}: engine verb intact`).toBe(VerbType.RelatedTo) + } + + await check('live') + await brain.flush() + await brain.close() + brain = await open() + await check('reopen') + }) + + it('FORGERY: user metadata keys spelled system.* refuse at every write door', async () => { + await expect( + brain.add({ data: 'forged', type: NounType.Document, metadata: { 'system.confidence': 1 } }) + ).rejects.toThrow(/system\./) + await expect( + brain.update({ id: colliderId, metadata: { 'system.type': 'x' } }) + ).rejects.toThrow(/system\./) + const a = await brain.add({ data: 'forgery endpoint a', type: NounType.Person, metadata: {} }) + const b = await brain.add({ data: 'forgery endpoint b', type: NounType.Person, metadata: {} }) + await expect( + brain.relate({ from: a, to: b, type: VerbType.RelatedTo, metadata: { 'system.verb': 'x' } }) + ).rejects.toThrow(/system\./) + }) + + it('CONFIG: the dead reservedFieldPolicy option refuses loudly, never ignored', () => { + expect( + () => new Brainy({ storage: { type: 'memory' }, reservedFieldPolicy: 'throw' } as never) + ).toThrow(/field-addressing law/) + }) + + it('LEGACY: a pre-law flat record still reads with engine fields top-level', async () => { + const storage = ( + brain as unknown as { + storage: { + saveNoun(n: unknown): Promise + saveNounMetadata(id: string, m: Record): Promise + } + } + ).storage + const legacyId = '00000000-0000-4000-8000-00000000f1a7' + await storage.saveNoun({ id: legacyId, vector: new Array(384).fill(0.01), connections: new Map(), level: 0 }) + // Legacy FLAT shape: engine + user keys mixed at one level, NO _fmt stamp. + // Sound to split by name — the pre-law door refused user colliders. + await storage.saveNounMetadata(legacyId, { + noun: NounType.Document, + confidence: 0.75, + createdAt: 1700000000000, + updatedAt: 1700000000000, + _rev: 1, + legacyField: 'legacy-value' + }) + const entity = await brain.get(legacyId) + expect(entity).toBeTruthy() + expect(entity!.confidence, 'legacy flat confidence = engine').toBe(0.75) + expect( + (entity!.metadata as Record).legacyField, + 'legacy custom field = user bag' + ).toBe('legacy-value') + expect( + (entity!.metadata as Record).confidence, + 'legacy flat engine key never leaks into the bag' + ).toBeUndefined() + }) +}) diff --git a/tests/integration/advanced-apis-regression.test.ts b/tests/integration/advanced-apis-regression.test.ts index 12c39112..069d3e62 100644 --- a/tests/integration/advanced-apis-regression.test.ts +++ b/tests/integration/advanced-apis-regression.test.ts @@ -164,19 +164,19 @@ describe('BR-ADV-FEATURES-BUN regression', () => { await b.close() }) - it('groupBy "noun" resolves to the entity type, not null', async () => { + it('groupBy "system.type" resolves to the entity type, not null (the legacy "noun" alias is dead)', async () => { const b: any = new Brainy({ requireSubtype: false, storage: { type: 'memory' } }) await b.init() await b.add({ data: 'p', type: NounType.Person }) b.defineAggregate({ name: 'byNoun', source: { type: NounType.Person }, - groupBy: ['noun'], + groupBy: ['system.type'], metrics: { count: { op: 'count' } } }) const rows: any[] = await b.find({ aggregate: 'byNoun' }) expect(rows.length).toBe(1) - expect(rows[0].groupKey.noun).toBe(NounType.Person) + expect(rows[0].groupKey['system.type']).toBe(NounType.Person) await b.close() }) }) diff --git a/tests/integration/aggregate-reserved-fields.test.ts b/tests/integration/aggregate-reserved-fields.test.ts index e81692d6..b8c11b4f 100644 --- a/tests/integration/aggregate-reserved-fields.test.ts +++ b/tests/integration/aggregate-reserved-fields.test.ts @@ -42,8 +42,11 @@ describe('aggregation + query field-resolution law', () => { it('reserved-field groupBy decrements on delete (the drift bug)', async () => { brain.defineAggregate({ name: 'by_subtype', + // system.subtype — subtype is an add() param (an engine scalar), never + // a user metadata field; bare 'subtype' now addresses the user's own + // metadata bag under the sealed field-addressing law. source: { type: NounType.Document }, - groupBy: ['subtype'], + groupBy: ['system.subtype'], metrics: { count: { op: 'count' } } }) @@ -60,7 +63,7 @@ describe('aggregation + query field-resolution law', () => { } let groups = await brain.queryAggregate('by_subtype') expect(groups).toHaveLength(1) - expect(groups[0].groupKey).toEqual({ subtype: 'note' }) + expect(groups[0].groupKey).toEqual({ 'system.subtype': 'note' }) expect(groups[0].metrics.count).toBe(5) await brain.remove(ids[0]) @@ -76,7 +79,7 @@ describe('aggregation + query field-resolution law', () => { brain.defineAggregate({ name: 'by_subtype', source: { type: NounType.Document }, - groupBy: ['subtype'], + groupBy: ['system.subtype'], metrics: { count: { op: 'count' } } }) const id = await brain.add({ @@ -88,7 +91,7 @@ describe('aggregation + query field-resolution law', () => { const groups = await brain.queryAggregate('by_subtype') const byKey = Object.fromEntries( - groups.map((g) => [String(g.groupKey.subtype), g.metrics.count]) + groups.map((g) => [String(g.groupKey['system.subtype']), g.metrics.count]) ) expect(byKey['published']).toBe(1) // The old group must be gone or zero — never still counting the entity. @@ -98,7 +101,7 @@ describe('aggregation + query field-resolution law', () => { it('source.where on a reserved field filters instead of matching nothing', async () => { brain.defineAggregate({ name: 'notes_only', - source: { type: NounType.Document, where: { subtype: 'note' } }, + source: { type: NounType.Document, where: { 'system.subtype': 'note' } }, groupBy: ['team'], metrics: { count: { op: 'count' } } }) diff --git a/tests/integration/all-apis-comprehensive.test.ts b/tests/integration/all-apis-comprehensive.test.ts index d25f23c8..7d82afea 100644 --- a/tests/integration/all-apis-comprehensive.test.ts +++ b/tests/integration/all-apis-comprehensive.test.ts @@ -331,8 +331,10 @@ describe('Comprehensive All-APIs Test', () => { it('should handle metadata queries efficiently', async () => { const start = Date.now() + // system.type — the legacy where.type→noun alias is dead; bare 'type' + // in where now addresses the user's own metadata field. const results = await brain.find({ - where: { type: NounType.Document }, + where: { 'system.type': NounType.Document }, limit: 100 }) diff --git a/tests/integration/fact-log-dual-write.test.ts b/tests/integration/fact-log-dual-write.test.ts index 5ec66273..3c4eee4a 100644 --- a/tests/integration/fact-log-dual-write.test.ts +++ b/tests/integration/fact-log-dual-write.test.ts @@ -11,7 +11,7 @@ import { describe, it, expect, beforeEach, afterEach } from 'vitest' import * as fs from 'node:fs' import * as os from 'node:os' import * as path from 'node:path' -import { Brainy, ProtectedArtifactError, type CommitFact } from '../../src/index.js' +import { Brainy, ProtectedArtifactError, splitNounMetadataRecord, type CommitFact } from '../../src/index.js' async function allFacts(brain: any): Promise { const scan = brain.scanFacts() @@ -65,7 +65,13 @@ describe('fact log dual-write (memory adapter)', () => { const updateFact = facts[facts.length - 1] const op = updateFact.ops.find((o) => o.id === id)! expect(op.record).not.toBeNull() - expect((op.record!.metadata as any).v).toBe('new') + // The fact log is byte-faithful: op.record.metadata is the RAW stored + // record (v2 nested-bag since the field-addressing law) — read the user + // field through the shape-aware split, like every other reader. + const { custom } = splitNounMetadataRecord( + op.record!.metadata as Record + ) + expect(custom.v).toBe('new') }) it('a transact commits ONE fact carrying all its ops, with meta', async () => { diff --git a/tests/integration/lens-consistency.test.ts b/tests/integration/lens-consistency.test.ts index 64484a38..1b1cef81 100644 --- a/tests/integration/lens-consistency.test.ts +++ b/tests/integration/lens-consistency.test.ts @@ -2,8 +2,8 @@ * @module tests/integration/lens-consistency * @description The three metadata "lenses" over one corpus must agree with * canonical ground truth id-for-id, warm AND after a cold reopen: - * - combined: find({ type: T, where: { subtype: S } }) - * - subtype-only: find({ where: { subtype: S } }) + * - combined: find({ type: T, where: { 'system.subtype': S } }) + * - subtype-only: find({ where: { 'system.subtype': S } }) * - type-only: find({ type: T }) * Ported from the fresh-brain probe that closed the type+subtype lens-drop * investigation (a restored pre-8.2.2 torn capture had entities visible to the @@ -63,8 +63,11 @@ async function assertAllLenses(brain: any): Promise { const subtypes = [...new Set(CORPUS.map((c) => c.subtype))] for (const { type, subtype } of CORPUS) { - const combined = idSet(await brain.find({ type, where: { subtype }, limit: 1000 })) - const subtypeOnly = idSet(await brain.find({ where: { subtype }, limit: 1000 })) + // system.subtype — subtype is an add()/update() param (an engine scalar), + // never a user metadata field; bare 'subtype' now addresses the user's + // own metadata bag under the sealed field-addressing law. + const combined = idSet(await brain.find({ type, where: { 'system.subtype': subtype }, limit: 1000 })) + const subtypeOnly = idSet(await brain.find({ where: { 'system.subtype': subtype }, limit: 1000 })) const truthPair = await groundTruth(brain, { type, subtype }) const truthSubtype = await groundTruth(brain, { subtype }) @@ -82,7 +85,7 @@ async function assertAllLenses(brain: any): Promise { // Count cross-check against the corpus definition itself. for (const subtype of subtypes) { const expected = CORPUS.filter((c) => c.subtype === subtype).reduce((s, c) => s + c.count, 0) - const got = (await brain.find({ where: { subtype }, limit: 1000 })).length + const got = (await brain.find({ where: { 'system.subtype': subtype }, limit: 1000 })).length expect(got).toBe(expected) } } @@ -123,16 +126,16 @@ describe('lens consistency — combined vs subtype-only vs canonical ground trut it('after an update() flips type AND subtype, every lens tracks the move exactly', async () => { // The historical cross-bucket-staleness path: change (concept, action) -> (task, review). - const victims = await brain.find({ type: 'concept', where: { subtype: 'action' }, limit: 1 }) + const victims = await brain.find({ type: 'concept', where: { 'system.subtype': 'action' }, limit: 1 }) expect(victims.length).toBe(1) const id = victims[0].id await brain.update({ id, type: 'task', subtype: 'review' }) - const oldCombined = idSet(await brain.find({ type: 'concept', where: { subtype: 'action' }, limit: 1000 })) + const oldCombined = idSet(await brain.find({ type: 'concept', where: { 'system.subtype': 'action' }, limit: 1000 })) expect(oldCombined.has(id)).toBe(false) // unposted from the old buckets - const newCombined = idSet(await brain.find({ type: 'task', where: { subtype: 'review' }, limit: 1000 })) + const newCombined = idSet(await brain.find({ type: 'task', where: { 'system.subtype': 'review' }, limit: 1000 })) expect(newCombined.has(id)).toBe(true) // posted to the new buckets - const subtypeOnly = idSet(await brain.find({ where: { subtype: 'review' }, limit: 1000 })) + const subtypeOnly = idSet(await brain.find({ where: { 'system.subtype': 'review' }, limit: 1000 })) expect(subtypeOnly.has(id)).toBe(true) }) }) diff --git a/tests/integration/migration.test.ts b/tests/integration/migration.test.ts index daa5fb2d..f6c3741b 100644 --- a/tests/integration/migration.test.ts +++ b/tests/integration/migration.test.ts @@ -20,6 +20,16 @@ import { MigrationRunner, MIGRATIONS } from '../../src/migration/index.js' import type { Migration } from '../../src/migration/index.js' import { NounType, VerbType } from '../../src/types/graphTypes.js' +// THE VIEW CONTRACT (field-addressing law): transforms receive engine fields +// top-level and the USER's bag nested under `metadata` — user-field changes +// go inside the bag. These two helpers keep the one-liner migrations tidy. +const bagOf = (m: Record): Record => + m.metadata as Record +const withBag = ( + m: Record, + patch: Record +): Record => ({ ...m, metadata: { ...bagOf(m), ...patch } }) + // Helper to temporarily inject migrations into the MIGRATIONS array function withMigrations(migrations: Migration[], fn: () => Promise): Promise { const original = MIGRATIONS.splice(0, MIGRATIONS.length) @@ -78,9 +88,11 @@ describe('Migration System', () => { description: 'Add version field to entities with status', applies: 'nouns', transform: (m) => { - // Only transform entities that have our specific 'status' field - if ('status' in m && !('version' in m)) { - return { ...m, version: 1 } + // Only transform entities that have our specific 'status' USER field + // (user fields live in the nested bag — the view contract). + const bag = m.metadata as Record + if ('status' in bag && !('version' in bag)) { + return { ...m, metadata: { ...bag, version: 1 } } } return null } @@ -94,7 +106,8 @@ describe('Migration System', () => { // All 3 entities have 'status' metadata expect(p.affectedEntities).toBeGreaterThanOrEqual(3) expect(p.sampleChanges.length).toBeGreaterThan(0) - expect(p.sampleChanges[0].after.version).toBe(1) + // Samples carry the VIEW shape: user fields inside `.metadata`. + expect(p.sampleChanges[0].after.metadata.version).toBe(1) // Verify no data was modified (dry-run) const entity = await brain.get(id1) @@ -111,9 +124,10 @@ describe('Migration System', () => { description: 'Rename state to status', applies: 'nouns', transform: (m) => { - if ('state' in m) { - const { state, ...rest } = m - return { ...rest, status: state } + const bag = m.metadata as Record + if ('state' in bag) { + const { state, ...rest } = bag + return { ...m, metadata: { ...rest, status: state } } } return null } @@ -124,11 +138,12 @@ describe('Migration System', () => { const p = preview as any expect(p.sampleChanges.length).toBeGreaterThanOrEqual(1) - // Find the sample for our entity (it has the 'state' field) - const sample = p.sampleChanges.find((s: any) => s.before.state === 'draft') + // Find the sample for our entity (it has the 'state' USER field — + // samples carry the VIEW shape, user fields inside `.metadata`) + const sample = p.sampleChanges.find((s: any) => s.before.metadata.state === 'draft') expect(sample).toBeDefined() - expect(sample.after.status).toBe('draft') - expect(sample.after.state).toBeUndefined() + expect(sample.after.metadata.status).toBe('draft') + expect(sample.after.metadata.state).toBeUndefined() }) }) }) @@ -149,8 +164,8 @@ describe('Migration System', () => { description: 'Add migrated flag to entities with priority', applies: 'nouns', transform: (m) => { - if ('priority' in m && !('migrated' in m)) { - return { ...m, migrated: true } + if ('priority' in bagOf(m) && !('migrated' in bagOf(m))) { + return withBag(m, { migrated: true }) } return null } @@ -179,8 +194,8 @@ describe('Migration System', () => { description: 'Uppercase status field only when present', applies: 'nouns', transform: (m) => { - if (typeof m.status === 'string') { - return { ...m, status: (m.status as string).toUpperCase() } + if (typeof bagOf(m).status === 'string') { + return withBag(m, { status: (bagOf(m).status as string).toUpperCase() }) } return null } @@ -203,7 +218,7 @@ describe('Migration System', () => { version: '1.0.0', description: 'Double count', applies: 'nouns', - transform: (m) => typeof m.count === 'number' ? { ...m, count: (m.count as number) * 2 } : null + transform: (m) => typeof bagOf(m).count === 'number' ? withBag(m, { count: (bagOf(m).count as number) * 2 }) : null } const migration2: Migration = { @@ -211,7 +226,7 @@ describe('Migration System', () => { version: '1.1.0', description: 'Add 10 to count', applies: 'nouns', - transform: (m) => typeof m.count === 'number' ? { ...m, count: (m.count as number) + 10 } : null + transform: (m) => typeof bagOf(m).count === 'number' ? withBag(m, { count: (bagOf(m).count as number) + 10 }) : null } await withMigrations([migration1, migration2], async () => { @@ -229,7 +244,7 @@ describe('Migration System', () => { version: '1.0.0', description: 'Increment v', applies: 'nouns', - transform: (m) => typeof m.v === 'number' ? { ...m, v: (m.v as number) + 1 } : null + transform: (m) => typeof bagOf(m).v === 'number' ? withBag(m, { v: (bagOf(m).v as number) + 1 }) : null } await withMigrations([migration], async () => { @@ -266,7 +281,7 @@ describe('Migration System', () => { version: '2.0.0', description: 'Add y field to entities with x', applies: 'nouns', - transform: (m) => 'x' in m && !('y' in m) ? { ...m, y: 2 } : null + transform: (m) => 'x' in bagOf(m) && !('y' in bagOf(m)) ? withBag(m, { y: 2 }) : null } await withMigrations([migration], async () => { @@ -290,8 +305,8 @@ describe('Migration System', () => { description: 'Replace original with migrated', applies: 'nouns', transform: (m) => { - if (m.original === true) { - return { ...m, original: false, migrated: true } + if (bagOf(m).original === true) { + return withBag(m, { original: false, migrated: true }) } return null } @@ -323,7 +338,7 @@ describe('Migration System', () => { version: '4.0.0', description: 'Add field', applies: 'nouns', - transform: (m) => 'q' in m && !('r' in m) ? { ...m, r: 2 } : null + transform: (m) => 'q' in bagOf(m) && !('r' in bagOf(m)) ? withBag(m, { r: 2 }) : null } await withMigrations([migration], async () => { @@ -384,7 +399,7 @@ describe('Migration System', () => { version: '1.0.0', description: 'Auto migrate test', applies: 'nouns', - transform: (m) => 'legacy' in m ? { ...m, legacy: false, upgraded: true } : null + transform: (m) => 'legacy' in bagOf(m) ? withBag(m, { legacy: false, upgraded: true }) : null } await withMigrations([migration], async () => { @@ -410,7 +425,7 @@ describe('Migration System', () => { version: '1.0.0', description: 'Add y to entities with x', applies: 'nouns', - transform: (m) => 'x' in m ? { ...m, y: true } : null + transform: (m) => 'x' in bagOf(m) ? withBag(m, { y: true }) : null } const progressCalls: any[] = [] @@ -444,7 +459,7 @@ describe('Migration System', () => { version: '1.0.0', description: 'Increment v on entities that have it', applies: 'nouns', - transform: (m) => typeof m.v === 'number' ? { ...m, v: (m.v as number) + 1 } : null + transform: (m) => typeof bagOf(m).v === 'number' ? withBag(m, { v: (bagOf(m).v as number) + 1 }) : null } await withMigrations([migration], async () => { @@ -477,9 +492,10 @@ describe('Migration System', () => { description: 'Rename strength to intensity', applies: 'verbs', transform: (m) => { - if ('strength' in m) { - const { strength, ...rest } = m - return { ...rest, intensity: strength } + const bag = bagOf(m) + if ('strength' in bag) { + const { strength, ...rest } = bag + return { ...m, metadata: { ...rest, intensity: strength } } } return null } @@ -507,7 +523,7 @@ describe('Migration System', () => { version: '1.0.0', description: 'Update tag from old to new', applies: 'both', - transform: (m) => m.tag === 'old' ? { ...m, tag: 'new' } : null + transform: (m) => bagOf(m).tag === 'old' ? withBag(m, { tag: 'new' }) : null } await withMigrations([migration], async () => { @@ -577,11 +593,11 @@ describe('Migration System', () => { description: 'Transform that throws on non-number values', applies: 'nouns', transform: (m) => { - if ('value' in m) { - if (typeof m.value !== 'number') { + if ('value' in bagOf(m)) { + if (typeof bagOf(m).value !== 'number') { throw new Error('value must be a number') } - return { ...m, value: (m.value as number) * 10 } + return withBag(m, { value: (bagOf(m).value as number) * 10 }) } return null } @@ -615,7 +631,7 @@ describe('Migration System', () => { description: 'Always throws', applies: 'nouns', transform: (m) => { - if ('boom' in m) { + if ('boom' in bagOf(m)) { throw new Error('deliberate failure') } return null diff --git a/tests/integration/orderby-sort-bug.test.ts b/tests/integration/orderby-sort-bug.test.ts index db40fe12..aeb7ff66 100644 --- a/tests/integration/orderby-sort-bug.test.ts +++ b/tests/integration/orderby-sort-bug.test.ts @@ -56,7 +56,7 @@ describe('find({ orderBy }) sort bug regression', () => { const results = await brain.find({ type: NounType.Concept, - orderBy: 'createdAt', + orderBy: 'system.createdAt', order: 'desc', limit: 1 }) @@ -76,7 +76,7 @@ describe('find({ orderBy }) sort bug regression', () => { const results = await brain.find({ type: NounType.Concept, - orderBy: 'createdAt', + orderBy: 'system.createdAt', order: 'asc', limit: 1 }) @@ -94,7 +94,7 @@ describe('find({ orderBy }) sort bug regression', () => { const results = await brain.find({ type: NounType.Concept, - orderBy: 'createdAt', + orderBy: 'system.createdAt', order: 'desc' }) @@ -115,7 +115,7 @@ describe('find({ orderBy }) sort bug regression', () => { const results = await brain.find({ type: NounType.Concept, - orderBy: 'updatedAt', + orderBy: 'system.updatedAt', order: 'desc', limit: 1 }) @@ -136,7 +136,7 @@ describe('find({ orderBy }) sort bug regression', () => { const id3 = await brain.add({ data: 'third', type: NounType.Concept }) const results = await brain.find({ - orderBy: 'createdAt', + orderBy: 'system.createdAt', order: 'desc', limit: 2 }) diff --git a/tests/regression/metadata-index-cleanup.unit.test.ts b/tests/regression/metadata-index-cleanup.unit.test.ts index 0984d727..3746e833 100644 --- a/tests/regression/metadata-index-cleanup.unit.test.ts +++ b/tests/regression/metadata-index-cleanup.unit.test.ts @@ -244,7 +244,10 @@ describe('Metadata index cleanup after remove / removeMany', () => { const noConfidenceId = await addEntity({ type: 'thing' }) const withConfidenceId = await addEntity({ type: 'thing', confidence: 0.9 }) - const results = await brain.find({ where: { confidence: { exists: true } } }) + // system.confidence — confidence is an engine scalar (an add() param), + // never a metadata field; bare 'confidence' now addresses the user's + // own metadata bag under the sealed field-addressing law. + const results = await brain.find({ where: { 'system.confidence': { exists: true } } }) const ids = results.map(r => r.id) expect(ids).toContain(withConfidenceId) @@ -255,7 +258,8 @@ describe('Metadata index cleanup after remove / removeMany', () => { const noWeightId = await addEntity({ type: 'thing' }) const withWeightId = await addEntity({ type: 'thing', weight: 0.5 }) - const results = await brain.find({ where: { weight: { exists: true } } }) + // system.weight — same reasoning as system.confidence above. + const results = await brain.find({ where: { 'system.weight': { exists: true } } }) const ids = results.map(r => r.id) expect(ids).toContain(withWeightId) @@ -269,11 +273,12 @@ describe('Metadata index cleanup after remove / removeMany', () => { const id = await addEntity({ type: 'thing' }) await brain.remove(id) - // Entity must not appear in any confidence query - const existsTrue = await brain.find({ where: { confidence: { exists: true } } }) + // Entity must not appear in any confidence query. system.confidence — + // same addressing as the two tests above. + const existsTrue = await brain.find({ where: { 'system.confidence': { exists: true } } }) expect(existsTrue.map(r => r.id)).not.toContain(id) - const existsFalse = await brain.find({ where: { confidence: { exists: false } } }) + const existsFalse = await brain.find({ where: { 'system.confidence': { exists: false } } }) expect(existsFalse.map(r => r.id)).not.toContain(id) }) }) diff --git a/tests/unit/brainy/find-orderby-pagek.test.ts b/tests/unit/brainy/find-orderby-pagek.test.ts index 9a453f8d..49fccb02 100644 --- a/tests/unit/brainy/find-orderby-pagek.test.ts +++ b/tests/unit/brainy/find-orderby-pagek.test.ts @@ -42,7 +42,8 @@ describe('find({ where, orderBy }) bounds the sort to the page (CTX-BR-FIND-ORDE return real(f, ob, o, topK) } - const results = await brain.find({ where: { bucket: 'x' }, orderBy: 'createdAt', order: 'desc', limit: 5 }) + // system.createdAt — entity age, not a user metadata field named 'createdAt'. + const results = await brain.find({ where: { bucket: 'x' }, orderBy: 'system.createdAt', order: 'desc', limit: 5 }) expect(results).toHaveLength(5) // Page-bounded: ~ limit (5) + a small hidden-tier over-fetch — NOT all 50 matches. diff --git a/tests/unit/brainy/reserved-field-policy.test.ts b/tests/unit/brainy/reserved-field-policy.test.ts deleted file mode 100644 index c5f37af4..00000000 --- a/tests/unit/brainy/reserved-field-policy.test.ts +++ /dev/null @@ -1,251 +0,0 @@ -/** - * @module tests/unit/brainy/reserved-field-policy - * @description The 8.0 `reservedFieldPolicy` matrix — what happens when an - * untyped (JavaScript) caller smuggles a Brainy-reserved field INSIDE the - * `metadata` bag of a write call, past the compile-time guard. - * - * 8.0 is a clean break with no silent failures. The decided contract: - * - `'throw'` (DEFAULT): a reserved key in the bag throws a clear Error naming - * the offending key(s) and the correct write path. No remap, no data loss. - * - `'warn'`: legacy remap PLUS a one-shot (per method+field, per process) - * warning for EVERY reserved key found. - * - `'remap'`: the pre-8.0 silent remap, no warning. - * - * The deep correctness of the remap itself (top-level precedence, system-managed - * drops, transact()/with() mirrors, read-side splitting) lives in - * tests/unit/brainy/update-reserved-metadata-remap.test.ts (which now runs under - * `reservedFieldPolicy: 'remap'`). This file pins the POLICY SELECTION and the - * throw/warn behaviors. - * - * Compile-time callers can't write these shapes at all (see - * tests/unit/types/reserved-metadata-keys.test-d.ts); the `as object` widenings - * below simulate untyped callers. - */ - -import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest' -import { Brainy } from '../../../src/index.js' -import { NounType, VerbType } from '../../../src/types/graphTypes.js' -import { createTestConfig } from '../../helpers/test-factory.js' -import { prodLog } from '../../../src/utils/logger.js' - -describe('reservedFieldPolicy', () => { - describe("default policy is 'throw'", () => { - let brain: Brainy - - beforeEach(async () => { - // No reservedFieldPolicy override → resolves to 'throw'. - brain = new Brainy(createTestConfig()) - await brain.init() - }) - - afterEach(async () => { - await brain.close() - }) - - it('add() throws naming the offending key and the correct write path', async () => { - await expect( - brain.add({ - type: NounType.Concept, - subtype: 'general', - data: 'x', - metadata: { confidence: 0.8 } as object - }) - ).rejects.toThrow(/metadata\.confidence is a reserved field/) - - // The error names the right param and the reserved list for discoverability. - await expect( - brain.add({ - type: NounType.Concept, - subtype: 'general', - data: 'x', - metadata: { confidence: 0.8 } as object - }) - ).rejects.toThrow(/'confidence' param.*RESERVED_ENTITY_FIELDS/s) - }) - - it('add() lists EVERY offending key when several are present', async () => { - const err = await brain - .add({ - type: NounType.Person, - data: 'multi', - metadata: { confidence: 0.5, weight: 0.6, subtype: 'employee' } as object - }) - .catch((e) => e as Error) - expect(err).toBeInstanceOf(Error) - expect(err.message).toMatch(/confidence/) - expect(err.message).toMatch(/weight/) - expect(err.message).toMatch(/subtype/) - }) - - it('update() throws on a reserved key in the patch', async () => { - const id = await brain.add({ type: NounType.Concept, subtype: 'general', data: 'y' }) - await expect( - brain.update({ id, metadata: { confidence: 0.3 } as object }) - ).rejects.toThrow(/metadata\.confidence is a reserved field/) - }) - - it('relate() throws on a reserved key in the bag', async () => { - const a = await brain.add({ type: NounType.Person, subtype: 'employee', data: 'A' }) - const b = await brain.add({ type: NounType.Person, subtype: 'employee', data: 'B' }) - await expect( - brain.relate({ - from: a, - to: b, - type: VerbType.RelatedTo, - subtype: 'colleague', - metadata: { confidence: 0.4 } as object - }) - ).rejects.toThrow(/metadata\.confidence is a reserved field.*RESERVED_RELATION_FIELDS/s) - }) - - it('updateRelation() throws on a reserved key in the patch', async () => { - const a = await brain.add({ type: NounType.Person, subtype: 'employee', data: 'A' }) - const b = await brain.add({ type: NounType.Person, subtype: 'employee', data: 'B' }) - const relId = await brain.relate({ - from: a, - to: b, - type: VerbType.ReportsTo, - subtype: 'direct' - }) - await expect( - brain.updateRelation({ id: relId, metadata: { weight: 0.2 } as object }) - ).rejects.toThrow(/metadata\.weight is a reserved field/) - }) - - it('transact() add op throws on a reserved key in the bag', async () => { - await expect( - brain.transact([ - { - op: 'add', - type: NounType.Concept, - subtype: 'general', - data: 'tx', - metadata: { confidence: 0.7 } as object - } - ]) - ).rejects.toThrow(/metadata\.confidence is a reserved field/) - }) - - it('a custom (non-reserved) key in the bag does NOT throw', async () => { - const id = await brain.add({ - type: NounType.Concept, - subtype: 'general', - data: 'ok', - metadata: { status: 'draft', rating: 4 } - }) - const entity = await brain.get(id) - expect(entity?.metadata).toEqual({ status: 'draft', rating: 4 }) - }) - }) - - describe("'remap' policy remaps silently (no warning)", () => { - let brain: Brainy - let warnSpy: ReturnType - - beforeEach(async () => { - warnSpy = vi.spyOn(prodLog, 'warn').mockImplementation(() => {}) - brain = new Brainy(createTestConfig({ reservedFieldPolicy: 'remap' })) - await brain.init() - }) - - afterEach(async () => { - await brain.close() - warnSpy.mockRestore() - }) - - it('lifts user-mutable reserved fields to top-level without warning', async () => { - const id = await brain.add({ - type: NounType.Person, - data: 'remap lift', - metadata: { confidence: 0.8, weight: 0.6, subtype: 'employee', dept: 'eng' } as object - }) - const entity = await brain.get(id) - expect(entity?.confidence).toBe(0.8) - expect(entity?.weight).toBe(0.6) - expect(entity?.subtype).toBe('employee') - expect(entity?.metadata).toEqual({ dept: 'eng' }) - // 'remap' is silent about reserved fields (unrelated storage logs may fire, - // so assert specifically that no reserved-field warning was emitted). - const reservedWarned = warnSpy.mock.calls.some((c) => - String(c[0]).includes('reserved field') - ) - expect(reservedWarned).toBe(false) - }) - - it('preserves _originalId on natural-key ids through the remap path', async () => { - // A speculative view applies the same normalization and maps a natural-key - // id to a stable UUID, preserving the caller's original string. - const base = await brain.now() - const speculative = await base.with([ - { - op: 'add', - id: 'remap-spec-entity', - type: NounType.Concept, - subtype: 'general', - data: 'spec', - metadata: { confidence: 0.65, custom: 'spec' } as object - } - ]) - const entity = await speculative.get('remap-spec-entity') - expect(entity?.confidence).toBe(0.65) - expect(entity?.metadata).toEqual({ custom: 'spec', _originalId: 'remap-spec-entity' }) - await speculative.release() - await base.release() - }) - }) - - describe("'warn' policy remaps AND warns once per key", () => { - let brain: Brainy - let warnSpy: ReturnType - - beforeEach(async () => { - warnSpy = vi.spyOn(prodLog, 'warn').mockImplementation(() => {}) - brain = new Brainy(createTestConfig({ reservedFieldPolicy: 'warn' })) - await brain.init() - }) - - afterEach(async () => { - await brain.close() - warnSpy.mockRestore() - }) - - it('remaps the value (same as remap) and emits a warning naming the field', async () => { - // Use a method+field combo unique to this test so the per-process one-shot - // registry has not already consumed it. - const id = await brain.add({ - type: NounType.Person, - data: 'warn lift', - // weight is user-mutable → remapped; this is the only 'warn'-policy - // add({ weight }) in the suite, so the one-shot warning fires here. - metadata: { weight: 0.42, dept: 'eng' } as object - }) - const entity = await brain.get(id) - // Value is honored (remap still happens under 'warn'). - expect(entity?.weight).toBe(0.42) - expect(entity?.metadata).toEqual({ dept: 'eng' }) - // And a warning was emitted naming the reserved field. - expect(warnSpy).toHaveBeenCalled() - const warned = warnSpy.mock.calls.some((c) => - String(c[0]).includes("'weight'") - ) - expect(warned).toBe(true) - }) - - it('warns for system-managed keys too (closes the historical gap)', async () => { - // Pre-8.0 only system-managed fields warned; 'warn' warns for every key. - // 'createdBy' (system-managed on update) is unique to this test. - const id = await brain.add({ type: NounType.Concept, subtype: 'general', data: 'sys' }) - warnSpy.mockClear() - await brain.update({ id, metadata: { createdBy: 'nope', keep: 'me' } as object }) - const entity = await brain.get(id) - // System-managed key dropped; custom field merged. - expect((entity?.metadata as Record)?.createdBy).toBeUndefined() - expect((entity?.metadata as Record)?.keep).toBe('me') - // A warning was emitted for the dropped system-managed key. - const warned = warnSpy.mock.calls.some((c) => - String(c[0]).includes("'createdBy'") - ) - expect(warned).toBe(true) - }) - }) -}) diff --git a/tests/unit/brainy/update-reserved-metadata-remap.test.ts b/tests/unit/brainy/update-reserved-metadata-remap.test.ts deleted file mode 100644 index 31713f99..00000000 --- a/tests/unit/brainy/update-reserved-metadata-remap.test.ts +++ /dev/null @@ -1,403 +0,0 @@ -/** - * @module tests/unit/brainy/update-reserved-metadata-remap - * @description Regression tests for the reserved-field metadata-bag trap, - * ported from the 7.x fix and extended to the full 8.0 contract. - * - * History: `add({metadata: {confidence}})` lifted reserved fields to their - * canonical top-level location, but `update({metadata: {confidence}})` - * silently dropped the same shape — the patch value survived the merge and - * was then clobbered by the preserve-existing spread. A production - * consumer's confidence-evolution writes no-oped for weeks before being - * caught by reading values back. - * - * These tests pin the LEGACY REMAP behavior, which in 8.0 is opt-in via - * `reservedFieldPolicy: 'remap'` (the default is `'throw'` — see the policy - * matrix in tests/unit/brainy/reserved-field-policy.test.ts). The brain in - * every test below is constructed with `reservedFieldPolicy: 'remap'` so these - * deep correctness assertions about the remap path stay exercised. - * - * Remap contract under test (every write path, entities AND relationships): - * - user-mutable reserved fields (`confidence`, `weight`, `subtype` — plus - * `service`/`createdBy` at add()/relate() time) remap from the metadata - * bag to their dedicated top-level param, with top-level winning when both - * are present; - * - system-managed reserved fields (`createdAt`, `_rev`, `noun`/`verb`, - * `data`, …) are dropped from the bag; - * - the same normalization applies to `transact()` operations and `with()` - * speculative views; - * - reads NEVER echo a reserved field inside `metadata`. - * - * TypeScript callers can't write these shapes at all (compile-time guard on - * the metadata param types — see tests/unit/types/reserved-metadata-keys.test-d.ts); - * these tests simulate untyped (JavaScript) callers, hence the `as object` - * widenings on the metadata literals. - */ - -import { describe, it, expect, beforeEach, afterEach } from 'vitest' -import { Brainy } from '../../../src/index.js' -import { NounType, VerbType } from '../../../src/types/graphTypes.js' -import { createTestConfig } from '../../helpers/test-factory.js' - -describe('reserved-field metadata remap (8.0 legacy remap path)', () => { - let brain: Brainy - - beforeEach(async () => { - // The remap path is opt-in in 8.0 (default policy is 'throw'). - brain = new Brainy(createTestConfig({ reservedFieldPolicy: 'remap' })) - await brain.init() - }) - - afterEach(async () => { - await brain.close() - }) - - describe('update() — the ported 7.x regression', () => { - it('remaps metadata.confidence to the top-level field (the production repro)', async () => { - const id = await brain.add({ - type: NounType.Concept, - subtype: 'general', - data: 'x', - metadata: { confidence: 0.8 } as object - }) - - // Top-level write works (always did) - await brain.update({ id, confidence: 0.42 }) - let entity = await brain.get(id) - expect(entity?.confidence).toBe(0.42) - - // Metadata-patch write — silently dropped pre-fix, remapped now - await brain.update({ id, metadata: { confidence: 0.33 } as object }) - entity = await brain.get(id) - expect(entity?.confidence).toBe(0.33) - // The reserved key must not linger inside the metadata bag - expect((entity?.metadata as Record)?.confidence).toBeUndefined() - }) - - it('remaps metadata.weight and metadata.subtype the same way', async () => { - const id = await brain.add({ - type: NounType.Concept, - subtype: 'general', - data: 'y', - metadata: {} - }) - - await brain.update({ id, metadata: { weight: 0.7, subtype: 'specialized' } as object }) - const entity = await brain.get(id) - expect(entity?.weight).toBe(0.7) - expect(entity?.subtype).toBe('specialized') - expect((entity?.metadata as Record)?.weight).toBeUndefined() - expect((entity?.metadata as Record)?.subtype).toBeUndefined() - }) - - it('top-level param wins when both top-level and metadata-patch carry the field', async () => { - const id = await brain.add({ - type: NounType.Concept, - subtype: 'general', - data: 'z', - metadata: { confidence: 0.5 } as object - }) - - await brain.update({ id, confidence: 0.9, metadata: { confidence: 0.1 } as object }) - const entity = await brain.get(id) - expect(entity?.confidence).toBe(0.9) - }) - - it('drops system-managed fields from patches without corrupting the entity', async () => { - const id = await brain.add({ - type: NounType.Concept, - subtype: 'general', - data: 'w', - metadata: { keep: 'me' } - }) - const before = await brain.get(id) - - await brain.update({ - id, - metadata: { createdAt: 1, _rev: 999, noun: 'organization', other: 'applied' } as object - }) - const after = await brain.get(id) - - expect(after?.createdAt).toBe(before?.createdAt) // immutable - expect(after?.type).toBe('concept') // noun patch ignored - expect(after?._rev).toBe((before?._rev ?? 1) + 1) // _rev patch ignored; normal bump applied - expect((after?.metadata as Record)?.other).toBe('applied') // custom fields still merge - expect((after?.metadata as Record)?.keep).toBe('me') - expect((after?.metadata as Record)?._rev).toBeUndefined() - expect((after?.metadata as Record)?.createdAt).toBeUndefined() - expect((after?.metadata as Record)?.noun).toBeUndefined() - }) - - it('custom (non-reserved) metadata patches are unaffected by the remap', async () => { - const id = await brain.add({ - type: NounType.Concept, - subtype: 'general', - data: 'v', - metadata: { status: 'draft' } - }) - - await brain.update({ id, metadata: { status: 'reviewed', rating: 4.5 } }) - const entity = await brain.get(id) - expect((entity?.metadata as Record)?.status).toBe('reviewed') - expect((entity?.metadata as Record)?.rating).toBe(4.5) - }) - }) - - describe('add() — explicit lift, identical contract', () => { - it('lifts confidence/weight/subtype out of the bag to top level', async () => { - const id = await brain.add({ - type: NounType.Person, - data: 'lift check', - metadata: { confidence: 0.8, weight: 0.6, subtype: 'employee', dept: 'eng' } as object - }) - - const entity = await brain.get(id) - expect(entity?.confidence).toBe(0.8) - expect(entity?.weight).toBe(0.6) - expect(entity?.subtype).toBe('employee') - expect(entity?.metadata).toEqual({ dept: 'eng' }) - }) - - it('lifts service (settable at add time) and lets the top-level param win', async () => { - const lifted = await brain.add({ - type: NounType.Person, - subtype: 'employee', - data: 'service lift', - metadata: { service: 'orders' } as object - }) - expect((await brain.get(lifted))?.service).toBe('orders') - - const topLevelWins = await brain.add({ - type: NounType.Person, - subtype: 'employee', - data: 'service precedence', - service: 'billing', - metadata: { service: 'orders' } as object - }) - const entity = await brain.get(topLevelWins) - expect(entity?.service).toBe('billing') - expect((entity?.metadata as Record)?.service).toBeUndefined() - }) - - it('a remapped subtype satisfies subtype enforcement like a top-level one', async () => { - brain.requireSubtype(NounType.Document) - - // Top-level missing, but the bag carries it — must not throw. - const id = await brain.add({ - type: NounType.Document, - data: 'enforcement via remap', - metadata: { subtype: 'invoice' } as object - }) - expect((await brain.get(id))?.subtype).toBe('invoice') - - // Neither place carries it — must throw. - await expect( - brain.add({ type: NounType.Document, data: 'no subtype anywhere' }) - ).rejects.toThrow(/subtype/) - }) - }) - - describe('transact() — same remap on add and update ops', () => { - it('normalizes reserved fields in transact add + update ops', async () => { - const db1 = await brain.transact([ - { - op: 'add', - type: NounType.Concept, - subtype: 'general', - data: 'tx', - metadata: { confidence: 0.7, custom: 'a' } as object - } - ]) - const id = db1.receipt!.ids[0] - - let entity = await brain.get(id) - expect(entity?.confidence).toBe(0.7) - expect(entity?.metadata).toEqual({ custom: 'a' }) - - await brain.transact([ - { op: 'update', id, metadata: { confidence: 0.25, custom: 'b' } as object } - ]) - entity = await brain.get(id) - expect(entity?.confidence).toBe(0.25) - expect(entity?.metadata).toEqual({ custom: 'b' }) - expect((entity?.metadata as Record)?.confidence).toBeUndefined() - }) - - it('historical asOf() reads surface reserved fields ONLY top-level', async () => { - const db1 = await brain.transact([ - { - op: 'add', - type: NounType.Concept, - subtype: 'general', - data: 'historical', - metadata: { confidence: 0.9, custom: 'past' } as object - } - ]) - const id = db1.receipt!.ids[0] - - // Move the world forward so generation db1 is historical. - await brain.transact([{ op: 'update', id, confidence: 0.1, metadata: { custom: 'now' } }]) - - const past = await brain.asOf(db1.generation) - const historical = await past.get(id) - expect(historical?.confidence).toBe(0.9) - expect(historical?.metadata).toEqual({ custom: 'past' }) - await past.release() - }) - - it('with() speculative views apply the same normalization', async () => { - const base = await brain.now() - const speculative = await base.with([ - { - op: 'add', - id: 'spec-entity', - type: NounType.Concept, - subtype: 'general', - data: 'spec', - metadata: { confidence: 0.65, custom: 'spec' } as object - } - ]) - - const entity = await speculative.get('spec-entity') - expect(entity?.confidence).toBe(0.65) - // 8.0 id normalization: a natural-key id is mapped to a stable UUID and - // the caller's original string is preserved under _originalId — surfaced - // here exactly as the durable transact()/add() paths do. - expect(entity?.metadata).toEqual({ custom: 'spec', _originalId: 'spec-entity' }) - await speculative.release() - await base.release() - }) - }) - - describe('read paths never echo reserved fields inside metadata', () => { - it('find() (storage pagination path) returns custom-only metadata with reserved fields top-level', async () => { - const id = await brain.add({ - type: NounType.Person, - subtype: 'employee', - data: 'pagination echo check', - confidence: 0.8, - weight: 0.6, - metadata: { dept: 'eng' } - }) - - // No query/filter → served by the direct storage pagination path - // (getNounsWithPagination), which historically echoed the full flat - // record (noun/subtype/createdAt/… inside metadata). - const results = await brain.find({ limit: 50 }) - const result = results.find((r) => r.id === id) - expect(result).toBeDefined() - expect(result?.entity.metadata).toEqual({ dept: 'eng' }) - expect(result?.entity.type).toBe(NounType.Person) - expect(result?.entity.subtype).toBe('employee') - expect(result?.entity.confidence).toBe(0.8) - expect(result?.entity.weight).toBe(0.6) - expect(typeof result?.entity.createdAt).toBe('number') - expect(result?.entity._rev).toBe(1) - }) - - it('related() by target surfaces reserved fields top-level, custom-only metadata', async () => { - const a = await brain.add({ type: NounType.Person, subtype: 'employee', data: 'src' }) - const b = await brain.add({ type: NounType.Person, subtype: 'employee', data: 'tgt' }) - const relId = await brain.relate({ - from: a, - to: b, - type: VerbType.ReportsTo, - subtype: 'direct', - confidence: 0.9, - weight: 0.5, - service: 'orders', - metadata: { note: 'target path' } - }) - - const relations = await brain.related({ to: b }) - const rel = relations.find((r) => r.id === relId) - expect(rel).toBeDefined() - expect(rel?.metadata).toEqual({ note: 'target path' }) - expect(rel?.subtype).toBe('direct') - expect(rel?.confidence).toBe(0.9) - expect(rel?.weight).toBe(0.5) - expect(rel?.service).toBe('orders') - expect(typeof rel?.createdAt).toBe('number') - }) - }) - - describe('relationships — relate() / updateRelation() mirror', () => { - let a: string - let b: string - - beforeEach(async () => { - a = await brain.add({ type: NounType.Person, subtype: 'employee', data: 'A' }) - b = await brain.add({ type: NounType.Person, subtype: 'employee', data: 'B' }) - }) - - it('relate() persists the top-level confidence and service params', async () => { - const relId = await brain.relate({ - from: a, - to: b, - type: VerbType.ReportsTo, - subtype: 'direct', - confidence: 0.77, - service: 'orders' - }) - - const relations = await brain.related({ from: a }) - const rel = relations.find((r) => r.id === relId) - expect(rel?.confidence).toBe(0.77) - expect(rel?.service).toBe('orders') - }) - - it('relate() remaps reserved fields out of the metadata bag', async () => { - const relId = await brain.relate({ - from: a, - to: b, - type: VerbType.RelatedTo, - subtype: 'colleague', - metadata: { confidence: 0.4, weight: 0.3, role: 'peer' } as object - }) - - const relations = await brain.related({ from: a }) - const rel = relations.find((r) => r.id === relId) - expect(rel?.confidence).toBe(0.4) - expect(rel?.weight).toBe(0.3) - expect(rel?.metadata).toEqual({ role: 'peer' }) - }) - - it('relation.metadata never echoes the verb type key', async () => { - const relId = await brain.relate({ - from: a, - to: b, - type: VerbType.RelatedTo, - subtype: 'colleague', - metadata: { note: 'no echo' } - }) - - const relations = await brain.related({ from: a }) - const rel = relations.find((r) => r.id === relId) - expect(rel?.type).toBe(VerbType.RelatedTo) - expect((rel?.metadata as Record)?.verb).toBeUndefined() - expect(rel?.metadata).toEqual({ note: 'no echo' }) - }) - - it('updateRelation() remaps the user-mutable trio and preserves service', async () => { - const relId = await brain.relate({ - from: a, - to: b, - type: VerbType.ReportsTo, - subtype: 'direct', - service: 'orders', - metadata: { keep: 'me' } - }) - - await brain.updateRelation({ - id: relId, - metadata: { confidence: 0.55, subtype: 'dotted-line', extra: 'applied' } as object - }) - - const relations = await brain.related({ from: a }) - const rel = relations.find((r) => r.id === relId) - expect(rel?.confidence).toBe(0.55) - expect(rel?.subtype).toBe('dotted-line') - expect(rel?.service).toBe('orders') // fixed at relate() time, never erased by updates - expect(rel?.metadata).toEqual({ keep: 'me', extra: 'applied' }) - }) - }) -}) diff --git a/tests/unit/brainy/visibility.test.ts b/tests/unit/brainy/visibility.test.ts index a5a02422..dd4540d7 100644 --- a/tests/unit/brainy/visibility.test.ts +++ b/tests/unit/brainy/visibility.test.ts @@ -198,60 +198,47 @@ describe('visibility (8.0 reserved field)', () => { expect(entity?.visibility).toBeUndefined() }) - it('an untyped caller passing visibility inside metadata is normalized under reservedFieldPolicy:"remap" (lifted to top-level)', async () => { - // Simulate a JavaScript caller smuggling the reserved key past the compile-time guard. - // The legacy remap behavior is now opt-in (8.0 default is 'throw'). - const remapBrain = new Brainy(createTestConfig({ reservedFieldPolicy: 'remap' })) - await remapBrain.init() - try { - const id = await remapBrain.add({ - type: NounType.Concept, - data: 'y', - metadata: { visibility: 'internal', tag: 't' } as object - }) - const entity = await remapBrain.get(id) - // Lifted to the top-level field… - expect(entity?.visibility).toBe('internal') - // …and stripped from the metadata bag. - expect((entity?.metadata as Record)?.visibility).toBeUndefined() - expect((entity?.metadata as Record)?.tag).toBe('t') - // It is excluded from the default count, exactly like a top-level internal write. - expect(await remapBrain.getNounCount()).toBe(0) - } finally { - await remapBrain.close() - } + it('metadata.visibility is the USER’s field (field-addressing law) — stored verbatim, never lifted to the engine tier', async () => { + const id = await brain.add({ + type: NounType.Concept, + data: 'y', + metadata: { visibility: 'internal', tag: 't' } as object + }) + const entity = await brain.get(id) + // The user's field lives in the bag, verbatim… + expect((entity?.metadata as Record)?.visibility).toBe('internal') + expect((entity?.metadata as Record)?.tag).toBe('t') + // …and the ENGINE tier is untouched: absent === public, so the entity + // stays visible on default reads (the engine tier is set only via the + // dedicated visibility param and reads at system.visibility). + expect(entity?.visibility).toBeUndefined() + const visible = await brain.find({ type: NounType.Concept, limit: 20 }) + expect(visible.map((r) => r.id)).toContain(id) }) - it('a "system" value smuggled through metadata is dropped under reservedFieldPolicy:"remap", not honored', async () => { - // 'system' is Brainy-only; an untyped caller must not be able to set it. - const remapBrain = new Brainy(createTestConfig({ reservedFieldPolicy: 'remap' })) - await remapBrain.init() - try { - const id = await remapBrain.add({ - type: NounType.Concept, - data: 'z', - metadata: { visibility: 'system' } as object - }) - const entity = await remapBrain.get(id) - // The smuggled 'system' was dropped → entity stays public (counted, visible). - expect(entity?.visibility).toBeUndefined() - expect(await remapBrain.getNounCount()).toBe(1) - const found = await remapBrain.find({ type: NounType.Concept, limit: 10 }) - expect(found.map((r) => r.id)).toContain(id) - } finally { - await remapBrain.close() - } + it('a user field valued "system" cannot smuggle the Brainy-only tier — it is just user data', async () => { + const id = await brain.add({ + type: NounType.Concept, + data: 'z', + metadata: { visibility: 'system' } as object + }) + const entity = await brain.get(id) + // Engine tier unaffected → entity stays public (counted, visible); + // the string 'system' is ordinary user data in the bag. + expect(entity?.visibility).toBeUndefined() + expect((entity?.metadata as Record)?.visibility).toBe('system') + const found = await brain.find({ type: NounType.Concept, limit: 10 }) + expect(found.map((r) => r.id)).toContain(id) }) - it('an untyped caller passing visibility inside metadata throws under the default policy', async () => { - // 8.0 default: no silent remap — a reserved key in the bag is a loud error. + it('a forged system.visibility key in metadata refuses loudly at the write door', async () => { await expect( brain.add({ type: NounType.Concept, data: 'throws', - metadata: { visibility: 'internal', tag: 't' } as object + metadata: { 'system.visibility': 'internal' } as object }) - ).rejects.toThrow(/visibility.*reserved field/) + ).rejects.toThrow(/system\./) }) }) }) diff --git a/tests/unit/db/whereMatcher.test.ts b/tests/unit/db/whereMatcher.test.ts index 2223117c..6c0252d6 100644 --- a/tests/unit/db/whereMatcher.test.ts +++ b/tests/unit/db/whereMatcher.test.ts @@ -32,7 +32,7 @@ function entity(overrides: Partial = {}): Entity { } describe('db/whereMatcher — resolveEntityField', () => { - it('resolves standard top-level fields', () => { + it('system. resolves the entity scalar; bare/metadata. reads the metadata bag only (sealed 2026-08-03)', () => { const e = entity({ subtype: 'invoice', service: 'billing', @@ -41,17 +41,32 @@ describe('db/whereMatcher — resolveEntityField', () => { _rev: 3, data: 'payload' }) - expect(resolveEntityField(e, 'id')).toBe('e-1') - expect(resolveEntityField(e, 'type')).toBe(NounType.Document) - expect(resolveEntityField(e, 'noun')).toBe(NounType.Document) // alias - expect(resolveEntityField(e, 'subtype')).toBe('invoice') - expect(resolveEntityField(e, 'service')).toBe('billing') - expect(resolveEntityField(e, 'confidence')).toBe(0.9) - expect(resolveEntityField(e, 'weight')).toBe(0.5) - expect(resolveEntityField(e, '_rev')).toBe(3) - expect(resolveEntityField(e, 'createdAt')).toBe(1000) - expect(resolveEntityField(e, 'updatedAt')).toBe(2000) - expect(resolveEntityField(e, 'data')).toBe('payload') + + // system. is the ONLY spelling that reaches an entity scalar. + expect(resolveEntityField(e, 'system.id')).toBe('e-1') + expect(resolveEntityField(e, 'system.type')).toBe(NounType.Document) + expect(resolveEntityField(e, 'system.subtype')).toBe('invoice') + expect(resolveEntityField(e, 'system.service')).toBe('billing') + expect(resolveEntityField(e, 'system.confidence')).toBe(0.9) + expect(resolveEntityField(e, 'system.weight')).toBe(0.5) + expect(resolveEntityField(e, 'system.createdAt')).toBe(1000) + expect(resolveEntityField(e, 'system.updatedAt')).toBe(2000) + + // Plumbing (_rev, data) is invisible even via system. — not in the + // ten-scalar map, so this internal resolver reads it as absent (the typed + // refusal for these lives one layer up, at the query-surface parser). + expect(resolveEntityField(e, 'system._rev')).toBeUndefined() + expect(resolveEntityField(e, 'system.data')).toBeUndefined() + + // Bare names are ALWAYS the user's metadata field — even when they share + // a spelling with an engine scalar, or with the now-dead 'noun' alias. + // This entity's metadata bag is empty, so every bare name below reads + // absent rather than silently falling back to the entity scalar. + expect(resolveEntityField(e, 'id')).toBeUndefined() + expect(resolveEntityField(e, 'type')).toBeUndefined() + expect(resolveEntityField(e, 'noun')).toBeUndefined() // legacy alias is dead + expect(resolveEntityField(e, 'subtype')).toBeUndefined() + expect(resolveEntityField(e, 'createdAt')).toBeUndefined() }) it('resolves custom fields from the metadata bag', () => { diff --git a/tests/unit/test-suite-coverage-guard.test.ts b/tests/unit/test-suite-coverage-guard.test.ts index 93db4421..21f918f1 100644 --- a/tests/unit/test-suite-coverage-guard.test.ts +++ b/tests/unit/test-suite-coverage-guard.test.ts @@ -30,6 +30,9 @@ function allTestFiles(dir: string, out: string[] = []): string[] { * conscious decision — a NEW orphan not listed here fails the guard below. */ const MANUAL_ONLY = new Set([ + // Conformance suites run as an explicit gate stage (both engines run them + // by direct invocation), never swept into the unit/integration configs. + 'tests/conformance/collider-fidelity.test.ts', 'tests/api/performance-benchmarks.test.ts', 'tests/critical-neural-validation.test.ts', 'tests/critical-performance-benchmark.test.ts', @@ -38,7 +41,15 @@ const MANUAL_ONLY = new Set([ 'tests/package-size-limit.test.ts', 'tests/performance/graph-scale-performance.test.ts', 'tests/performance/triple-intelligence-scale.test.ts', - 'tests/performance/typeAware.bench.test.ts' + 'tests/performance/typeAware.bench.test.ts', + // Cross-engine field-addressing conformance suite: pinned bit-for-bit against + // the native accelerator's implementation of the SAME contract, and invoked + // directly (`npx vitest run tests/conformance/namespace-law.test.ts`), never + // swept into the unit/integration gates — a run against a branch where the + // resolver hasn't landed yet must SKIP loudly (see the file's own SELF-SKIP + // doc), not silently pass/fail as a side effect of which gate happened to + // pick it up. + 'tests/conformance/namespace-law.test.ts' ]) function inGate(rel: string): boolean { diff --git a/tests/unit/types/nestedBagRecord.test.ts b/tests/unit/types/nestedBagRecord.test.ts new file mode 100644 index 00000000..b8e9be46 --- /dev/null +++ b/tests/unit/types/nestedBagRecord.test.ts @@ -0,0 +1,127 @@ +/** + * @module tests/unit/types/nestedBagRecord + * @description Unit pins for the v2 (nested-bag) stored-record layer — the + * storage half of the field-addressing law. The write door accepts ANY user + * metadata name; what makes that lossless on disk is the record shape: + * engine fields top-level, the user bag NESTED verbatim, discriminated by + * the engine-written format stamp (never by names — names are the user's). + * These pins hold the builders, the discriminator, and the shape-aware + * split that every read path (live, batch, historical) routes through. + */ +import { describe, it, expect } from 'vitest' +import { + buildNounMetadataRecord, + buildVerbMetadataRecord, + splitNounMetadataRecord, + splitVerbMetadataRecord, + isNestedBagRecord, + METADATA_RECORD_FORMAT_KEY, + NESTED_BAG_FORMAT +} from '../../../src/types/reservedFields.js' + +const COLLIDER_BAG = { + confidence: 'user-confidence', + weight: 'user-weight', + subtype: 'user-subtype', + createdAt: 'user-createdAt', + service: 'user-service', + data: 'user-data', + noun: 'user-noun', + _rev: 'user-rev', + level: 7, + plain: 'control' +} + +describe('v2 nested-bag stored records — build / discriminate / split', () => { + it('build → split round-trips a fully colliding user bag VERBATIM', () => { + const record = buildNounMetadataRecord( + { noun: 'document', confidence: 0.25, createdAt: 111, updatedAt: 222, _rev: 1 }, + { ...COLLIDER_BAG } + ) + expect(isNestedBagRecord(record)).toBe(true) + expect(record[METADATA_RECORD_FORMAT_KEY]).toBe(NESTED_BAG_FORMAT) + + const { reserved, custom } = splitNounMetadataRecord(record) + // The engine half is exactly what the engine wrote… + expect(reserved.noun).toBe('document') + expect(reserved.confidence).toBe(0.25) + expect(reserved._rev).toBe(1) + // …and the user bag comes back byte-for-byte, colliders included. + expect(custom).toEqual(COLLIDER_BAG) + }) + + it('the verb mirror round-trips an edge collider bag verbatim', () => { + const record = buildVerbMetadataRecord( + { verb: 'relatedTo', weight: 1.0, confidence: 0.5, createdAt: 333 }, + { verb: 'user-verb', confidence: 'user-c', tag: 't' } + ) + expect(isNestedBagRecord(record)).toBe(true) + const { reserved, custom } = splitVerbMetadataRecord(record) + expect(reserved.verb).toBe('relatedTo') + expect(reserved.confidence).toBe(0.5) + expect(custom).toEqual({ verb: 'user-verb', confidence: 'user-c', tag: 't' }) + }) + + it('a LEGACY flat record (no stamp) splits BY NAME — sound because the pre-law door refused colliders', () => { + const legacy = { + noun: 'document', + confidence: 0.75, + createdAt: 111, + _rev: 2, + legacyField: 'legacy-value' + } + expect(isNestedBagRecord(legacy)).toBe(false) + const { reserved, custom } = splitNounMetadataRecord(legacy) + expect(reserved.confidence).toBe(0.75) + expect(reserved._rev).toBe(2) + expect(custom).toEqual({ legacyField: 'legacy-value' }) + }) + + it('the stamp is the discriminator, never the name: a legacy user OBJECT field named `metadata` does not fake a v2 record', () => { + // Pre-law, 'metadata' was never a reserved name — a flat record could + // legally carry a user object field spelled exactly 'metadata'. Without + // the engine-written stamp it must split as legacy, with that object + // preserved as an ordinary user field. + const legacyWithMetadataField = { + noun: 'document', + confidence: 0.5, + metadata: { nested: 'user-object' } + } + expect(isNestedBagRecord(legacyWithMetadataField)).toBe(false) + const { reserved, custom } = splitNounMetadataRecord(legacyWithMetadataField) + expect(reserved.confidence).toBe(0.5) + expect(custom).toEqual({ metadata: { nested: 'user-object' } }) + }) + + it('a malformed stamp (right key, wrong value / non-object bag) never discriminates as v2', () => { + expect( + isNestedBagRecord({ [METADATA_RECORD_FORMAT_KEY]: 999, metadata: {} }) + ).toBe(false) + expect( + isNestedBagRecord({ [METADATA_RECORD_FORMAT_KEY]: NESTED_BAG_FORMAT, metadata: 'not-a-bag' }) + ).toBe(false) + expect( + isNestedBagRecord({ [METADATA_RECORD_FORMAT_KEY]: NESTED_BAG_FORMAT, metadata: [1, 2] }) + ).toBe(false) + expect(isNestedBagRecord(null)).toBe(false) + expect(isNestedBagRecord(undefined)).toBe(false) + }) + + it('the v2 split never surfaces the stamp or the bag container as fields', () => { + const record = buildNounMetadataRecord({ noun: 'document', _rev: 1 }, { a: 1 }) + const { reserved, custom } = splitNounMetadataRecord(record) + expect(METADATA_RECORD_FORMAT_KEY in reserved).toBe(false) + expect(METADATA_RECORD_FORMAT_KEY in custom).toBe(false) + expect('metadata' in reserved).toBe(false) + expect(custom).toEqual({ a: 1 }) + }) + + it('builders copy the bag (no aliasing): later caller mutation cannot reach the record', () => { + const bag: Record = { a: 1 } + const record = buildNounMetadataRecord({ noun: 'document' }, bag) + bag.a = 999 + bag.b = 'sneaky' + expect((record.metadata as Record).a).toBe(1) + expect('b' in (record.metadata as Record)).toBe(false) + }) +}) diff --git a/tests/unit/types/reserved-metadata-keys.test-d.ts b/tests/unit/types/reserved-metadata-keys.test-d.ts deleted file mode 100644 index 37fceefa..00000000 --- a/tests/unit/types/reserved-metadata-keys.test-d.ts +++ /dev/null @@ -1,265 +0,0 @@ -/** - * @module tests/unit/types/reserved-metadata-keys.test-d - * @description Compile-time tests for the reserved-field contract (layer 1 of - * three — see src/types/reservedFields.ts): a literal reserved key inside any - * `metadata` param is a TypeScript error, while the generic `T` ergonomics - * stay intact (typed bags, untyped brains, index-signature shapes, and the - * documented exemption for consumers who explicitly declare a reserved key in - * their own metadata type). - * - * Runs under vitest typecheck mode (`test.typecheck` in - * tests/configs/vitest.unit.config.ts) — these assertions are validated by - * `tsc`, never executed. The runtime half of the contract (the write-path - * remap for untyped callers) is pinned by - * tests/unit/brainy/update-reserved-metadata-remap.test.ts. - */ - -import { describe, it, assertType } from 'vitest' -import type { - AddParams, - UpdateParams, - RelateParams, - UpdateRelationParams, - TxOperation -} from '../../../src/index.js' -import { NounType, VerbType } from '../../../src/types/graphTypes.js' - -describe('reserved entity keys in metadata are compile errors', () => { - it('AddParams (untyped brain) rejects every reserved key but stays open for custom fields', () => { - // Custom fields of any shape remain legal — exactly the pre-8.0 latitude. - assertType({ - type: NounType.Person, - subtype: 'employee', - data: 'x', - metadata: { dept: 'eng', level: 3, tags: ['a', 'b'], nested: { ok: true } } - }) - - assertType({ - type: NounType.Person, - subtype: 'employee', - data: 'x', - // @ts-expect-error — 'noun' is reserved (the entity type travels via the top-level 'type' param) - metadata: { noun: 'organization' } - }) - assertType({ - type: NounType.Person, - subtype: 'employee', - data: 'x', - // @ts-expect-error — 'subtype' is reserved (use the top-level 'subtype' param) - metadata: { subtype: 'contractor' } - }) - assertType({ - type: NounType.Person, - subtype: 'employee', - data: 'x', - // @ts-expect-error — 'createdAt' is reserved (system-managed) - metadata: { createdAt: Date.now() } - }) - assertType({ - type: NounType.Person, - subtype: 'employee', - data: 'x', - // @ts-expect-error — 'updatedAt' is reserved (system-managed) - metadata: { updatedAt: Date.now() } - }) - assertType({ - type: NounType.Person, - subtype: 'employee', - data: 'x', - // @ts-expect-error — 'confidence' is reserved (use the top-level 'confidence' param) - metadata: { confidence: 0.8 } - }) - assertType({ - type: NounType.Person, - subtype: 'employee', - data: 'x', - // @ts-expect-error — 'weight' is reserved (use the top-level 'weight' param) - metadata: { weight: 0.5 } - }) - assertType({ - type: NounType.Person, - subtype: 'employee', - data: 'x', - // @ts-expect-error — 'service' is reserved (use the top-level 'service' param) - metadata: { service: 'orders' } - }) - assertType({ - type: NounType.Person, - subtype: 'employee', - data: 'x', - // @ts-expect-error — 'data' is reserved (use the top-level 'data' param) - metadata: { data: 'content' } - }) - assertType({ - type: NounType.Person, - subtype: 'employee', - data: 'x', - // @ts-expect-error — 'createdBy' is reserved (use the top-level 'createdBy' param) - metadata: { createdBy: { augmentation: 'importer', version: '1.0' } } - }) - assertType({ - type: NounType.Person, - subtype: 'employee', - data: 'x', - // @ts-expect-error — '_rev' is reserved (system-managed revision counter) - metadata: { _rev: 7 } - }) - }) - - it('AddParams (typed brain) rejects reserved keys alongside the declared shape', () => { - interface EmployeeMeta { - dept: string - level: number - } - - assertType>({ - type: NounType.Person, - subtype: 'employee', - data: 'x', - metadata: { dept: 'eng', level: 3 } - }) - - assertType>({ - type: NounType.Person, - subtype: 'employee', - data: 'x', - // @ts-expect-error — 'confidence' is reserved even when T declares other fields - metadata: { dept: 'eng', level: 3, confidence: 0.8 } - }) - }) - - it('documented exemptions: T-declared reserved keys and index-signature shapes stay assignable', () => { - // A consumer who *explicitly* types a reserved key into their metadata - // shape keeps a working (if unwise) type — the guard exempts keyof T. - interface LegacyMeta { - confidence: number - note: string - } - assertType>({ - type: NounType.Person, - subtype: 'employee', - data: 'x', - metadata: { confidence: 0.8, note: 'declared by the consumer type' } - }) - - // Index-signature metadata types (keyof T = string) remain fully open. - assertType>>({ - type: NounType.Person, - subtype: 'employee', - data: 'x', - metadata: { anything: 'goes', confidence: 0.8 } - }) - }) - - it('UpdateParams patch rejects reserved keys but accepts partial custom patches', () => { - interface EmployeeMeta { - dept: string - level: number - } - - // Partial patch of the declared shape is legal. - assertType>({ id: 'e1', metadata: { dept: 'sales' } }) - // Untyped patch with custom fields is legal. - assertType({ id: 'e1', metadata: { status: 'reviewed', rating: 4.5 } }) - - // @ts-expect-error — 'confidence' is reserved (use the top-level 'confidence' param) - assertType({ id: 'e1', metadata: { confidence: 0.33 } }) - // @ts-expect-error — 'subtype' is reserved (use the top-level 'subtype' param) - assertType({ id: 'e1', metadata: { subtype: 'specialized' } }) - // @ts-expect-error — '_rev' is reserved (pass 'ifRev' for optimistic concurrency) - assertType({ id: 'e1', metadata: { _rev: 3 } }) - // @ts-expect-error — 'confidence' is reserved even when T declares other fields - assertType>({ id: 'e1', metadata: { confidence: 0.1 } }) - }) -}) - -describe('reserved relationship keys in metadata are compile errors', () => { - it('RelateParams rejects reserved keys but stays open for custom edge fields', () => { - assertType({ - from: 'a', - to: 'b', - type: VerbType.ReportsTo, - subtype: 'direct', - metadata: { role: 'peer', since: 2024 } - }) - - assertType({ - from: 'a', - to: 'b', - type: VerbType.ReportsTo, - subtype: 'direct', - // @ts-expect-error — 'verb' is reserved (the relationship type travels via the top-level 'type' param) - metadata: { verb: 'relatedTo' } - }) - assertType({ - from: 'a', - to: 'b', - type: VerbType.ReportsTo, - subtype: 'direct', - // @ts-expect-error — 'confidence' is reserved (use the top-level 'confidence' param) - metadata: { confidence: 0.9 } - }) - assertType({ - from: 'a', - to: 'b', - type: VerbType.ReportsTo, - subtype: 'direct', - // @ts-expect-error — 'weight' is reserved (use the top-level 'weight' param) - metadata: { weight: 0.4 } - }) - assertType({ - from: 'a', - to: 'b', - type: VerbType.ReportsTo, - subtype: 'direct', - // @ts-expect-error — 'service' is reserved (use the top-level 'service' param) - metadata: { service: 'orders' } - }) - }) - - it('UpdateRelationParams patch rejects reserved keys', () => { - assertType({ id: 'r1', metadata: { note: 'fine' } }) - - // @ts-expect-error — 'confidence' is reserved (use the top-level 'confidence' param) - assertType({ id: 'r1', metadata: { confidence: 0.5 } }) - // @ts-expect-error — 'subtype' is reserved (use the top-level 'subtype' param) - assertType({ id: 'r1', metadata: { subtype: 'dotted-line' } }) - // @ts-expect-error — 'createdAt' is reserved (system-managed) - assertType({ id: 'r1', metadata: { createdAt: 1 } }) - }) -}) - -describe('transact() operations inherit the same guard', () => { - it('TxOperation add/update/relate metadata rejects reserved keys', () => { - assertType({ - op: 'add', - type: NounType.Concept, - subtype: 'general', - data: 'tx', - metadata: { custom: 'a' } - }) - assertType({ - op: 'add', - type: NounType.Concept, - subtype: 'general', - data: 'tx', - // @ts-expect-error — 'confidence' is reserved on transact add ops too - metadata: { confidence: 0.7 } - }) - assertType({ - op: 'update', - id: 'e1', - // @ts-expect-error — 'weight' is reserved on transact update ops too - metadata: { weight: 0.2 } - }) - assertType({ - op: 'relate', - from: 'a', - to: 'b', - type: VerbType.RelatedTo, - subtype: 'colleague', - // @ts-expect-error — 'verb' is reserved on transact relate ops too - metadata: { verb: 'contains' } - }) - }) -}) diff --git a/tests/unit/utils/paramValidation.test.ts b/tests/unit/utils/paramValidation.test.ts index 4dc83554..7e5212b8 100644 --- a/tests/unit/utils/paramValidation.test.ts +++ b/tests/unit/utils/paramValidation.test.ts @@ -56,11 +56,15 @@ describe('Zero-Config Parameter Validation', () => { })).toThrow('cannot specify both query and vector') }) - it('should reject both cursor and offset', () => { + it('should refuse cursor outright — even paired with offset — as an unimplemented option', () => { + // cursor is now a typed, unconditional refusal (UnsupportedFindOptionError): + // it used to be accepted-and-ignored, only conflicting when offset was also + // given. Accepted-and-ignored died as a class — cursor refuses on its own, + // so pairing it with offset refuses too, but with the SAME message. expect(() => validateFindParams({ cursor: 'abc123', offset: 10 - })).toThrow('cannot use both cursor and offset pagination') + })).toThrow("find() option 'cursor' is not implemented") }) it('should validate vector dimensions', () => {