diff --git a/docs/concepts/field-addressing.md b/docs/concepts/field-addressing.md index 863f7474..dcae1057 100644 --- a/docs/concepts/field-addressing.md +++ b/docs/concepts/field-addressing.md @@ -114,6 +114,43 @@ Reach for the explicit spelling when it reads more clearly next to a `system.` field in the same query — for example, sorting by your own `score` while filtering on `system.confidence`. +## No special names — the write side + +The same law governs writes: + +> **Data is either in main space, where developers can use anything, or it +> is in `system.*`.** + +There are **no reserved metadata names**. A field called `confidence`, +`type`, `id`, `data`, `content`, or anything else inside your `metadata` bag +is an ordinary user field: it is stored verbatim, indexed, filterable, +sortable, aggregatable, and it survives restarts, index rebuilds, and +time-travel (`asOf`) reads exactly as written — even when an engine scalar +shares its spelling. The engine's values are written only through their +dedicated params (`confidence`, `weight`, `subtype`, `visibility`, …) and +read at `system.`; your bag can never touch them and they can never +shadow your bag. + +```typescript +const id = await brain.add({ + data: 'Ada Lovelace', + type: NounType.Person, + confidence: 0.9, // the ENGINE scalar + metadata: { confidence: 'self-rated' } // YOUR field, same spelling — both live +}) + +await brain.find({ where: { confidence: 'self-rated' } }) // finds it (yours) +await brain.find({ where: { 'system.confidence': 0.9 } }) // finds it (engine's) +``` + +The one spelling a write refuses is a metadata key that literally starts +with `system.` — the explicit address namespace cannot be forged as a user +field name. That refusal is typed and names the fix. + +Value **shape** rules still apply uniformly to every name (they are not name +carve-outs): arrays longer than 10 elements are not turned into posting-list +scalars, and very long values are indexed by hash. + ## Refusal semantics A name that resolves to neither your metadata nor a system scalar is a typed @@ -190,7 +227,6 @@ against the new rule. ## Where to go next -- [Consistency Model](./consistency-model.md) — the separate (and - longer-standing) contract for *reserved* fields: which names may never - appear inside a `metadata` bag at write time, distinct from this page's +- [Consistency Model](./consistency-model.md) — visibility tiers, revision + counters, and the rest of the read/write contract this page's read-time addressing rule. diff --git a/src/brainy.ts b/src/brainy.ts index 3dd8ef93..600e4474 100644 --- a/src/brainy.ts +++ b/src/brainy.ts @@ -148,7 +148,9 @@ import { import { NounType, VerbType, TypeUtils } from './types/graphTypes.js' import { splitNounMetadataRecord, - splitVerbMetadataRecord + splitVerbMetadataRecord, + buildNounMetadataRecord, + buildVerbMetadataRecord } from './types/reservedFields.js' import { BrainyInterface } from './types/brainyInterface.js' import type { IntegrationHub, IntegrationHubConfig } from './integrations/core/IntegrationHub.js' @@ -746,6 +748,21 @@ export class Brainy implements BrainyInterface { private lazyRebuildPromise: Promise | null = null constructor(config?: BrainyConfig) { + // The reserved-field write policy died with the field-addressing law: + // every metadata name is the user's now (engine scalars write via their + // dedicated params and read at `system.*`), so there is nothing left for + // the policy to govern. A config still passing it refuses loudly rather + // than being silently ignored. + if (config && 'reservedFieldPolicy' in (config as Record)) { + throw new Error( + `reservedFieldPolicy was removed by the field-addressing law: metadata field ` + + `names are never reserved anymore — every name in the metadata bag is the ` + + `user's and works like any other field. Set engine scalars via their ` + + `dedicated params (confidence, weight, subtype, …) and query them as ` + + `system.. Remove the reservedFieldPolicy option.` + ) + } + // Normalize configuration with defaults this.config = this.normalizeConfig(config) @@ -2018,12 +2035,6 @@ export class Brainy implements BrainyInterface { // Zero-config validation (static import for performance) validateAddParams(params) - // Reserved fields arriving via the metadata bag (untyped callers — the - // compile-time guard stops TypeScript callers) are normalized to their - // canonical top-level location BEFORE any enforcement runs, so a - // remapped subtype participates in subtype-pairing enforcement and the - // indexed metadata bag carries only custom fields. - params = this.remapReservedAddMetadata(params) // Tracked-field vocabulary enforcement (Layer 2). Walks both bags so a // tracked field declared at top level (e.g. 'subtype') and one declared in @@ -2095,28 +2106,33 @@ export class Brainy implements BrainyInterface { ) } - // Prepare metadata for storage - // data is stored opaquely in the 'data' field - NOT spread into top-level metadata. - // Only metadata fields are queryable via find({ where }). - const storageMetadata = { - ...params.metadata, - // Preserve the caller's original (non-UUID) id when normalized, so reads - // can surface it. A real UUID passes through with no _originalId. - ...(originalId !== undefined && { [ORIGINAL_ID_KEY]: originalId }), - data: params.data, - noun: params.type, - ...(params.subtype !== undefined && { subtype: params.subtype }), - // visibility: stored only when not 'public' (absent === public, keeps records lean) - ...(params.visibility !== undefined && - params.visibility !== 'public' && { visibility: params.visibility }), - service: params.service, - createdAt: Date.now(), - updatedAt: Date.now(), - _rev: 1, - ...(params.confidence !== undefined && { confidence: params.confidence }), - ...(params.weight !== undefined && { weight: params.weight }), - ...(params.createdBy && { createdBy: params.createdBy }) - } + // Prepare metadata for storage: a v2 nested-bag record — engine fields + // top-level, the user's bag nested VERBATIM (any name, including engine + // spellings like `confidence` or `type`, is the user's and survives + // faithfully; the field-addressing law). + const storageMetadata = buildNounMetadataRecord( + { + data: params.data, + noun: params.type, + ...(params.subtype !== undefined && { subtype: params.subtype }), + // visibility: stored only when not 'public' (absent === public, keeps records lean) + ...(params.visibility !== undefined && + params.visibility !== 'public' && { visibility: params.visibility }), + service: params.service, + createdAt: Date.now(), + updatedAt: Date.now(), + _rev: 1, + ...(params.confidence !== undefined && { confidence: params.confidence }), + ...(params.weight !== undefined && { weight: params.weight }), + ...(params.createdBy && { createdBy: params.createdBy }) + }, + { + ...params.metadata, + // Preserve the caller's original (non-UUID) id when normalized, so reads + // can surface it. A real UUID passes through with no _originalId. + ...(originalId !== undefined && { [ORIGINAL_ID_KEY]: originalId }) + } + ) // Build entity structure for indexing (NEW - with top-level fields) // Optional fields must use conditional spreading to match storageMetadata exactly. @@ -2627,320 +2643,6 @@ export class Brainy implements BrainyInterface { return entity } - /** One-shot registry for reserved-field warnings (per process, per method+field). */ - private static warnedReservedFields = new Set() - - /** - * @description Resolve the human-readable "correct write path" guidance for a - * reserved field on a given write method. Single source of truth shared by the - * `'throw'` (Error message) and `'warn'` (one-shot warning) paths so the two - * never drift. The trio `confidence` / `weight` / `subtype` and the - * add()/relate()-time fields `service` / `createdBy` / `visibility` map to a - * dedicated param; everything else is system-managed. - * @param method - The public write method the bag arrived through. - * @param field - The reserved field name found in the metadata bag. - * @returns Guidance naming the correct way to set the field. - */ - private reservedWritePath( - method: 'add' | 'update' | 'relate' | 'updateRelation', - field: string - ): string { - const typeParam = "the top-level 'type' param" - switch (field) { - case 'noun': - case 'verb': - return typeParam - case 'data': - return "the top-level 'data' param" - case 'confidence': - return "the 'confidence' param" - case 'weight': - return "the 'weight' param" - case 'subtype': - return "the 'subtype' param" - case 'visibility': - return "the 'visibility' param ('public' | 'internal')" - case 'service': - return method === 'add' - ? "the 'service' param of add()" - : method === 'relate' - ? "the 'service' param of relate()" - : 'nothing — service is fixed at create time' - case 'createdBy': - return method === 'add' - ? "the 'createdBy' param of add()" - : 'nothing — createdBy is system-managed' - case 'createdAt': - return 'nothing — creation time is set automatically' - case 'updatedAt': - return 'nothing — set automatically on every write' - case '_rev': - return method === 'update' - ? "the 'ifRev' param for optimistic concurrency" - : 'nothing — revisions are system-managed' - default: - return 'a dedicated top-level param' - } - } - - /** - * @description Enforce {@link BrainyConfig.reservedFieldPolicy} for reserved - * fields found inside a metadata bag. Called by every write-path remap once - * the bag has been split and at least one reserved key is present. - * - * - `'throw'` (default): throw a clear Error naming every offending key and - * its correct write path. The caller never reaches the remap. - * - `'warn'`: emit a ONE-SHOT (per method+field, per process) warning for - * EVERY reserved key found — both the user-mutable fields that are about to - * be remapped and the system-managed fields that are about to be dropped — - * then fall through to the legacy remap. - * - `'remap'`: silent legacy remap, no warning. - * - * @param method - The public write method the bag arrived through. - * @param reserved - The reserved half of the split metadata bag (non-empty). - * @param reservedListName - `'RESERVED_ENTITY_FIELDS'` or - * `'RESERVED_RELATION_FIELDS'` — named in the thrown Error for discoverability. - * @returns `true` when the caller should proceed with the legacy remap - * (`'warn'` / `'remap'`); `'throw'` never returns (it throws first). - * @throws {Error} When the policy is `'throw'` and any reserved key is present. - */ - private enforceReservedPolicy( - method: 'add' | 'update' | 'relate' | 'updateRelation', - reserved: Partial>, - reservedListName: 'RESERVED_ENTITY_FIELDS' | 'RESERVED_RELATION_FIELDS' - ): boolean { - const policy = this.config.reservedFieldPolicy ?? 'throw' - const keys = Object.keys(reserved) - if (keys.length === 0) return true - - if (policy === 'throw') { - const detail = keys - .map((k) => { - const path = this.reservedWritePath(method, k) - // System-managed fields resolve to a "nothing — …" sentinel; phrase - // those as "is system-managed" rather than "pass it as the nothing". - return path.startsWith('nothing') - ? `metadata.${k} is a reserved field (${path.replace(/^nothing\s*—\s*/, '')}) and cannot be set through ${method}()` - : `metadata.${k} is a reserved field — pass it as ${path} to ${method}()` - }) - .join('; ') - throw new Error( - `${detail} (reserved: see ${reservedListName}). ` + - `Set reservedFieldPolicy:'remap' to opt into legacy remapping, ` + - `or reservedFieldPolicy:'warn' to remap with a warning.` - ) - } - - if (policy === 'warn') { - // One-shot warning for EVERY reserved key (today only system-managed ones - // warn — this closes that gap so user-mutable remaps are visible too). - for (const k of keys) { - this.warnReservedRemapped(method, k, this.reservedWritePath(method, k)) - } - } - - // 'warn' and 'remap' both fall through to the legacy remap. - return true - } - - /** - * @description One-shot (per method+field, per process) warning that a - * reserved field arrived inside a metadata bag under the `'warn'` policy. The - * wording is neutral on "remapped vs dropped" — `reservedWritePath()` already - * tells the caller where the value goes (a dedicated param, or "nothing"). - * @param method - The public write method the bag arrived through. - * @param field - The reserved field name found in the bag. - * @param rightPath - Guidance naming the correct write path. - */ - private warnReservedRemapped(method: string, field: string, rightPath: string): void { - const key = `${method}:${field}` - if (Brainy.warnedReservedFields.has(key)) return - Brainy.warnedReservedFields.add(key) - // System-managed fields resolve to a "nothing — …" sentinel; phrase the - // guidance so it reads cleanly in both the remapped and dropped cases. - const guidance = rightPath.startsWith('nothing') - ? `it is ${rightPath.replace(/^nothing\s*—\s*/, '')} and was dropped` - : `set it via ${rightPath} instead` - prodLog.warn( - `[brainy] ${method}(): '${field}' is a reserved field and was found inside the ` + - `metadata bag — ${guidance}. (Legacy remap applied because ` + - `reservedFieldPolicy is 'warn'. This warning is shown once per field per process.)` - ) - } - - /** - * @description Normalize an `add()` params object with respect to - * Brainy-reserved fields arriving inside `metadata` (untyped callers only — - * the compile-time guard on `AddParams.metadata` stops TypeScript callers). - * Governed by {@link BrainyConfig.reservedFieldPolicy} (default `'throw'`): - * `'throw'` rejects the write naming the offending key(s); `'warn'`/`'remap'` - * fall through to the legacy remap, where fields with a dedicated `add()` - * param (`confidence`, `weight`, `subtype`, `visibility`, `service`, - * `createdBy`) are remapped to that param unless the caller also passed it - * explicitly (top-level wins) and system-managed fields (`noun`, `data`, - * `createdAt`, `updatedAt`, `_rev`) are dropped. A remapped `subtype` flows - * through subtype-pairing enforcement exactly like a top-level one. - * @param params - The caller's add params (not mutated). - * @returns Params with reserved fields normalized out of `metadata`. - * @throws {Error} When `reservedFieldPolicy` is `'throw'` and the bag carries a reserved key. - */ - private remapReservedAddMetadata(params: AddParams): AddParams { - const bag = params.metadata as Record | undefined - if (!bag || typeof bag !== 'object') return params - const { reserved, custom } = splitNounMetadataRecord(bag) - if (Object.keys(reserved).length === 0) return params - - // Policy gate: 'throw' (default) throws here; 'warn' warns once per key then - // remaps; 'remap' silently remaps. (Throw never returns.) - this.enforceReservedPolicy('add', reserved, 'RESERVED_ENTITY_FIELDS') - - const createdBy = reserved.createdBy as { augmentation?: unknown; version?: unknown } | undefined - const createdByValid = - typeof createdBy === 'object' && - createdBy !== null && - typeof createdBy.augmentation === 'string' && - typeof createdBy.version === 'string' - - return { - ...params, - metadata: custom as AddParams['metadata'], - ...(params.confidence === undefined && - typeof reserved.confidence === 'number' && { confidence: reserved.confidence }), - ...(params.weight === undefined && - typeof reserved.weight === 'number' && { weight: reserved.weight }), - ...(params.subtype === undefined && - typeof reserved.subtype === 'string' && { subtype: reserved.subtype }), - ...(params.visibility === undefined && - (reserved.visibility === 'public' || reserved.visibility === 'internal') && { - visibility: reserved.visibility as 'public' | 'internal' - }), - ...(params.service === undefined && - typeof reserved.service === 'string' && { service: reserved.service }), - ...(params.createdBy === undefined && - createdByValid && { createdBy: createdBy as { augmentation: string; version: string } }) - } - } - - /** - * @description Normalize an `update()` params object with respect to - * Brainy-reserved fields arriving inside the metadata patch — the `update()` - * mirror of {@link remapReservedAddMetadata}, closing the historical trap - * where `add({metadata:{confidence}})` lifted the field but - * `update({metadata:{confidence}})` silently dropped it (the patch value - * survived the merge and was then clobbered by the preserve-existing - * spread; a production consumer's confidence-evolution writes no-oped until - * read back). Governed by {@link BrainyConfig.reservedFieldPolicy} (default - * `'throw'`): `'throw'` rejects the write; `'warn'`/`'remap'` remap - * user-mutable fields (`confidence`, `weight`, `subtype`) to their dedicated - * param unless the caller also passed it (top-level wins) and drop everything - * else (`noun`, `data`, `createdAt`, `updatedAt`, `service`, `createdBy`, - * `_rev`) as system-managed or fixed at `add()` time. - * @param params - The caller's update params (not mutated). - * @returns Params with reserved fields normalized out of `metadata`. - * @throws {Error} When `reservedFieldPolicy` is `'throw'` and the bag carries a reserved key. - */ - private remapReservedUpdateMetadata(params: UpdateParams): UpdateParams { - const bag = params.metadata as Record | undefined - if (!bag || typeof bag !== 'object') return params - const { reserved, custom } = splitNounMetadataRecord(bag) - if (Object.keys(reserved).length === 0) return params - - // Policy gate: 'throw' (default) throws; 'warn' warns once per key then - // remaps; 'remap' silently remaps. - this.enforceReservedPolicy('update', reserved, 'RESERVED_ENTITY_FIELDS') - - return { - ...params, - metadata: custom as UpdateParams['metadata'], - ...(params.confidence === undefined && - typeof reserved.confidence === 'number' && { confidence: reserved.confidence }), - ...(params.weight === undefined && - typeof reserved.weight === 'number' && { weight: reserved.weight }), - ...(params.subtype === undefined && - typeof reserved.subtype === 'string' && { subtype: reserved.subtype }) - } - } - - /** - * @description Normalize a `relate()` params object with respect to - * Brainy-reserved fields arriving inside `metadata` — the relationship - * mirror of {@link remapReservedAddMetadata}. Governed by - * {@link BrainyConfig.reservedFieldPolicy} (default `'throw'`): `'throw'` - * rejects the write; `'warn'`/`'remap'` remap fields with a dedicated - * `relate()` param (`confidence`, `weight`, `subtype`, `visibility`, - * `service`) to that param (top-level wins) and drop system-managed fields - * (`verb`, `data`, `createdAt`, `updatedAt`, `createdBy`, `_rev`). - * @param params - The caller's relate params (not mutated). - * @returns Params with reserved fields normalized out of `metadata`. - * @throws {Error} When `reservedFieldPolicy` is `'throw'` and the bag carries a reserved key. - */ - private remapReservedRelateMetadata(params: RelateParams): RelateParams { - const bag = params.metadata as Record | undefined - if (!bag || typeof bag !== 'object') return params - const { reserved, custom } = splitVerbMetadataRecord(bag) - if (Object.keys(reserved).length === 0) return params - - // Policy gate: 'throw' (default) throws; 'warn' warns once per key then - // remaps; 'remap' silently remaps. - this.enforceReservedPolicy('relate', reserved, 'RESERVED_RELATION_FIELDS') - - return { - ...params, - metadata: custom as RelateParams['metadata'], - ...(params.confidence === undefined && - typeof reserved.confidence === 'number' && { confidence: reserved.confidence }), - ...(params.weight === undefined && - typeof reserved.weight === 'number' && { weight: reserved.weight }), - ...(params.subtype === undefined && - typeof reserved.subtype === 'string' && { subtype: reserved.subtype }), - ...(params.visibility === undefined && - (reserved.visibility === 'public' || reserved.visibility === 'internal') && { - visibility: reserved.visibility as 'public' | 'internal' - }), - ...(params.service === undefined && - typeof reserved.service === 'string' && { service: reserved.service }) - } - } - - /** - * @description Normalize an `updateRelation()` params object with respect - * to Brainy-reserved fields arriving inside the metadata patch — the - * relationship mirror of {@link remapReservedUpdateMetadata}. Governed by - * {@link BrainyConfig.reservedFieldPolicy} (default `'throw'`): `'throw'` - * rejects the write; `'warn'`/`'remap'` remap user-mutable fields - * (`confidence`, `weight`, `subtype`, `visibility`) to their dedicated param - * (top-level wins) and drop everything else. - * @param params - The caller's update-relation params (not mutated). - * @returns Params with reserved fields normalized out of `metadata`. - * @throws {Error} When `reservedFieldPolicy` is `'throw'` and the bag carries a reserved key. - */ - private remapReservedUpdateRelationMetadata( - params: UpdateRelationParams - ): UpdateRelationParams { - const bag = params.metadata as Record | undefined - if (!bag || typeof bag !== 'object') return params - const { reserved, custom } = splitVerbMetadataRecord(bag) - if (Object.keys(reserved).length === 0) return params - - // Policy gate: 'throw' (default) throws; 'warn' warns once per key then - // remaps; 'remap' silently remaps. - this.enforceReservedPolicy('updateRelation', reserved, 'RESERVED_RELATION_FIELDS') - - return { - ...params, - metadata: custom as UpdateRelationParams['metadata'], - ...(params.confidence === undefined && - typeof reserved.confidence === 'number' && { confidence: reserved.confidence }), - ...(params.weight === undefined && - typeof reserved.weight === 'number' && { weight: reserved.weight }), - ...(params.subtype === undefined && - typeof reserved.subtype === 'string' && { subtype: reserved.subtype }), - ...(params.visibility === undefined && - (reserved.visibility === 'public' || reserved.visibility === 'internal') && { - visibility: reserved.visibility as 'public' | 'internal' - }) - } - } /** * Update an existing entity @@ -3006,12 +2708,6 @@ export class Brainy implements BrainyInterface { // Reserved fields arriving via the metadata patch are remapped to their // canonical top-level location, mirroring add()'s lift. Without this the // patch value survived the merge but was then clobbered by the - // preserve-existing spreads below — a silent no-op consumers could only - // detect by reading values back. User-mutable fields (confidence, - // weight, subtype) remap unless the same field was also passed top-level - // (top-level wins); system-managed fields are dropped with a one-shot - // warning naming the right path. - params = this.remapReservedUpdateMetadata(params) // Tracked-field vocabulary enforcement (Layer 2). Same as add() — the // metadata bag carries fields registered via trackField(), and subtype is @@ -3078,31 +2774,33 @@ export class Brainy implements BrainyInterface { ? { ...existing.metadata, ...params.metadata } : params.metadata || existing.metadata - // Prepare updated metadata object - // data is stored opaquely in the 'data' field - NOT spread into top-level metadata. - const updatedMetadata = { - ...newMetadata, - data: params.data !== undefined ? params.data : existing.data, - noun: params.type || existing.type, - service: existing.service, - createdAt: existing.createdAt, - updatedAt: Date.now(), - _rev: currentRev + 1, - // Update confidence and weight if provided, otherwise preserve existing - ...(params.confidence !== undefined && { confidence: params.confidence }), - ...(params.weight !== undefined && { weight: params.weight }), - ...(params.confidence === undefined && existing.confidence !== undefined && { confidence: existing.confidence }), - ...(params.weight === undefined && existing.weight !== undefined && { weight: existing.weight }), - // Update subtype if provided, otherwise preserve existing - ...(params.subtype !== undefined && { subtype: params.subtype }), - ...(params.subtype === undefined && existing.subtype !== undefined && { subtype: existing.subtype }), - // Visibility: take the new value if provided, else preserve existing. Stored only - // when the effective value is not 'public' (absent === public, keeps records lean). - // A change to 'public' therefore drops the field entirely. - ...(((params.visibility ?? existing.visibility) ?? 'public') !== 'public' && { - visibility: params.visibility ?? existing.visibility - }) - } + // Prepare the updated v2 nested-bag record: engine fields top-level, + // the merged user bag nested verbatim (collider names stay the user's). + const updatedMetadata = buildNounMetadataRecord( + { + data: params.data !== undefined ? params.data : existing.data, + noun: params.type || existing.type, + service: existing.service, + createdAt: existing.createdAt, + updatedAt: Date.now(), + _rev: currentRev + 1, + // Update confidence and weight if provided, otherwise preserve existing + ...(params.confidence !== undefined && { confidence: params.confidence }), + ...(params.weight !== undefined && { weight: params.weight }), + ...(params.confidence === undefined && existing.confidence !== undefined && { confidence: existing.confidence }), + ...(params.weight === undefined && existing.weight !== undefined && { weight: existing.weight }), + // Update subtype if provided, otherwise preserve existing + ...(params.subtype !== undefined && { subtype: params.subtype }), + ...(params.subtype === undefined && existing.subtype !== undefined && { subtype: existing.subtype }), + // Visibility: take the new value if provided, else preserve existing. Stored only + // when the effective value is not 'public' (absent === public, keeps records lean). + // A change to 'public' therefore drops the field entirely. + ...(((params.visibility ?? existing.visibility) ?? 'public') !== 'public' && { + visibility: params.visibility ?? existing.visibility + }) + }, + newMetadata as Record + ) // Build entity structure for metadata index (with top-level fields). // No `level`: engine plumbing never enters the indexing view (it @@ -4043,9 +3741,6 @@ export class Brainy implements BrainyInterface { // engine-minted UUID — relation ids are never caller-supplied here.) params = { ...params, from: resolveEntityId(params.from), to: resolveEntityId(params.to) } - // Reserved fields arriving via the metadata bag are normalized to their - // canonical top-level params before enforcement — mirror of add()'s lift. - params = this.remapReservedRelateMetadata(params) // Subtype pairing enforcement (Layer 3 — 7.30.0). Per-type rules registered // via brain.requireSubtype() compose with the brain-wide strict-mode flag. @@ -4097,25 +3792,28 @@ export class Brainy implements BrainyInterface { (v, i) => (v + toEntity.vector[i]) / 2 ) - // Prepare verb metadata - // User metadata spread FIRST, then system fields ALWAYS win (prevents collision) + // Prepare verb metadata: a v2 nested-bag record — engine fields + // top-level, the user's edge bag nested verbatim (any name is the + // user's; the field-addressing law). // One timestamp for both createdAt and updatedAt so a never-updated edge reports a // stable updatedAt (=== createdAt) instead of a fresh Date.now() fabricated per read. const relateTs = Date.now() - const verbMetadata = { - ...(params.metadata || {}), - verb: params.type, - ...(params.subtype !== undefined && { subtype: params.subtype }), - // visibility: stored only when not 'public' (absent === public, keeps records lean) - ...(params.visibility !== undefined && - params.visibility !== 'public' && { visibility: params.visibility }), - weight: params.weight ?? 1.0, - ...(params.confidence !== undefined && { confidence: params.confidence }), - ...(params.service !== undefined && { service: params.service }), - createdAt: relateTs, - updatedAt: relateTs, - ...(params.data !== undefined && { data: params.data }) - } + const verbMetadata = buildVerbMetadataRecord( + { + verb: params.type, + ...(params.subtype !== undefined && { subtype: params.subtype }), + // visibility: stored only when not 'public' (absent === public, keeps records lean) + ...(params.visibility !== undefined && + params.visibility !== 'public' && { visibility: params.visibility }), + weight: params.weight ?? 1.0, + ...(params.confidence !== undefined && { confidence: params.confidence }), + ...(params.service !== undefined && { service: params.service }), + createdAt: relateTs, + updatedAt: relateTs, + ...(params.data !== undefined && { data: params.data }) + }, + (params.metadata as Record) || {} + ) // Save to storage (vector and metadata separately) const verb: GraphVerb = { @@ -4347,9 +4045,6 @@ export class Brainy implements BrainyInterface { validateUpdateRelationParams(params) - // Reserved fields arriving via the metadata patch are remapped to their - // canonical top-level params — mirror of update()'s normalization. - params = this.remapReservedUpdateRelationMetadata(params) const existing = await this.storage.getVerb(params.id) if (!existing) { @@ -4378,32 +4073,36 @@ export class Brainy implements BrainyInterface { ? { ...(existingRec.metadata || {}), ...(params.metadata || {}) } : params.metadata || existingRec.metadata - // Build updated stored metadata. System fields ALWAYS win — same shape as relate(). - const updatedMetadata = { - ...newMetadata, - verb: newVerbType, - ...(params.subtype !== undefined - ? { subtype: params.subtype } - : existingRec.subtype !== undefined && { subtype: existingRec.subtype }), - // Visibility: new value if provided, else preserve existing; stored only when the - // effective value is not 'public' (a change to 'public' drops the field). - ...(((params.visibility ?? existingRec.visibility) ?? 'public') !== 'public' && { - visibility: params.visibility ?? existingRec.visibility - }), - weight: params.weight ?? existingRec.weight ?? 1.0, - ...(params.confidence !== undefined - ? { confidence: params.confidence } - : existingRec.confidence !== undefined && { confidence: existingRec.confidence }), - // service/createdBy are fixed at relate() time — always carried forward - // (omitting them here silently erased them on every updateRelation()). - ...(existingRec.service !== undefined && { service: existingRec.service }), - ...(existingRec.createdBy !== undefined && { createdBy: existingRec.createdBy }), - createdAt: existingRec.createdAt, - updatedAt: Date.now(), - ...(params.data !== undefined - ? { data: params.data } - : existingRec.data !== undefined && { data: existingRec.data }) - } + // Build the updated stored record: v2 nested-bag — engine fields + // top-level, the merged user bag nested verbatim (mirror of update()). + const updatedWeight = params.weight ?? existingRec.weight ?? 1.0 + const updatedData = + params.data !== undefined ? params.data : existingRec.data + const updatedMetadata = buildVerbMetadataRecord( + { + verb: newVerbType, + ...(params.subtype !== undefined + ? { subtype: params.subtype } + : existingRec.subtype !== undefined && { subtype: existingRec.subtype }), + // Visibility: new value if provided, else preserve existing; stored only when the + // effective value is not 'public' (a change to 'public' drops the field). + ...(((params.visibility ?? existingRec.visibility) ?? 'public') !== 'public' && { + visibility: params.visibility ?? existingRec.visibility + }), + weight: updatedWeight, + ...(params.confidence !== undefined + ? { confidence: params.confidence } + : existingRec.confidence !== undefined && { confidence: existingRec.confidence }), + // service/createdBy are fixed at relate() time — always carried forward + // (omitting them here silently erased them on every updateRelation()). + ...(existingRec.service !== undefined && { service: existingRec.service }), + ...(existingRec.createdBy !== undefined && { createdBy: existingRec.createdBy }), + createdAt: existingRec.createdAt, + updatedAt: Date.now(), + ...(updatedData !== undefined && { data: updatedData }) + }, + newMetadata as Record + ) // Build the verb view used by the graph index — top-level fields mirror relate()'s. const verbForIndex: GraphVerb = { @@ -4419,9 +4118,9 @@ export class Brainy implements BrainyInterface { ...(((params.visibility ?? existingRec.visibility) ?? 'public') !== 'public' && { visibility: params.visibility ?? existingRec.visibility }), - weight: updatedMetadata.weight, + weight: updatedWeight, metadata: newMetadata, - data: updatedMetadata.data, + data: updatedData, createdAt: existingRec.createdAt } @@ -6027,8 +5726,12 @@ export class Brainy implements BrainyInterface { ): Promise> { const excluded = this.excludedVisibilityTiers(params) if (!excluded) return new Set() + // 'system.visibility' — the engine scalar's frozen address. A bare + // 'visibility' key would address the USER's metadata bag under the + // field-addressing law and silently hide nothing (VFS/system entities + // would leak into every default read). const ids = await this.metadataIndex.getIdsForFilter({ - visibility: excluded.length === 1 ? excluded[0] : { oneOf: excluded } + 'system.visibility': excluded.length === 1 ? excluded[0] : { oneOf: excluded } }) return new Set(ids) } @@ -9282,10 +8985,7 @@ export class Brainy implements BrainyInterface { ): Promise { const { op: _discriminator, ...rawParams } = op validateAddParams(rawParams as AddParams) - // Same reserved-field normalization as add() — the metadata bag is - // cleaned BEFORE enforcement so a remapped subtype participates in - // subtype-pairing enforcement and only custom fields reach the index. - const params = this.remapReservedAddMetadata(rawParams as AddParams) + const params = rawParams as AddParams this.enforceTrackedFieldValues(params.metadata as Record | undefined, 'metadata') this.enforceTrackedFieldValues({ subtype: params.subtype } as Record, 'top-level') this.enforceSubtypeOnAdd('add', params.type, params.subtype, params.metadata) @@ -9360,25 +9060,31 @@ export class Brainy implements BrainyInterface { plan.createdNouns.add(id) const now = Date.now() - const storageMetadata = { - ...params.metadata, - // Preserve the caller's original (non-UUID) id when normalized — mirror - // of add(). A real UUID passes through with no _originalId. - ...(originalId !== undefined && { [ORIGINAL_ID_KEY]: originalId }), - data: params.data, - noun: params.type, - ...(params.subtype !== undefined && { subtype: params.subtype }), - // visibility: stored only when not 'public' (absent === public, keeps records lean) - ...(params.visibility !== undefined && - params.visibility !== 'public' && { visibility: params.visibility }), - service: params.service, - createdAt: now, - updatedAt: now, - _rev: 1, - ...(params.confidence !== undefined && { confidence: params.confidence }), - ...(params.weight !== undefined && { weight: params.weight }), - ...(params.createdBy && { createdBy: params.createdBy }) - } + // v2 nested-bag record — mirror of add(): engine fields top-level, the + // user's bag nested verbatim (collider names stay the user's). + const storageMetadata = buildNounMetadataRecord( + { + data: params.data, + noun: params.type, + ...(params.subtype !== undefined && { subtype: params.subtype }), + // visibility: stored only when not 'public' (absent === public, keeps records lean) + ...(params.visibility !== undefined && + params.visibility !== 'public' && { visibility: params.visibility }), + service: params.service, + createdAt: now, + updatedAt: now, + _rev: 1, + ...(params.confidence !== undefined && { confidence: params.confidence }), + ...(params.weight !== undefined && { weight: params.weight }), + ...(params.createdBy && { createdBy: params.createdBy }) + }, + { + ...params.metadata, + // Preserve the caller's original (non-UUID) id when normalized — mirror + // of add(). A real UUID passes through with no _originalId. + ...(originalId !== undefined && { [ORIGINAL_ID_KEY]: originalId }) + } + ) const entityForIndexing = { id, vector, @@ -9441,10 +9147,7 @@ export class Brainy implements BrainyInterface { ): Promise { const { op: _discriminator, ...rawParams } = op validateUpdateParams(rawParams as UpdateParams) - // Same reserved-field normalization as update() — user-mutable fields - // remap to their dedicated param (top-level wins), system-managed fields - // drop with a one-shot warning. - const params = this.remapReservedUpdateMetadata(rawParams as UpdateParams) + const params = rawParams as UpdateParams // Id normalization (8.0) — mirror of update(): a natural key resolves to the // canonical UUID add() stored. A real UUID passes through. params.id = resolveEntityId(params.id) @@ -9496,29 +9199,33 @@ export class Brainy implements BrainyInterface { ? { ...existing.metadata, ...params.metadata } : params.metadata || existing.metadata const now = Date.now() - const updatedMetadata = { - ...newMetadata, - data: params.data !== undefined ? params.data : existing.data, - noun: params.type || existing.type, - service: existing.service, - createdAt: existing.createdAt, - updatedAt: now, - _rev: currentRev + 1, - ...(params.confidence !== undefined && { confidence: params.confidence }), - ...(params.weight !== undefined && { weight: params.weight }), - ...(params.confidence === undefined && - existing.confidence !== undefined && { confidence: existing.confidence }), - ...(params.weight === undefined && - existing.weight !== undefined && { weight: existing.weight }), - ...(params.subtype !== undefined && { subtype: params.subtype }), - ...(params.subtype === undefined && - existing.subtype !== undefined && { subtype: existing.subtype }), - // Visibility: new value if provided, else preserve existing; stored only when the - // effective value is not 'public' (a change to 'public' drops the field). - ...(((params.visibility ?? existing.visibility) ?? 'public') !== 'public' && { - visibility: params.visibility ?? existing.visibility - }) - } + // v2 nested-bag record — mirror of update(): engine fields top-level, + // the merged user bag nested verbatim. + const updatedMetadata = buildNounMetadataRecord( + { + data: params.data !== undefined ? params.data : existing.data, + noun: params.type || existing.type, + service: existing.service, + createdAt: existing.createdAt, + updatedAt: now, + _rev: currentRev + 1, + ...(params.confidence !== undefined && { confidence: params.confidence }), + ...(params.weight !== undefined && { weight: params.weight }), + ...(params.confidence === undefined && + existing.confidence !== undefined && { confidence: existing.confidence }), + ...(params.weight === undefined && + existing.weight !== undefined && { weight: existing.weight }), + ...(params.subtype !== undefined && { subtype: params.subtype }), + ...(params.subtype === undefined && + existing.subtype !== undefined && { subtype: existing.subtype }), + // Visibility: new value if provided, else preserve existing; stored only when the + // effective value is not 'public' (a change to 'public' drops the field). + ...(((params.visibility ?? existing.visibility) ?? 'public') !== 'public' && { + visibility: params.visibility ?? existing.visibility + }) + }, + newMetadata as Record + ) // Register for the authoritative under-mutex CAS re-verify + rev re-stamp // (see PlannedTransact.casUpdates). The staged UpdateNounMetadataOperation @@ -9739,8 +9446,7 @@ export class Brainy implements BrainyInterface { ): Promise { const { op: _discriminator, ...rawParams } = op validateRelateParams(rawParams as RelateParams) - // Same reserved-field normalization as relate(). - const params = this.remapReservedRelateMetadata(rawParams as RelateParams) + const params = rawParams as RelateParams // Id normalization (8.0) — mirror of relate(): resolve BOTH endpoints to the // canonical UUID add() stored, so a relate op may reference either side by // natural key. Real UUIDs pass through. (Relationship ids are engine-minted.) @@ -9790,19 +9496,23 @@ export class Brainy implements BrainyInterface { const id = uuidv4() const relationVector = fromEntity.vector.map((v, i) => (v + toEntity.vector[i]) / 2) const now = Date.now() - const verbMetadata = { - ...(params.metadata || {}), - verb: params.type, - ...(params.subtype !== undefined && { subtype: params.subtype }), - // visibility: stored only when not 'public' (absent === public, keeps records lean) - ...(params.visibility !== undefined && - params.visibility !== 'public' && { visibility: params.visibility }), - weight: params.weight ?? 1.0, - ...(params.confidence !== undefined && { confidence: params.confidence }), - ...(params.service !== undefined && { service: params.service }), - createdAt: now, - ...(params.data !== undefined && { data: params.data }) - } + // v2 nested-bag record — mirror of relate(): engine fields top-level, + // the user's edge bag nested verbatim. + const verbMetadata = buildVerbMetadataRecord( + { + verb: params.type, + ...(params.subtype !== undefined && { subtype: params.subtype }), + // visibility: stored only when not 'public' (absent === public, keeps records lean) + ...(params.visibility !== undefined && + params.visibility !== 'public' && { visibility: params.visibility }), + weight: params.weight ?? 1.0, + ...(params.confidence !== undefined && { confidence: params.confidence }), + ...(params.service !== undefined && { service: params.service }), + createdAt: now, + ...(params.data !== undefined && { data: params.data }) + }, + (params.metadata as Record) || {} + ) const verb: GraphVerb = { id, vector: relationVector, @@ -15115,12 +14825,7 @@ export class Brainy implements BrainyInterface { requireSubtype: config?.requireSubtype ?? true, // Multi-process safety mode: config?.mode ?? 'writer', - force: config?.force ?? false, - // Reserved-field-in-metadata-bag policy (8.0 — no silent failures). - // Default 'throw': an untyped caller that smuggles a reserved key past - // the compile guard gets a loud Error naming the correct write path. - // 'warn' = remap + one-shot warning per key; 'remap' = legacy silent remap. - reservedFieldPolicy: config?.reservedFieldPolicy ?? 'throw' + force: config?.force ?? false } } diff --git a/src/db/db.ts b/src/db/db.ts index c5cbad8b..68428a7c 100644 --- a/src/db/db.ts +++ b/src/db/db.ts @@ -59,10 +59,6 @@ import type { import type { StorageAdapter } from '../coreTypes.js' import { exportGraph } from './portableGraph.js' import type { ExportSelector, ExportOptions, PortableGraph } from './portableGraph.js' -import { - splitNounMetadataRecord, - splitVerbMetadataRecord -} from '../types/reservedFields.js' import { v4 as uuidv4 } from '../universal/uuid.js' import { coerceNewEntityId, resolveEntityId, ORIGINAL_ID_KEY } from '../utils/idNormalization.js' import { EntityNotFoundError } from '../errors/notFound.js' @@ -705,23 +701,15 @@ export class Db { for (const op of ops) { switch (op.op) { case 'add': { - // Reserved-field normalization — mirror of the brain.transact() - // write path: user-settable fields lift to their dedicated field - // (top-level wins), system-managed fields drop, and the entity's - // metadata bag carries ONLY custom fields. Speculative views skip - // the one-shot warnings — committing the same ops through - // `brain.transact()` warns on the real write path. - const { reserved, custom } = splitNounMetadataRecord( - op.metadata as Record | undefined - ) - const confidence = - op.confidence ?? (typeof reserved.confidence === 'number' ? reserved.confidence : undefined) - const weight = - op.weight ?? (typeof reserved.weight === 'number' ? reserved.weight : undefined) - const subtype = - op.subtype ?? (typeof reserved.subtype === 'string' ? reserved.subtype : undefined) - const service = - op.service ?? (typeof reserved.service === 'string' ? reserved.service : undefined) + // Field-addressing law: the metadata bag is the user's, VERBATIM — + // no reserved-name lift, no drops. Engine scalars come ONLY from + // their dedicated op fields; a bag field named `confidence` is an + // ordinary user field, exactly as on the committed write path. + const custom = { ...(op.metadata as Record | undefined) } + const confidence = op.confidence + const weight = op.weight + const subtype = op.subtype + const service = op.service // Id normalization (8.0) — mirror of the committed transact() add // path: a natural key coerces to a STABLE UUID (v5), preserving the @@ -759,16 +747,12 @@ export class Db { `with(): entity ${updateId} not found at generation ${this.gen}` ) } - // Same reserved-field normalization as the committed update path. - const { reserved, custom } = splitNounMetadataRecord( - op.metadata as Record | undefined - ) - const confidence = - op.confidence ?? (typeof reserved.confidence === 'number' ? reserved.confidence : undefined) - const weight = - op.weight ?? (typeof reserved.weight === 'number' ? reserved.weight : undefined) - const subtype = - op.subtype ?? (typeof reserved.subtype === 'string' ? reserved.subtype : undefined) + // Field-addressing law — mirror of the add case: the patch bag is + // the user's verbatim; engine scalars only from dedicated op fields. + const custom = { ...(op.metadata as Record | undefined) } + const confidence = op.confidence + const weight = op.weight + const subtype = op.subtype const mergedMetadata = op.merge !== false ? ({ ...(base.metadata as object), ...custom } as T) @@ -830,19 +814,14 @@ export class Db { } if (duplicate) break - // Reserved-field normalization — relationship mirror of the add - // op above (and of the committed relate() path). - const { reserved, custom } = splitVerbMetadataRecord( - op.metadata as Record | undefined - ) - const confidence = - op.confidence ?? (typeof reserved.confidence === 'number' ? reserved.confidence : undefined) - const weight = - op.weight ?? (typeof reserved.weight === 'number' ? reserved.weight : undefined) - const subtype = - op.subtype ?? (typeof reserved.subtype === 'string' ? reserved.subtype : undefined) - const service = - op.service ?? (typeof reserved.service === 'string' ? reserved.service : undefined) + // Field-addressing law — relationship mirror of the add case: the + // edge bag is the user's verbatim; engine scalars only from + // dedicated op fields. + const custom = { ...(op.metadata as Record | undefined) } + const confidence = op.confidence + const weight = op.weight + const subtype = op.subtype + const service = op.service const id = uuidv4() overlay.verbs.set(id, { diff --git a/src/db/fieldAddressing.ts b/src/db/fieldAddressing.ts index ae83ea18..21689319 100644 --- a/src/db/fieldAddressing.ts +++ b/src/db/fieldAddressing.ts @@ -174,23 +174,26 @@ export function readEntityFieldAddress( : null if (address.scope === 'system') { - // Entity views carry system scalars top-level; raw storage shapes carry - // them inside the stored metadata record (where `type` is spelled `noun`). - // Read top-level first, then the record — never the user's namespace. + // System scalars live at the record's top level, NEVER in the user's + // bag — a user field named `confidence` must be unreachable from + // system.confidence (and vice versa). Entity views carry the scalars + // top-level directly; record-derived views spell the type `noun`. const top = rec[address.field] if (top !== undefined) return top - if (bag) { - if (address.field === 'type') return bag.type ?? bag.noun - return bag[address.field] - } + if (address.field === 'type') return rec.noun return undefined } - // User scope. The write-path remap guarantees the user can never OWN a - // field named like a system scalar (those lift top-level at write), so a - // bare system name reads as ABSENT — reading the stored record's reserved - // key here would re-create the shadow this module exists to kill. Same for - // plumbing and the legacy 'noun' spelling. + // User scope: the bag IS the user's namespace, authoritative — EVERY name + // reads from it, engine spellings included (`bag.confidence` is the user's + // confidence field under the field-addressing law). + if (bag) return bag[address.field] + + // No bag at all: a LEGACY flat record (pre-nested-bag storage). Its keys + // matching system/plumbing names are the ENGINE's — the pre-law write door + // refused user colliders — so a bare system name reads as ABSENT rather + // than resurrecting the shadow this module exists to kill. Same for the + // legacy 'noun' spelling. if ( SYSTEM_ENTITY_SCALARS.has(address.field) || PLUMBING_FIELDS.has(address.field) || @@ -198,7 +201,6 @@ export function readEntityFieldAddress( ) { return undefined } - if (bag) return bag[address.field] return rec[address.field] } diff --git a/src/import/ImportCoordinator.ts b/src/import/ImportCoordinator.ts index 1e1316b7..dd145045 100644 --- a/src/import/ImportCoordinator.ts +++ b/src/import/ImportCoordinator.ts @@ -22,7 +22,6 @@ import { SmartYAMLImporter } from '../importers/SmartYAMLImporter.js' import { SmartDOCXImporter } from '../importers/SmartDOCXImporter.js' import { VFSStructureGenerator } from '../importers/VFSStructureGenerator.js' import { NounType, VerbType } from '../types/graphTypes.js' -import { splitNounMetadataRecord, splitVerbMetadataRecord } from '../types/reservedFields.js' import { v4 as uuidv4 } from '../universal/uuid.js' import * as fs from 'fs' import * as path from 'path' @@ -871,35 +870,18 @@ export class ImportCoordinator { } /** - * Strip Brainy-reserved entity keys out of an extractor-supplied metadata bag. - * - * Extractors (and consumer `customMetadata`) can carry reserved keys - * (`confidence`, `subtype`, `weight`, …) inside `metadata`. Brainy 8.0's - * default `reservedFieldPolicy` is `'throw'`, so spreading such a bag into - * `add({ metadata })` would reject the whole import. The import pipeline owns - * the correct write path: user-mutable reserved values are passed as dedicated - * `AddParams` params (see the call sites), so here we simply drop the reserved - * half of the bag and keep only the custom fields that belong in `metadata`. - * + * Normalize an extractor/consumer metadata bag for spreading — the + * field-addressing law: the bag is the user's, VERBATIM. No name is + * reserved anymore ('confidence', 'subtype', 'type', … in a source bag + * import as ordinary user fields); the old reserved-key strip was data + * loss under the law and is gone. A forged 'system.'-prefixed key still + * refuses loudly at the write door (`rejectForgedSystemKeys`). * @param bag - The extractor/consumer metadata bag (may be undefined). - * @returns The custom-only metadata (reserved keys removed). + * @returns The bag itself, or `{}` for non-object inputs. */ - private stripReservedFromBag(bag: Record | undefined | null): Record { + private bagVerbatim(bag: Record | undefined | null): Record { if (!bag || typeof bag !== 'object') return {} - return splitNounMetadataRecord(bag).custom - } - - /** - * Relationship mirror of {@link stripReservedFromBag} — strips reserved verb - * keys (`verb`, `confidence`, `weight`, `subtype`, …) out of an edge metadata - * bag so it carries only custom fields. Reserved values that have a dedicated - * `RelateParams` param are passed there by the call site instead. - * @param bag - The extractor/consumer edge metadata bag (may be undefined). - * @returns The custom-only edge metadata (reserved keys removed). - */ - private stripReservedFromRelationBag(bag: Record | undefined | null): Record { - if (!bag || typeof bag !== 'object') return {} - return splitVerbMetadataRecord(bag).custom + return bag } /** @@ -1017,7 +999,7 @@ export class ImportCoordinator { importedAt: trackingContext.importedAt, importFormat: trackingContext.importFormat, importSource: trackingContext.importSource, - ...this.stripReservedFromBag(trackingContext.customMetadata) + ...this.bagVerbatim(trackingContext.customMetadata) }) } }) @@ -1045,13 +1027,11 @@ export class ImportCoordinator { data: entity.description || entity.name, type: entity.type, subtype: entity.subtype ?? options.defaultSubtype ?? 'imported', - // `confidence` is a reserved field — pass it as the dedicated param, - // never inside the metadata bag (8.0 reservedFieldPolicy defaults to 'throw'). + // Engine confidence rides its dedicated param; the bag below is + // the user's verbatim (no name is reserved — field-addressing law). confidence: entity.confidence, metadata: { - // Extractor/consumer bags may smuggle reserved keys — strip them so - // the bag carries only custom fields. - ...this.stripReservedFromBag(entity.metadata), + ...this.bagVerbatim(entity.metadata), name: entity.name, vfsPath: vfsFile?.path, importedFrom: 'import-coordinator', @@ -1064,7 +1044,7 @@ export class ImportCoordinator { importSource: trackingContext.importSource, sourceRow: row.rowNumber, sourceSheet: row.sheet, - ...this.stripReservedFromBag(trackingContext.customMetadata) + ...this.bagVerbatim(trackingContext.customMetadata) }) } } @@ -1145,7 +1125,7 @@ export class ImportCoordinator { importIds: [trackingContext.importId], projectId: trackingContext.projectId, importFormat: trackingContext.importFormat, - ...this.stripReservedFromRelationBag(trackingContext.customMetadata) + ...this.bagVerbatim(trackingContext.customMetadata) }) } } @@ -1180,7 +1160,7 @@ export class ImportCoordinator { confidence: entity.confidence, metadata: { // Strip any reserved keys an extractor smuggled into the bag. - ...this.stripReservedFromBag(entity.metadata), + ...this.bagVerbatim(entity.metadata), name: entity.name, vfsPath: vfsFile?.path, importedFrom: 'import-coordinator', @@ -1194,7 +1174,7 @@ export class ImportCoordinator { importSource: trackingContext.importSource, sourceRow: row.rowNumber, sourceSheet: row.sheet, - ...this.stripReservedFromBag(trackingContext.customMetadata) + ...this.bagVerbatim(trackingContext.customMetadata) }) } }) @@ -1234,7 +1214,7 @@ export class ImportCoordinator { importIds: [trackingContext.importId], projectId: trackingContext.projectId, importFormat: trackingContext.importFormat, - ...this.stripReservedFromRelationBag(trackingContext.customMetadata) + ...this.bagVerbatim(trackingContext.customMetadata) }) } }) @@ -1289,7 +1269,7 @@ export class ImportCoordinator { projectId: trackingContext.projectId, importedAt: trackingContext.importedAt, importFormat: trackingContext.importFormat, - ...this.stripReservedFromBag(trackingContext.customMetadata) + ...this.bagVerbatim(trackingContext.customMetadata) }) } }) @@ -1319,7 +1299,7 @@ export class ImportCoordinator { projectId: trackingContext.projectId, importedAt: trackingContext.importedAt, importFormat: trackingContext.importFormat, - ...this.stripReservedFromRelationBag(trackingContext.customMetadata) + ...this.bagVerbatim(trackingContext.customMetadata) }) } }) @@ -1422,7 +1402,7 @@ export class ImportCoordinator { ...(typeof (rel as any).confidence === 'number' && { confidence: (rel as any).confidence }), ...(typeof (rel as any).weight === 'number' && { weight: (rel as any).weight }), metadata: { - ...this.stripReservedFromRelationBag(rel.metadata), + ...this.bagVerbatim(rel.metadata), relationshipType: 'semantic', // Distinguish from VFS/provenance inferredType: verbType !== rel.type, // Track if type was enhanced originalType: rel.type diff --git a/src/index.ts b/src/index.ts index 3876a903..3186a6a7 100644 --- a/src/index.ts +++ b/src/index.ts @@ -89,7 +89,12 @@ export { RESERVED_ENTITY_FIELDS, RESERVED_RELATION_FIELDS, splitNounMetadataRecord, - splitVerbMetadataRecord + splitVerbMetadataRecord, + buildNounMetadataRecord, + buildVerbMetadataRecord, + isNestedBagRecord, + METADATA_RECORD_FORMAT_KEY, + NESTED_BAG_FORMAT } from './types/reservedFields.js' export type { ReservedEntityField, diff --git a/src/migration/MigrationRunner.ts b/src/migration/MigrationRunner.ts index d2e251a2..6a8a34bd 100644 --- a/src/migration/MigrationRunner.ts +++ b/src/migration/MigrationRunner.ts @@ -9,6 +9,67 @@ import type { BaseStorage } from '../storage/baseStorage.js' import type { NounMetadata, VerbMetadata } from '../coreTypes.js' import type { Migration, MigrationState, MigrationPreview, MigrationResult, MigrateOptions, MigrationError } from './types.js' import { MIGRATIONS } from './migrations.js' +import { + splitNounMetadataRecord, + splitVerbMetadataRecord, + buildNounMetadataRecord, + buildVerbMetadataRecord, + RESERVED_ENTITY_FIELDS, + RESERVED_RELATION_FIELDS +} from '../types/reservedFields.js' + +const RESERVED_NOUN_SET: ReadonlySet = new Set(RESERVED_ENTITY_FIELDS) +const RESERVED_VERB_SET: ReadonlySet = new Set(RESERVED_RELATION_FIELDS) + +/** + * Normalize a stored record (either era: legacy flat OR v2 nested-bag) into + * THE transform view — the one shape every migration transform receives: + * engine fields top-level, the user's metadata bag nested under `metadata`. + * Transforms never see the storage era; a migration written today works on + * a brain of any age. + */ +function toTransformView( + record: Record, + kind: 'noun' | 'verb' +): Record { + const { reserved, custom } = + kind === 'noun' ? splitNounMetadataRecord(record) : splitVerbMetadataRecord(record) + return { ...reserved, metadata: { ...custom } } +} + +/** + * Convert a transform's returned view back into a stamped v2 stored record. + * LOUD CONTRACT: user fields belong inside `.metadata` — a stray top-level + * key that is not an engine field is a migration bug under the + * field-addressing law (pre-law transforms wrote user fields flat), and it + * refuses with the fix in the message rather than silently dropping or + * silently storing it as an engine key. + */ +function fromTransformView( + view: Record, + kind: 'noun' | 'verb' +): Record { + const reservedSet = kind === 'noun' ? RESERVED_NOUN_SET : RESERVED_VERB_SET + const engine: Record = {} + for (const [key, value] of Object.entries(view)) { + if (key === 'metadata') continue + if (!reservedSet.has(key)) { + throw new Error( + `migration transform returned a top-level key '${key}' that is not an ` + + `engine field — under the field-addressing law user fields live inside ` + + `.metadata (return { ...view, metadata: { ...view.metadata, ${key}: … } }).` + ) + } + engine[key] = value + } + const bag = + view.metadata && typeof view.metadata === 'object' && !Array.isArray(view.metadata) + ? (view.metadata as Record) + : {} + return kind === 'noun' + ? buildNounMetadataRecord(engine, bag) + : buildVerbMetadataRecord(engine, bag) +} const MIGRATION_STATE_KEY = '__migration_state__' const PREVIEW_SAMPLE_SIZE = 5 @@ -125,14 +186,16 @@ export class MigrationRunner { const entityMeta = metadataBatch.get(entity.id) if (!entityMeta) continue - const metadata = entityMeta as Record - const result = this.applyTransforms(metadata, nounMigrations) + // Transforms see THE view (engine fields + nested user bag), + // never the raw storage era. + const view = toTransformView(entityMeta as Record, 'noun') + const result = this.applyTransforms(view, nounMigrations) if (result !== null) { affectedEntities++ if (sampleChanges.length < PREVIEW_SAMPLE_SIZE) { sampleChanges.push({ id: entity.id, - before: { ...metadata }, + before: view, after: result }) } @@ -157,14 +220,14 @@ export class MigrationRunner { const verbMeta = await this.storage.getVerbMetadata(verb.id) if (!verbMeta) continue - const metadata = verbMeta as Record - const result = this.applyTransforms(metadata, verbMigrations) + const view = toTransformView(verbMeta as Record, 'verb') + const result = this.applyTransforms(view, verbMigrations) if (result !== null) { affectedEntities++ if (sampleChanges.length < PREVIEW_SAMPLE_SIZE) { sampleChanges.push({ id: verb.id, - before: { ...metadata }, + before: view, after: result }) } @@ -289,9 +352,16 @@ export class MigrationRunner { if (!entityMeta) continue try { - const transformed = migration.transform(entityMeta as Record) + const transformed = migration.transform( + toTransformView(entityMeta as Record, 'noun') + ) if (transformed !== null) { - await this.storage.saveNounMetadata(entity.id, transformed as NounMetadata) + // Re-stamp as a v2 record (also upgrades legacy records touched + // by a migration onto the nested-bag shape). + await this.storage.saveNounMetadata( + entity.id, + fromTransformView(transformed, 'noun') as NounMetadata + ) modified++ } } catch (err) { @@ -357,9 +427,14 @@ export class MigrationRunner { if (!metadata) continue try { - const transformed = migration.transform(metadata as Record) + const transformed = migration.transform( + toTransformView(metadata as Record, 'verb') + ) if (transformed !== null) { - await this.storage.saveVerbMetadata(verb.id, transformed as VerbMetadata) + await this.storage.saveVerbMetadata( + verb.id, + fromTransformView(transformed, 'verb') as VerbMetadata + ) modified++ } } catch (err) { diff --git a/src/migration/types.ts b/src/migration/types.ts index 2dcc1d1a..a63e40b9 100644 --- a/src/migration/types.ts +++ b/src/migration/types.ts @@ -14,7 +14,19 @@ export interface Migration { description: string /** Which entity types this migration applies to */ applies: 'nouns' | 'verbs' | 'both' - /** Return transformed metadata, or null if no change needed */ + /** + * Return the transformed record view, or null if no change needed. + * + * THE VIEW CONTRACT (field-addressing law): the transform receives ONE + * normalized shape regardless of how old the stored record is — engine + * fields top-level (`noun`/`verb`, `subtype`, `confidence`, `weight`, + * timestamps, `_rev`, …) and the USER's metadata bag nested under + * `metadata` (where every name is the user's, engine spellings included). + * Return the same shape: user-field changes go inside `.metadata`; a + * stray non-engine top-level key in the returned object refuses loudly + * (it is the pre-law flat habit, and silently guessing its namespace + * would corrupt data). + */ transform: (metadata: Record) => Record | null } diff --git a/src/neural/neuralImport.ts b/src/neural/neuralImport.ts index c8d19eb5..ed240a3f 100644 --- a/src/neural/neuralImport.ts +++ b/src/neural/neuralImport.ts @@ -7,7 +7,6 @@ import { Brainy } from '../brainy.js' import { NounType, VerbType } from '../types/graphTypes.js' -import { splitNounMetadataRecord, splitVerbMetadataRecord } from '../types/reservedFields.js' import * as fs from '../universal/fs.js' import * as path from '../universal/path.js' // @ts-ignore @@ -803,12 +802,14 @@ export class NeuralImport { data: this.extractMainText(entity.originalData), type: entity.nounType as NounType, subtype: entity.subtype ?? options.defaultSubtype ?? 'extracted', - // `confidence` is a reserved field — dedicated param, not metadata - // (8.0 reservedFieldPolicy defaults to 'throw'). + // Engine confidence rides its dedicated param; the source object + // imports as the user's bag VERBATIM — no name is reserved + // (field-addressing law). confidence: entity.confidence, metadata: { - // Strip any reserved keys the source data smuggled into the bag. - ...splitNounMetadataRecord(entity.originalData).custom, + ...(typeof entity.originalData === 'object' && entity.originalData !== null + ? entity.originalData + : {}), id: entity.suggestedId } }) @@ -822,11 +823,13 @@ export class NeuralImport { type: relationship.verbType as VerbType, subtype: relationship.subtype ?? options.defaultSubtype ?? 'extracted', weight: relationship.weight, - confidence: relationship.confidence, // reserved field — dedicated param, not metadata + confidence: relationship.confidence, // engine confidence — dedicated param metadata: { context: relationship.context, - // Strip any reserved keys smuggled into the edge metadata bag. - ...splitVerbMetadataRecord(relationship.metadata).custom + // The edge bag imports verbatim — no name is reserved. + ...(typeof relationship.metadata === 'object' && relationship.metadata !== null + ? relationship.metadata + : {}) } }) } diff --git a/src/storage/baseStorage.ts b/src/storage/baseStorage.ts index 1d3e245d..b78b4a49 100644 --- a/src/storage/baseStorage.ts +++ b/src/storage/baseStorage.ts @@ -36,7 +36,8 @@ import { BrainyError, ProtectedArtifactError, DerivedArtifactMissingError } from import { MetadataWriteBuffer } from '../utils/metadataWriteBuffer.js' import { splitNounMetadataRecord, - splitVerbMetadataRecord + splitVerbMetadataRecord, + isNestedBagRecord } from '../types/reservedFields.js' /** @@ -1013,8 +1014,14 @@ export abstract class BaseStorage extends BaseStorageAdapter { const hashes: string[] = [] for (const record of records) { if (record.kind !== 'noun') continue - const storage = (record.metadata as { storage?: { type?: string; hash?: unknown } } | null) - ?.storage + // The VFS blob pointer (`storage: {type:'blob', hash}`) is a USER-bag + // field: in a v2 nested-bag record it lives inside `metadata`, in a + // legacy flat record it sits at the top level — read shape-aware. + const raw = record.metadata as Record | null + const bag = isNestedBagRecord(raw) + ? (raw!.metadata as Record) + : raw + const storage = (bag as { storage?: { type?: string; hash?: unknown } } | null)?.storage if (storage?.type === 'blob' && typeof storage.hash === 'string') { hashes.push(storage.hash) } diff --git a/src/types/brainy.types.ts b/src/types/brainy.types.ts index 6c133cf0..6c1f0ffd 100644 --- a/src/types/brainy.types.ts +++ b/src/types/brainy.types.ts @@ -320,15 +320,18 @@ export interface AddParams { */ visibility?: 'public' | 'internal' /** - * Structured queryable fields — indexed by MetadataIndex, used in `where` filters. + * Structured queryable fields — indexed by MetadataIndex, used in `where` + * filters, `orderBy`, and aggregation. * - * Reserved entity fields (`RESERVED_ENTITY_FIELDS` — `noun`, `subtype`, `visibility`, - * `createdAt`, `updatedAt`, `confidence`, `weight`, `service`, `data`, `createdBy`, - * `_rev`) may NOT appear here — they have dedicated top-level params and the type makes - * a literal reserved key a compile error. Untyped (JavaScript) callers that pass one - * anyway are normalized at write time: user-settable fields remap to their top-level - * param (top-level wins when both are supplied), system-managed fields are dropped with - * a one-shot warning. + * THE FIELD-ADDRESSING LAW: every name here is YOURS. There are no + * reserved metadata names — `confidence`, `type`, `id`, `level`, `data`, + * `content`, … are ordinary user fields that index, filter, sort, and + * aggregate like any other, and survive faithfully across restarts and + * rebuilds. Engine scalars are set only via their dedicated params + * (`confidence`, `weight`, `subtype`, …) and are queried explicitly as + * `system.` (`where: { 'system.confidence': … }`). The ONE illegal + * spelling is a key starting `'system.'` — the engine's explicit address + * namespace cannot be forged; such a write refuses with a typed error. */ metadata?: EntityMetadataInput /** Custom entity ID. When omitted, a time-ordered UUID v7 is generated; a supplied natural-key string is normalized to a stable UUID v5. */ @@ -386,12 +389,11 @@ export interface UpdateParams { */ visibility?: EntityVisibility /** - * Metadata fields to merge (or replace when `merge: false`). Reserved entity - * fields (`RESERVED_ENTITY_FIELDS`) may NOT appear here — `confidence` / - * `weight` / `subtype` / `visibility` have dedicated params on this call, and the rest - * are system-managed. A literal reserved key is a compile error; untyped callers - * are normalized at write time (remap user-settable, drop system-managed - * with a one-shot warning). + * Metadata fields to merge (or replace when `merge: false`). Every name is + * the user's (the field-addressing law) — a patch field named `confidence` + * updates YOUR field of that name, never the engine scalar (use the + * dedicated `confidence` param for that). Keys spelled `'system.…'` refuse + * with a typed error (namespace forgery). */ metadata?: EntityMetadataPatch merge?: boolean // Merge or replace metadata (default: true) @@ -444,11 +446,11 @@ export interface RelateParams { /** Content for the relationship (optional — overrides auto-computed vector) */ data?: any /** - * Structured queryable fields on the edge. Reserved relationship fields - * (`RESERVED_RELATION_FIELDS` — `verb`, `subtype`, `visibility`, `createdAt`, - * `updatedAt`, `confidence`, `weight`, `service`, `data`, `createdBy`, `_rev`) may NOT - * appear here — they have dedicated params. A literal reserved key is a - * compile error; untyped callers are normalized at write time. + * Structured queryable fields on the edge. Every name is the user's (the + * field-addressing law) — `verb`, `confidence`, `weight`, … in this bag are + * ordinary user fields; engine scalars ride their dedicated params and are + * addressed as `system.`. Keys spelled `'system.…'` refuse with a + * typed error (namespace forgery). */ metadata?: RelationMetadataInput /** Create reverse edge too (default: false) */ @@ -478,10 +480,9 @@ export interface UpdateRelationParams { confidence?: number // New confidence (0-1) data?: any // New content /** - * Metadata fields to merge (or replace when `merge: false`). Reserved - * relationship fields (`RESERVED_RELATION_FIELDS`) may NOT appear here — - * a literal reserved key is a compile error; untyped callers are - * normalized at write time. + * Metadata fields to merge (or replace when `merge: false`). Every name is + * the user's (the field-addressing law); engine scalars ride their + * dedicated params. Keys spelled `'system.…'` refuse with a typed error. */ metadata?: RelationMetadataPatch merge?: boolean // Merge or replace metadata @@ -2027,32 +2028,6 @@ export interface BrainyConfig { */ force?: boolean - /** - * How write paths react when an untyped (JavaScript) caller smuggles a - * Brainy-reserved field (`RESERVED_ENTITY_FIELDS` / `RESERVED_RELATION_FIELDS` - * — `confidence`, `weight`, `subtype`, `visibility`, `service`, `createdBy`, - * `noun`/`verb`, `data`, `createdAt`, `updatedAt`, `_rev`) **inside the - * `metadata` bag** of `add()` / `update()` / `relate()` / `updateRelation()` - * (and their `transact()` / `with()` mirrors). TypeScript callers can't write - * these shapes at all — the compile-time guard on the metadata param types - * (`NoReservedEntityKeys` / `NoReservedRelationKeys`) rejects a literal - * reserved key — so this policy only governs untyped callers that slip one - * past the compiler. - * - * - `'throw'` (**default, 8.0**): a reserved key in the bag throws a clear - * `Error` naming the offending key(s) and the correct write path. No silent - * remap, no data loss, no surprise. This is the 8.0 "no silent failures" - * contract. - * - `'warn'`: legacy remapping with a loud, one-shot (per key, per process) - * warning for EVERY reserved key found — user-mutable fields are remapped to - * their dedicated top-level param (top-level wins when both are supplied), - * system-managed fields are dropped. Use while migrating untyped call sites. - * - `'remap'`: the pre-8.0 silent remapping, no warning. Last-resort - * compatibility hatch for code that intentionally relies on the bag path. - * - * @default 'throw' - */ - reservedFieldPolicy?: 'throw' | 'warn' | 'remap' } // ============= Neural API Types ============= diff --git a/src/types/reservedFields.ts b/src/types/reservedFields.ts index a0606e1d..15b585c5 100644 --- a/src/types/reservedFields.ts +++ b/src/types/reservedFields.ts @@ -1,35 +1,54 @@ /** * @module types/reservedFields - * @description The canonical reserved-field contract — ONE place that defines - * which keys belong to Brainy (top-level entity/relationship fields) and may - * therefore never live inside a `metadata` bag. + * @description The stored-record layout contract — ONE place that defines + * which keys of a persisted metadata record belong to the ENGINE (top-level + * entity/relationship fields) and how the USER's metadata bag is kept apart + * from them, faithfully, across flush / reopen / rebuild / time travel. * - * Three layers enforce the contract, all driven by the constants below: + * THE FIELD-ADDRESSING LAW (ruled 2026-08-03, VENUE-BRAINY-ORDERBY-NOOP): + * data is either in main space — where developers can use ANY name, and it + * all works with every database function — or it is in `system.*`. There are + * NO reserved user-facing metadata names anymore: `confidence`, `type`, + * `level`, `data`, `id`, `content` … inside a metadata bag are ordinary user + * fields. The only refused write is a user metadata key literally starting + * with `'system.'` (namespace forgery — see `rejectForgedSystemKeys`). * - * 1. **Compile time** — `AddParams.metadata`, `UpdateParams.metadata`, - * `RelateParams.metadata` and `UpdateRelationParams.metadata` are typed so - * a literal reserved key is a TypeScript error (see - * {@link EntityMetadataInput} / {@link RelationMetadataInput}). - * 2. **Write time** — for untyped (JavaScript) callers that smuggle a - * reserved key past the compiler anyway, every write path normalizes the - * bag: user-mutable fields are remapped to their dedicated top-level - * param (top-level wins when both are supplied) and system-managed fields - * are dropped with a one-shot warning naming the correct write path. - * 3. **Read time** — every read path splits the stored flat record through - * {@link splitNounMetadataRecord} / {@link splitVerbMetadataRecord}, so a - * reserved field is surfaced ONLY at top level and `entity.metadata` / - * `relation.metadata` contain ONLY custom fields, always — live reads, - * batch reads, and historical (`asOf`) reads alike. + * That law makes name-based storage discrimination unsound for NEW records + * (a user field named `confidence` may now legally sit beside the engine's + * confidence scalar), so persisted metadata records carry the user bag + * NESTED, shape-discriminated by a format stamp: * - * Documented for consumers in `docs/concepts/consistency-model.md` - * ("Reserved fields"). + * - **v2 (nested-bag)** — `{ …engine fields…, [METADATA_RECORD_FORMAT_KEY]: + * NESTED_BAG_FORMAT, metadata: { …user bag, verbatim… } }`. Built ONLY by + * {@link buildNounMetadataRecord} / {@link buildVerbMetadataRecord}; the + * engine half and the user bag can never collide because they never share + * a level. + * - **legacy (flat)** — engine fields and user fields mixed at one level, + * discriminated BY NAME through the RESERVED_* lists. Sound for legacy + * records precisely because the pre-law write door REFUSED user metadata + * carrying those names — a flat key matching a reserved name IS the + * engine's value in any record the old door admitted. + * + * {@link splitNounMetadataRecord} / {@link splitVerbMetadataRecord} read + * BOTH shapes (stamp first, name split as the legacy fallback) and are the + * single read-side choke point for live, batch, AND historical (`asOf`) + * reads — the generation store snapshots whole records, so time travel + * rides the same split. + * + * The RESERVED_* lists therefore no longer describe a user-facing ban — they + * describe the ENGINE HALF of the stored record layout (and drive the legacy + * split). The write-door remap machinery and the compile-time metadata key + * bans that used to enforce the old contract are gone. */ /** - * @description Entity (noun) field names reserved by Brainy. These keys are - * stored in the flat per-entity metadata record alongside custom fields, but - * they belong to Brainy: every read path extracts them to top-level - * `Entity` fields, and no write path accepts them inside `metadata`. + * @description Entity (noun) field names owned by the ENGINE in a stored + * metadata record. In v2 (nested-bag) records these are the legal TOP-LEVEL + * keys beside the nested `metadata` bag; in legacy flat records they drive + * the by-name split. They are NOT a user-facing ban list: since the + * field-addressing law, a user metadata field may carry any of these names + * and remains the user's — it lives inside the nested bag, never at the + * record's top level. * * | Key | Canonical write path | * |-----|----------------------| @@ -119,68 +138,54 @@ export type ReservedRelationField = (typeof RESERVED_RELATION_FIELDS)[number] type IsAny = 0 extends 1 & T ? true : false /** - * @description Compile-time tripwire: marks every reserved entity key as - * `never` so an object literal carrying one fails to type-check. Keys that - * `T` itself declares (including via an index signature, where - * `keyof T = string`) are exempted — a consumer who *explicitly* types a - * reserved key into their metadata shape keeps a working (if unwise) type, - * and index-signature metadata types remain assignable. + * @deprecated The compile-time reserved-key ban died with the + * field-addressing law: every name is legal user metadata now. Kept as an + * empty (no-op) guard so external type references keep compiling; it bans + * nothing. */ -export type NoReservedEntityKeys = { - readonly [K in ReservedEntityField as K extends keyof T ? never : K]?: never -} +export type NoReservedEntityKeys = unknown /** - * @description Relationship mirror of {@link NoReservedEntityKeys}. + * @deprecated Relationship mirror of {@link NoReservedEntityKeys} — no-op + * for the same reason. */ -export type NoReservedRelationKeys = { - readonly [K in ReservedRelationField as K extends keyof T ? never : K]?: never -} - -/** - * @description The metadata bag shape for untyped brains (`T = any`): an - * open index signature (any custom key, any value — exactly the pre-8.0 - * latitude) intersected with the reserved-key guard, whose declared - * `?: never` properties take precedence over the index signature so a - * literal reserved key is still a compile error. - */ -type OpenBag = { [key: string]: any } & Guard +export type NoReservedRelationKeys = unknown /** * @description The type of `AddParams.metadata`: the consumer's metadata - * shape `T` with reserved entity keys forbidden at compile time. For untyped - * brains (`T = any`) the bag stays open ({@link OpenBag}), so arbitrary - * custom fields remain legal while literal reserved keys still error. + * shape `T`, open. Under the field-addressing law EVERY key is a legal user + * field (engine scalars are written only via their dedicated params and read + * at `system.*`), so no name is banned at compile time. The one illegal + * spelling — a key starting `'system.'` — cannot be expressed as a mapped + * type ban and is refused at runtime (`rejectForgedSystemKeys`). */ export type EntityMetadataInput = IsAny extends true - ? OpenBag> - : T & NoReservedEntityKeys + ? { [key: string]: any } + : T /** * @description The type of `UpdateParams.metadata`: a partial patch of the - * consumer's metadata shape with reserved entity keys forbidden at compile - * time. Same `T = any` handling as {@link EntityMetadataInput}. + * consumer's metadata shape. Same openness as {@link EntityMetadataInput}. */ export type EntityMetadataPatch = IsAny extends true - ? OpenBag> - : Partial & NoReservedEntityKeys + ? { [key: string]: any } + : Partial /** * @description The type of `RelateParams.metadata`: the consumer's edge - * metadata shape with reserved relationship keys forbidden at compile time. + * metadata shape, open — the relation mirror of {@link EntityMetadataInput}. */ export type RelationMetadataInput = IsAny extends true - ? OpenBag> - : T & NoReservedRelationKeys + ? { [key: string]: any } + : T /** * @description The type of `UpdateRelationParams.metadata`: a partial patch - * of the consumer's edge metadata shape with reserved relationship keys - * forbidden at compile time. + * of the consumer's edge metadata shape, open. */ export type RelationMetadataPatch = IsAny extends true - ? OpenBag> - : Partial & NoReservedRelationKeys + ? { [key: string]: any } + : Partial /** * @description Result of splitting a stored flat metadata record into its @@ -196,6 +201,103 @@ export interface SplitMetadataRecord { const RESERVED_ENTITY_SET: ReadonlySet = new Set(RESERVED_ENTITY_FIELDS) const RESERVED_RELATION_SET: ReadonlySet = new Set(RESERVED_RELATION_FIELDS) +/** + * @description The format-stamp key of a persisted metadata record. Its + * presence with the exact value {@link NESTED_BAG_FORMAT} marks a v2 + * (nested-bag) record; its absence marks a legacy flat record. The stamp is + * what makes the shape check collision-proof against legacy user data: a + * pre-law record COULD carry a user field named `metadata` (the name was + * never reserved), but it cannot also carry this engine-written stamp. + */ +export const METADATA_RECORD_FORMAT_KEY = '_fmt' + +/** + * @description The nested-bag record format stamp (v2, the field-addressing + * law's storage shape, 2026-08-03): engine fields at top level, the user's + * metadata bag NESTED verbatim under `metadata`. Cross-engine: the native + * provider discriminates record shapes by the same stamp. + */ +export const NESTED_BAG_FORMAT = 2 + +/** + * @description `true` when a persisted record carries the v2 nested-bag + * stamp (and a structurally valid nested bag). + */ +export function isNestedBagRecord( + record: Record | null | undefined +): boolean { + return ( + record !== null && + record !== undefined && + typeof record === 'object' && + record[METADATA_RECORD_FORMAT_KEY] === NESTED_BAG_FORMAT && + typeof record.metadata === 'object' && + record.metadata !== null && + !Array.isArray(record.metadata) + ) +} + +/** + * @description Build a v2 (nested-bag) entity metadata record — THE only + * sanctioned way to construct a persisted noun metadata record. The engine + * half goes top-level; the user bag nests verbatim under `metadata`; the + * format stamp seals the shape. Because the two halves never share a level, + * a user field named `confidence` (or any other engine spelling) survives + * flush / reopen / rebuild / time travel exactly as written. + * @param engineFields - The engine-owned half (keys from + * {@link RESERVED_ENTITY_FIELDS} — `noun`, timestamps, `_rev`, …). + * @param userBag - The consumer's metadata bag, stored verbatim. + * @returns The stamped v2 record. + */ +export function buildNounMetadataRecord( + engineFields: Partial>, + userBag: Record | undefined +): Record { + return { + ...engineFields, + [METADATA_RECORD_FORMAT_KEY]: NESTED_BAG_FORMAT, + metadata: { ...(userBag ?? {}) } + } +} + +/** + * @description Build a v2 (nested-bag) relationship metadata record — the + * verb mirror of {@link buildNounMetadataRecord}. + * @param engineFields - The engine-owned half (keys from + * {@link RESERVED_RELATION_FIELDS} — `verb`, `weight`, timestamps, …). + * @param userBag - The consumer's edge metadata bag, stored verbatim. + * @returns The stamped v2 record. + */ +export function buildVerbMetadataRecord( + engineFields: Partial>, + userBag: Record | undefined +): Record { + return { + ...engineFields, + [METADATA_RECORD_FORMAT_KEY]: NESTED_BAG_FORMAT, + metadata: { ...(userBag ?? {}) } + } +} + +/** + * @description Shape-first split of a v2 record: the engine half is the top + * level filtered through the reserved list (belt — the builders only ever + * write reserved names there), the user bag is `record.metadata` verbatim. + */ +function splitNestedRecord( + record: Record, + reservedSet: ReadonlySet +): SplitMetadataRecord { + const reserved: Record = {} + for (const [key, value] of Object.entries(record)) { + if (reservedSet.has(key)) reserved[key] = value + } + return { + reserved: reserved as Partial>, + custom: { ...(record.metadata as Record) } + } +} + /** * @description Shared splitter — partitions a record's keys against a * reserved-name set. `null`/`undefined` records split to two empty objects. @@ -222,33 +324,45 @@ function splitRecord( } /** - * @description Split a stored entity (noun) flat metadata record into - * reserved fields and custom metadata — THE canonical read-side split. Every - * entity read path (live `get()`, batch reads, paginated listings, and - * historical `asOf()` materialization) goes through this function, so the - * reserved list can never drift between read paths. - * @param record - The stored flat metadata record. - * @returns `reserved` (Brainy-owned fields) and `custom` (the consumer's metadata bag). + * @description Split a stored entity (noun) metadata record into engine + * fields and the user's metadata bag — THE canonical read-side split, shape + * aware. v2 (nested-bag) records split by SHAPE: engine half top-level, bag + * = `record.metadata` verbatim (user collider names survive faithfully). + * Legacy flat records split BY NAME through the reserved list — sound for + * them because the pre-law write door refused user metadata carrying those + * names. Every entity read path (live `get()`, batch reads, paginated + * listings, and historical `asOf()` materialization — the generation store + * snapshots whole records) goes through this function, so the two shapes + * can never drift between read paths. + * @param record - The stored metadata record (either shape). + * @returns `reserved` (engine-owned fields) and `custom` (the consumer's metadata bag). * @example * const { reserved, custom } = splitNounMetadataRecord(stored) * // reserved.noun → entity.type, reserved.confidence → entity.confidence, … - * // custom → entity.metadata (custom fields only, always) + * // custom → entity.metadata (the user's fields only, always — ANY names) */ export function splitNounMetadataRecord( record: Record | null | undefined ): SplitMetadataRecord { + if (isNestedBagRecord(record)) { + return splitNestedRecord(record as Record, RESERVED_ENTITY_SET) + } return splitRecord(record, RESERVED_ENTITY_SET) } /** - * @description Split a stored relationship (verb) flat metadata record into - * reserved fields and custom metadata — the verb mirror of - * {@link splitNounMetadataRecord}, used by every relationship read path. - * @param record - The stored flat metadata record. - * @returns `reserved` (Brainy-owned fields) and `custom` (the consumer's metadata bag). + * @description Split a stored relationship (verb) metadata record into + * engine fields and the user's edge metadata bag — the verb mirror of + * {@link splitNounMetadataRecord}, shape aware, used by every relationship + * read path. + * @param record - The stored metadata record (either shape). + * @returns `reserved` (engine-owned fields) and `custom` (the consumer's metadata bag). */ export function splitVerbMetadataRecord( record: Record | null | undefined ): SplitMetadataRecord { + if (isNestedBagRecord(record)) { + return splitNestedRecord(record as Record, RESERVED_RELATION_SET) + } return splitRecord(record, RESERVED_RELATION_SET) } diff --git a/src/utils/metadataIndex.ts b/src/utils/metadataIndex.ts index 6deeb811..f010560d 100644 --- a/src/utils/metadataIndex.ts +++ b/src/utils/metadataIndex.ts @@ -73,8 +73,11 @@ export interface MetadataIndexConfig { maxIndexSize?: number // Max number of entries per field value (default: 10000) rebuildThreshold?: number // Rebuild if index is this % stale (default: 0.1) autoOptimize?: boolean // Auto-cleanup unused entries (default: true) - indexedFields?: string[] // Only index these fields (default: all) - excludeFields?: string[] // Never index these fields + // NOTE: the name-based indexedFields/excludeFields knobs died with the + // field-addressing law ("no special names"): EVERY user field indexes, + // whatever its name. Bulk-payload protection is value-SHAPE based and + // uniform across all names (large arrays never become posting scalars; + // long values index hashed) — shape is not a name carve-out. } export interface MetadataIndexOptions { @@ -185,31 +188,12 @@ export class MetadataIndexManager implements MetadataIndexProvider { this.config = { maxIndexSize: config.maxIndexSize ?? 10000, rebuildThreshold: config.rebuildThreshold ?? 0.1, - autoOptimize: config.autoOptimize ?? true, - indexedFields: config.indexedFields ?? [], - excludeFields: config.excludeFields ?? [ - // ONLY exclude truly un-indexable fields (binary data, large content) - // Timestamps are NOW indexed with automatic bucketing (prevents pollution) - - // Vectors and embeddings (binary data, already have HNSW indexes) - 'embedding', - 'vector', - 'embeddings', - 'vectors', - - // Large content fields (too large for metadata indexing) - 'content', - 'data', - 'originalData', - '_data', - - // Primary keys (use direct lookups instead) - 'id' - - // NOTE: 'accessed', 'modified', 'createdAt', etc. are NO LONGER excluded! - // They are now indexed with automatic 1-minute bucketing to prevent file pollution - // This enables range queries like: modified > yesterday - ] + autoOptimize: config.autoOptimize ?? true + // No name-based exclude/allow lists — the field-addressing law: every + // user field indexes, whatever its name ('content', 'data', 'id', + // 'vector', … included). Bulk payloads are kept out by uniform value- + // SHAPE rules in extractIndexableFields (arrays >10 never become + // posting scalars; >100-char values index hashed), never by name. } // Initialize metadata cache with similar config to search cache @@ -301,7 +285,7 @@ export class MetadataIndexManager implements MetadataIndexProvider { } // Warm the cache with common fields (lazy loading optimization) - // This loads the 'noun' sparse index which is needed for type counts + // This loads the type column ('system.type') needed for type counts await this.warmCache() // Load type counts AFTER warmCache (sparse index is now cached) @@ -350,8 +334,9 @@ export class MetadataIndexManager implements MetadataIndexProvider { * Target: >80% cache hit rate for typical workloads */ async warmCache(): Promise { - // Common fields used in most queries - const commonFields = ['noun', 'type', 'service', 'createdAt'] + // Common columns used in most queries — the frozen system keys, plus + // legacy spellings for a pre-epoch-3 brain read before its rebuild runs. + const commonFields = ['system.type', 'system.service', 'system.createdAt', 'noun'] prodLog.debug(`🔥 Warming metadata cache with common fields: ${commonFields.join(', ')}`) @@ -537,9 +522,11 @@ export class MetadataIndexManager implements MetadataIndexProvider { } /** - * Lazy load entity counts from the 'noun' field sparse index (O(n) where n = number of types) + * Lazy load entity counts from the type column (O(n) where n = number of + * types). The frozen key is 'system.type' (epoch 3); the legacy 'noun' + * column is read as a fallback for a pre-epoch-3 brain observed before its + * rebuild has run (e.g. a reader-mode open against an old writer). * FIX: Previously read from stats.nounCount which was SERVICE-keyed, not TYPE-keyed - * Now computes counts from the sparse index which has the correct type information */ private async lazyLoadCounts(): Promise { try { @@ -549,23 +536,31 @@ export class MetadataIndexManager implements MetadataIndexProvider { this.entityCountsByTypeFixed.fill(0) this.verbCountsByTypeFixed.fill(0) - // PRIMARY (8.0+): rehydrate per-type counts from the column store's 'noun' - // field — the authoritative on-disk source after a cold reopen. + // PRIMARY (8.0+): rehydrate per-type counts from the column store's + // type column — the authoritative on-disk source after a cold reopen. + // Frozen key first ('system.type', epoch 3), legacy 'noun' as the + // pre-rebuild fallback. // // The chunked sparse-index WRITE path was removed in 7.20.0 (commit - // 11be039): new workspaces persist the 'noun' field ONLY to the column - // store, never to a `__sparse_index__noun` blob. So the legacy sparse - // path below finds nothing and leaves every count at 0 — which is exactly - // why counts.byType/byTypeEnum/topTypes/allNounTypeCounts all read empty + // 11be039): new workspaces persist the type column ONLY to the column + // store, never to a sparse-index blob. So the legacy sparse path below + // finds nothing and leaves every count at 0 — which is exactly why + // counts.byType/byTypeEnum/topTypes/allNounTypeCounts all read empty // after close()+reopen while find()/getNounCount() (different sources) // stay correct. The column store's per-value cardinality matches the warm // `updateTypeFieldAffinity` counts EXACTLY because both are driven from the // same `addToIndex` field set, in lockstep, with no visibility gate on // either — so this rehydration reproduces the warm values precisely. - if (this.columnStore && this.columnStore.getIndexedFields().includes('noun')) { - const nounValues = await this.columnStore.getFilterValues('noun') + const indexedCols = this.columnStore ? this.columnStore.getIndexedFields() : [] + const typeCol = indexedCols.includes('system.type') + ? 'system.type' + : indexedCols.includes('noun') + ? 'noun' + : null + if (this.columnStore && typeCol) { + const nounValues = await this.columnStore.getFilterValues(typeCol) for (const value of nounValues) { - const bitmap = await this.columnStore.filter('noun', value) + const bitmap = await this.columnStore.filter(typeCol, value) if (bitmap.size > 0) { // Use the stored value directly as the key (the legacy sparse path // did the same): it is already the normalized type string that @@ -580,16 +575,17 @@ export class MetadataIndexManager implements MetadataIndexProvider { } // LEGACY FALLBACK (pre-7.20.0 workspaces still on the chunked sparse index). - const nounSparseIndex = await this.loadSparseIndex('noun') + const sparseCol = (await this.loadSparseIndex('system.type')) ? 'system.type' : 'noun' + const nounSparseIndex = await this.loadSparseIndex(sparseCol) if (!nounSparseIndex) { - // No column-store 'noun' field and no sparse index yet — counts will be + // No column-store type column and no sparse index yet — counts will be // populated as entities are added. return } // Iterate through all chunks and sum up bitmap sizes by type for (const chunkId of nounSparseIndex.getAllChunkIds()) { - const chunk = await this.chunkManager.loadChunk('noun', chunkId) + const chunk = await this.chunkManager.loadChunk(sparseCol, chunkId) if (chunk) { for (const [type, bitmap] of chunk.entries) { const currentCount = this.totalEntitiesByType.get(type) || 0 @@ -1179,66 +1175,46 @@ export class MetadataIndexManager implements MetadataIndexProvider { return `__HASH_${Math.abs(hash).toString(36)}` } - /** - * Check if field should be indexed - */ - private shouldIndexField(field: string): boolean { - if (this.config.excludeFields.includes(field)) return false - if (this.config.indexedFields.length > 0) { - return this.config.indexedFields.includes(field) - } - return true - } - /** * Extract indexable field-value pairs from entity or metadata * - * Now handles BOTH entity structure (with top-level fields) AND plain metadata - * - Extracts from top-level fields (confidence, weight, timestamps, type, service, etc.) - * - Also extracts from nested metadata field (custom user fields) - * - Skips HNSW-specific fields (vector, connections, level, id) - * - Maps 'type' → 'noun' for backward compatibility with existing indexes - * - * BUG FIX: Exclude vector embeddings and large arrays from indexing - * BUG FIX: Also exclude purely numeric field names (array indices) - * - Vector fields (384+ dimensions) were creating 825K chunk files for 1,144 entities - * - Arrays converted to objects with numeric keys were still being indexed + * Handles BOTH entity structure (with top-level fields) AND record shapes + * - Record-frame system scalars index under literal 'system.' keys + * - The user's metadata bag indexes under bare keys — EVERY name (the + * field-addressing law: no special names; 'level', 'data', 'id', + * 'content', 'vector' in a bag are ordinary user fields) + * - Record-frame plumbing (vector, connections, level, data, _rev, id) + * never indexes — that is namespace routing, not a name carve-out + * - Value-SHAPE rules apply uniformly to all names: arrays >10 never + * become posting scalars; purely numeric key names (array indices) + * skip; >100-char values index hashed (normalizeValue) */ private extractIndexableFields(data: any): Array<{ field: string, value: any }> { const fields: Array<{ field: string, value: any }> = [] - // Fields that should NEVER be indexed: bulk structural payloads that would - // blow up the index (the 384-dim vector, embeddings, the adjacency list). - // These are also caught by the array-size guard below, but naming them is - // belt-and-suspenders. NOTE: `level` was previously here (an HNSW node's - // layer) but it never actually reaches this path — every caller passes a - // metadata bag or Entity record, neither of which carries the node's - // `level` — so its only effect was to silently drop a legitimate USER - // metadata field named `level` (log level, skill level, access level…), - // making `where: { level: … }` return nothing. Removed. (`id` stays: it is - // the reserved entity-identity field, resolved specially by find().) - const NEVER_INDEX = new Set(['vector', 'embedding', 'embeddings', 'connections', 'id']) + // RECORD-FRAME-ONLY plumbing guard: on an entity/stored-record frame + // these keys are the engine's structural payloads (the 384-dim vector, + // embeddings, the adjacency list, the identity field) and never index. + // This set is NEVER applied inside the user's metadata bag — under the + // field-addressing law every user name indexes; a real vector-sized + // value in a bag is kept out by the uniform array-size shape guard, not + // by its name. + const RECORD_PLUMBING = new Set(['vector', 'embedding', 'embeddings', 'connections', 'id']) // THE FROZEN INDEX KEY FORMAT (cross-engine, sealed 2026-08-03; the native // accelerator keys identically — epoch 3 rebuilds every brain onto it): // user fields index under BARE keys exactly as the caller wrote them; // the ten system scalars index under literal 'system.' keys — the // key IS the query address, so the two namespaces can never collide - // inside the index again. `origin` tracks which side of the record a key - // came from: 'record' = the entity/stored-record frame (system scalars, - // plumbing, and the metadata bag live here — the WRITE PATH's reserved- - // name remap guarantees a record-frame key matching a system name IS the - // system value); 'user' = inside the flattened metadata bag (everything - // is the user's, including natural names like `level` and `data`). - // Frame kinds: 'entity-record' = entityForIndexing shape (user fields - // nested under `metadata`; stray top-level keys are DROPPED, not guessed — - // epoch-3's rebuild-from-canonical normalizes historical shapes); - // 'flat-record' = the stored metadata-record shape (user fields FLAT - // beside the reserved ones — the write path's reserved-name remap - // guarantees a key matching a system name IS the system value, so - // non-system keys here are the user's and index bare); 'user' = inside - // the metadata bag (everything is the user's, including natural names - // like `level` and `data`). + // inside the index again. + // Frame kinds: 'entity-record' = entityForIndexing shape / v2 nested-bag + // stored record (user fields nested under `metadata`; stray top-level + // keys are DROPPED, not guessed); 'flat-record' = the LEGACY stored + // metadata-record shape (user fields flat beside the engine's — sound to + // split by name because the pre-law write door refused user metadata + // carrying engine names, so a flat key matching a system name IS the + // system value); 'user' = inside the metadata bag, where EVERY key is + // the user's and indexes bare — collider names included. type Frame = 'entity-record' | 'flat-record' | 'user' const extract = (obj: any, prefix = '', frame: Frame = 'entity-record'): void => { for (const [key, value] of Object.entries(obj)) { @@ -1254,30 +1230,25 @@ export class MetadataIndexManager implements MetadataIndexProvider { } else if (SYSTEM_ENTITY_SCALARS.has(key) && key !== 'id') { fullKey = `system.${key}` } else if ( - key === 'data' || key === '_rev' || key === 'level' || NEVER_INDEX.has(key) + key === 'data' || key === '_rev' || key === 'level' || key === '_fmt' || + RECORD_PLUMBING.has(key) ) { - continue // plumbing / identity / bulk payloads — never indexed from a record frame + continue // plumbing / identity / format stamp — never indexed from a record frame } else if (frame === 'entity-record') { continue // stray entity-frame key: dropped, not guessed } // flat-record fallthrough: a non-system, non-plumbing key IS a user - // field (flat beside the reserved ones) — indexes bare via fullKey. - } else if (!prefix && NEVER_INDEX.has(key)) { - // User frame: only the bulk-payload guards apply — natural names - // like `level` and `data` are real user fields here. (`id` as a - // user metadata field remains un-indexed this train — documented - // limitation; system.id resolves via the id mapper, never a column.) - continue + // field (flat beside the engine's, legacy shape) — indexes bare. } + // User frame: NO name-based skips — every user field indexes, whatever + // its name (the field-addressing law). Only the uniform value-shape + // guards below apply. // Skip purely numeric field names (array indices converted to object keys) // Legitimate field names should never be purely numeric // This catches vectors stored as objects: {0: 0.1, 1: 0.2, ...} if (/^\d+$/.test(key)) continue - // Skip fields based on user configuration - if (!this.shouldIndexField(fullKey)) continue - // Skip large arrays (> 10 elements) - likely vectors or bulk data if (Array.isArray(value) && value.length > 10) continue @@ -1510,10 +1481,11 @@ export class MetadataIndexManager implements MetadataIndexProvider { prodLog.debug(`Entity ${id} has ${wordFields.length} indexed words (large document)`) } - // Sort fields to process 'noun' field first for type-field affinity tracking + // Sort fields to process the type column first for type-field affinity + // tracking ('system.type' is the frozen key; 'noun' died at epoch 3). fields.sort((a, b) => { - if (a.field === 'noun') return -1 - if (b.field === 'noun') return 1 + if (a.field === 'system.type') return -1 + if (b.field === 'system.type') return 1 return 0 }) @@ -2861,6 +2833,17 @@ export class MetadataIndexManager implements MetadataIndexProvider { // VFS Statistics Methods (uses existing Roaring bitmap infrastructure) // ============================================================================ + /** + * Read the type column's bitmap for one type value — frozen key first + * ('system.type', epoch 3), legacy 'noun' as the pre-rebuild fallback. + */ + private async getTypeBitmap(type: string): Promise { + return ( + (await this.getBitmapFromChunks('system.type', type)) ?? + (await this.getBitmapFromChunks('noun', type)) + ) + } + /** * Get VFS entity count for a specific type using Roaring bitmap intersection * Uses hardware-accelerated SIMD operations (AVX2/SSE4.2) @@ -2869,7 +2852,7 @@ export class MetadataIndexManager implements MetadataIndexProvider { */ async getVFSEntityCountByType(type: string): Promise { const vfsBitmap = await this.getBitmapFromChunks('isVFSEntity', true) - const typeBitmap = await this.getBitmapFromChunks('noun', type) + const typeBitmap = await this.getTypeBitmap(type) if (!vfsBitmap || !typeBitmap) return 0 @@ -2892,7 +2875,7 @@ export class MetadataIndexManager implements MetadataIndexProvider { // Iterate through all known types and compute VFS count via intersection for (const type of this.totalEntitiesByType.keys()) { - const typeBitmap = await this.getBitmapFromChunks('noun', type) + const typeBitmap = await this.getTypeBitmap(type) if (typeBitmap) { const intersection = RoaringBitmap32.and(vfsBitmap, typeBitmap) if (intersection.size > 0) { @@ -3486,18 +3469,21 @@ export class MetadataIndexManager implements MetadataIndexProvider { * Tracks which fields commonly appear with which entity types */ private updateTypeFieldAffinity(entityId: string, field: string, value: any, operation: 'add' | 'remove', metadata?: any): void { - // Only track affinity for non-system fields (but allow 'noun' for type detection) - if (this.config.excludeFields.includes(field) && field !== 'noun') return + // Only track affinity for user fields (plus the type column itself, + // which drives detection). Engine columns carry the literal 'system.' + // prefix under the frozen key format. + if (field.startsWith('system.') && field !== 'system.type') return - // For the 'noun' field, the value IS the entity type + // For the type column ('system.type'), the value IS the entity type let entityType: string | null = null - if (field === 'noun') { + if (field === 'system.type') { // This is the type definition itself entityType = this.normalizeValue(value, field) // Pass field for bucketing! - } else if (metadata && metadata.noun) { - // Extract entity type from metadata - entityType = this.normalizeValue(metadata.noun, 'noun') + } else if (metadata && (metadata.noun ?? metadata.type)) { + // Extract entity type from the source shape: stored records carry it + // under 'noun', entity-for-indexing views under 'type'. + entityType = this.normalizeValue(metadata.noun ?? metadata.type, 'system.type') } else { // No type information available, skip affinity tracking return @@ -3520,8 +3506,9 @@ export class MetadataIndexManager implements MetadataIndexProvider { const currentCount = typeFields.get(field) || 0 typeFields.set(field, currentCount + 1) - // Update total entities of this type (only count once per entity) - if (field === 'noun') { + // Update total entities of this type (only count once per entity — + // the type column appears exactly once per entity) + if (field === 'system.type') { const newCount = this.totalEntitiesByType.get(entityType)! + 1 this.totalEntitiesByType.set(entityType, newCount) @@ -3544,7 +3531,7 @@ export class MetadataIndexManager implements MetadataIndexProvider { } // Update total entities of this type - if (field === 'noun') { + if (field === 'system.type') { const total = this.totalEntitiesByType.get(entityType)! if (total > 1) { const newCount = total - 1 diff --git a/src/utils/paramValidation.ts b/src/utils/paramValidation.ts index 749849b3..359413d7 100644 --- a/src/utils/paramValidation.ts +++ b/src/utils/paramValidation.ts @@ -618,6 +618,7 @@ export function validateUpdateParams(params: UpdateParams): void { * Validate relate parameters */ export function validateRelateParams(params: RelateParams): void { + rejectForgedSystemKeys(params.metadata as Record | undefined, 'relate()') // 8.0 verb-id contract (L.7): verb ids are UUIDs, generated by brainy. // RelateParams has no `id` field — an untyped caller passing one would // previously have it silently ignored (a generated UUID was used instead). @@ -666,6 +667,7 @@ export function validateRelateParams(params: RelateParams): void { * accepts type/subtype/weight/confidence/data/metadata changes. */ export function validateUpdateRelationParams(params: UpdateRelationParams): void { + rejectForgedSystemKeys(params.metadata as Record | undefined, 'updateRelation()') if (!params.id) { throw new Error('id is required for updateRelation') } diff --git a/tests/conformance/collider-fidelity.test.ts b/tests/conformance/collider-fidelity.test.ts new file mode 100644 index 00000000..61c9413d --- /dev/null +++ b/tests/conformance/collider-fidelity.test.ts @@ -0,0 +1,307 @@ +/** + * @module tests/conformance/collider-fidelity + * @description THE REOPEN-COLLIDER CONFORMANCE CASE (required cross-engine + * before any RC counts as gates-green — ruled 2026-08-03). The + * field-addressing law's fidelity half: user metadata may carry ANY name — + * including every engine spelling (`confidence`, `type`, `id`, `createdAt`, + * …) and every plumbing name (`level`, `data`, `vector`, `_rev`) — and the + * value survives, verbatim and reachable, across the FULL lifecycle: live + * reads, where/orderBy, flush, close+reopen, a forced epoch rebuild, and + * time travel. The engine scalars stay separately reachable at `system.*` + * the whole way. No halfway states. + * + * Self-arming like the namespace-law suite: skips loudly until the arming + * exports are present, so the suite can sit on a branch ahead of the build. + */ +import { describe, it, expect, beforeAll, afterAll } from 'vitest' +import { mkdtempSync, rmSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import * as brainyExports from '../../src/index.js' +import { Brainy, NounType, VerbType } from '../../src/index.js' +import { + BRAIN_FORMAT_PATH, + EXPECTED_INDEX_EPOCH +} from '../../src/storage/brainFormat.js' + +const ARMED = 'UnresolvableFieldError' in brainyExports +const suite = ARMED ? describe : describe.skip +if (!ARMED) { + // eslint-disable-next-line no-console + console.warn( + '[collider-fidelity] SKIPPING: package root does not export the ' + + 'field-addressing law surface yet (UnresolvableFieldError absent).' + ) +} + +/** Every entity system scalar name written as a USER metadata field, with + * unmistakable user values, plus the plumbing names and naturals. */ +const COLLIDER_BAG = { + // the ten entity system scalars, as user fields + id: 'user-id', + type: 'user-type', + subtype: 'user-subtype', + createdAt: 'user-createdAt', + updatedAt: 'user-updatedAt', + confidence: 'user-confidence', + weight: 'user-weight', + visibility: 'user-visibility', + service: 'user-service', + createdBy: 'user-createdBy', + // plumbing names, as user fields + level: 7, + data: 'user-data', + vector: 'user-vector', + _rev: 'user-rev', + // naturals previously silently un-indexed by name + content: 'user-content', + // a plain control field + plain: 'control' +} as const + + +suite('collider fidelity — the reopen-collider case (both suites, ruled)', () => { + let dir: string + let brain: Brainy + let colliderId: string + + const open = async (): Promise => { + const b = new Brainy({ + storage: { type: 'filesystem', path: dir }, + requireSubtype: false + }) + await b.init() + return b + } + + /** The full read battery — run at every lifecycle boundary. */ + const verifyColliderTruth = async (label: string): Promise => { + // 1. get(): the bag comes back verbatim; engine scalars stay engine. + const entity = await brain.get(colliderId) + expect(entity, `${label}: entity readable`).toBeTruthy() + for (const [k, v] of Object.entries(COLLIDER_BAG)) { + expect( + (entity!.metadata as Record)[k], + `${label}: bag.${k} verbatim` + ).toEqual(v) + } + expect(entity!.type, `${label}: engine type intact`).toBe(NounType.Document) + expect(entity!.confidence, `${label}: engine confidence intact`).toBe(0.25) + + // 2. where on collider names (bare = the user's field, always). + for (const [k, v] of [ + ['confidence', 'user-confidence'], + ['type', 'user-type'], + ['id', 'user-id'], + ['content', 'user-content'], + ['data', 'user-data'], + ['level', 7] + ] as const) { + const rows = await brain.find({ where: { [k]: v }, limit: 10 }) + expect( + rows.map((r) => r.id), + `${label}: where {${k}} finds the collider row` + ).toContain(colliderId) + } + + // 3. system.* keeps reading the ENGINE values. + const byEngine = await brain.find({ + where: { 'system.confidence': 0.25 }, + limit: 10 + }) + expect( + byEngine.map((r) => r.id), + `${label}: system.confidence reads the engine scalar` + ).toContain(colliderId) + const byUserSpelledSystem = await brain.find({ + where: { 'system.confidence': 'user-confidence' }, + limit: 10 + }) + expect( + byUserSpelledSystem.map((r) => r.id), + `${label}: the user's value is NOT reachable via system.*` + ).not.toContain(colliderId) + + // 4. orderBy a collider name orders by the USER values. + const ordered = await brain.find({ + type: NounType.Document, + orderBy: 'level', + order: 'desc', + limit: 10 + }) + expect(ordered.length, `${label}: ordered read complete`).toBe(3) + expect( + (ordered[0].metadata as Record).plain, + `${label}: user level orders desc (7 first)` + ).toBe('control') + } + + beforeAll(async () => { + dir = mkdtempSync(join(tmpdir(), 'brainy-collider-')) + brain = await open() + + colliderId = await brain.add({ + data: 'the collider probe document', + type: NounType.Document, + confidence: 0.25, + metadata: { ...COLLIDER_BAG } + }) + // two ordering companions with smaller user `level`s + await brain.add({ + data: 'ordering companion low', + type: NounType.Document, + metadata: { level: 3, plain: 'low' } + }) + await brain.add({ + data: 'ordering companion mid', + type: NounType.Document, + metadata: { level: 5, plain: 'mid' } + }) + }, 120000) + + afterAll(async () => { + await brain.close().catch(() => {}) + rmSync(dir, { recursive: true, force: true }) + }) + + it('LIVE: colliders are the user’s, verbatim and fully queryable', async () => { + await verifyColliderTruth('live') + }) + + it('REOPEN: the restart boundary loses nothing', async () => { + await brain.flush() + await brain.close() + brain = await open() + await verifyColliderTruth('reopen') + }) + + it('REBUILD: a forced epoch rebuild re-indexes the colliders from canonical', async () => { + await brain.close() + // Simulate epoch drift: a missing marker forces the full derived-index + // rebuild at open — the exact path every pre-law brain takes once. + rmSync(join(dir, BRAIN_FORMAT_PATH), { force: true }) + brain = await open() + await verifyColliderTruth('rebuild') + // And the rebuild re-stamps the current epoch. + const marker = await ( + brain as unknown as { + storage: { readRawObject(p: string): Promise<{ indexEpoch?: number } | null> } + } + ).storage.readRawObject(BRAIN_FORMAT_PATH) + expect(marker?.indexEpoch).toBe(EXPECTED_INDEX_EPOCH) + }) + + it('TIME TRAVEL: asOf reads historical collider values faithfully', async () => { + const gen = brain.generation() + await brain.update({ id: colliderId, metadata: { confidence: 'user-confidence-v2' } }) + const now = await brain.get(colliderId) + expect((now!.metadata as Record).confidence).toBe('user-confidence-v2') + + const past = await brain.asOf(gen) + try { + const then = await past.get(colliderId) + expect( + (then!.metadata as Record).confidence, + 'asOf reads the pre-update USER value' + ).toBe('user-confidence') + } finally { + await past.release() + } + // engine scalar untouched throughout + expect(now!.confidence).toBe(0.25) + }) + + it('RELATION MIRROR: edge collider bags survive write → read → reopen', async () => { + const a = await brain.add({ data: 'edge endpoint a', type: NounType.Person, metadata: { plain: 'a' } }) + const b = await brain.add({ data: 'edge endpoint b', type: NounType.Person, metadata: { plain: 'b' } }) + const edgeBag = { + verb: 'user-verb', + confidence: 'user-edge-confidence', + weight: 'user-edge-weight', + subtype: 'user-edge-subtype', + createdAt: 'user-edge-createdAt', + service: 'user-edge-service' + } + const relId = await brain.relate({ + from: a, + to: b, + type: VerbType.RelatedTo, + confidence: 0.5, + metadata: { ...edgeBag } + }) + + const check = async (label: string): Promise => { + const rels = await brain.related({ from: a, type: VerbType.RelatedTo }) + const rel = rels.find((r) => r.id === relId) + expect(rel, `${label}: relation readable`).toBeTruthy() + for (const [k, v] of Object.entries(edgeBag)) { + expect( + (rel!.metadata as Record)[k], + `${label}: edge bag.${k} verbatim` + ).toEqual(v) + } + expect(rel!.confidence, `${label}: engine edge confidence intact`).toBe(0.5) + expect(rel!.type, `${label}: engine verb intact`).toBe(VerbType.RelatedTo) + } + + await check('live') + await brain.flush() + await brain.close() + brain = await open() + await check('reopen') + }) + + it('FORGERY: user metadata keys spelled system.* refuse at every write door', async () => { + await expect( + brain.add({ data: 'forged', type: NounType.Document, metadata: { 'system.confidence': 1 } }) + ).rejects.toThrow(/system\./) + await expect( + brain.update({ id: colliderId, metadata: { 'system.type': 'x' } }) + ).rejects.toThrow(/system\./) + const a = await brain.add({ data: 'forgery endpoint a', type: NounType.Person, metadata: {} }) + const b = await brain.add({ data: 'forgery endpoint b', type: NounType.Person, metadata: {} }) + await expect( + brain.relate({ from: a, to: b, type: VerbType.RelatedTo, metadata: { 'system.verb': 'x' } }) + ).rejects.toThrow(/system\./) + }) + + it('CONFIG: the dead reservedFieldPolicy option refuses loudly, never ignored', () => { + expect( + () => new Brainy({ storage: { type: 'memory' }, reservedFieldPolicy: 'throw' } as never) + ).toThrow(/field-addressing law/) + }) + + it('LEGACY: a pre-law flat record still reads with engine fields top-level', async () => { + const storage = ( + brain as unknown as { + storage: { + saveNoun(n: unknown): Promise + saveNounMetadata(id: string, m: Record): Promise + } + } + ).storage + const legacyId = '00000000-0000-4000-8000-00000000f1a7' + await storage.saveNoun({ id: legacyId, vector: new Array(384).fill(0.01), connections: new Map(), level: 0 }) + // Legacy FLAT shape: engine + user keys mixed at one level, NO _fmt stamp. + // Sound to split by name — the pre-law door refused user colliders. + await storage.saveNounMetadata(legacyId, { + noun: NounType.Document, + confidence: 0.75, + createdAt: 1700000000000, + updatedAt: 1700000000000, + _rev: 1, + legacyField: 'legacy-value' + }) + const entity = await brain.get(legacyId) + expect(entity).toBeTruthy() + expect(entity!.confidence, 'legacy flat confidence = engine').toBe(0.75) + expect( + (entity!.metadata as Record).legacyField, + 'legacy custom field = user bag' + ).toBe('legacy-value') + expect( + (entity!.metadata as Record).confidence, + 'legacy flat engine key never leaks into the bag' + ).toBeUndefined() + }) +}) diff --git a/tests/integration/advanced-apis-regression.test.ts b/tests/integration/advanced-apis-regression.test.ts index 12c39112..069d3e62 100644 --- a/tests/integration/advanced-apis-regression.test.ts +++ b/tests/integration/advanced-apis-regression.test.ts @@ -164,19 +164,19 @@ describe('BR-ADV-FEATURES-BUN regression', () => { await b.close() }) - it('groupBy "noun" resolves to the entity type, not null', async () => { + it('groupBy "system.type" resolves to the entity type, not null (the legacy "noun" alias is dead)', async () => { const b: any = new Brainy({ requireSubtype: false, storage: { type: 'memory' } }) await b.init() await b.add({ data: 'p', type: NounType.Person }) b.defineAggregate({ name: 'byNoun', source: { type: NounType.Person }, - groupBy: ['noun'], + groupBy: ['system.type'], metrics: { count: { op: 'count' } } }) const rows: any[] = await b.find({ aggregate: 'byNoun' }) expect(rows.length).toBe(1) - expect(rows[0].groupKey.noun).toBe(NounType.Person) + expect(rows[0].groupKey['system.type']).toBe(NounType.Person) await b.close() }) }) diff --git a/tests/integration/aggregate-reserved-fields.test.ts b/tests/integration/aggregate-reserved-fields.test.ts index e81692d6..b8c11b4f 100644 --- a/tests/integration/aggregate-reserved-fields.test.ts +++ b/tests/integration/aggregate-reserved-fields.test.ts @@ -42,8 +42,11 @@ describe('aggregation + query field-resolution law', () => { it('reserved-field groupBy decrements on delete (the drift bug)', async () => { brain.defineAggregate({ name: 'by_subtype', + // system.subtype — subtype is an add() param (an engine scalar), never + // a user metadata field; bare 'subtype' now addresses the user's own + // metadata bag under the sealed field-addressing law. source: { type: NounType.Document }, - groupBy: ['subtype'], + groupBy: ['system.subtype'], metrics: { count: { op: 'count' } } }) @@ -60,7 +63,7 @@ describe('aggregation + query field-resolution law', () => { } let groups = await brain.queryAggregate('by_subtype') expect(groups).toHaveLength(1) - expect(groups[0].groupKey).toEqual({ subtype: 'note' }) + expect(groups[0].groupKey).toEqual({ 'system.subtype': 'note' }) expect(groups[0].metrics.count).toBe(5) await brain.remove(ids[0]) @@ -76,7 +79,7 @@ describe('aggregation + query field-resolution law', () => { brain.defineAggregate({ name: 'by_subtype', source: { type: NounType.Document }, - groupBy: ['subtype'], + groupBy: ['system.subtype'], metrics: { count: { op: 'count' } } }) const id = await brain.add({ @@ -88,7 +91,7 @@ describe('aggregation + query field-resolution law', () => { const groups = await brain.queryAggregate('by_subtype') const byKey = Object.fromEntries( - groups.map((g) => [String(g.groupKey.subtype), g.metrics.count]) + groups.map((g) => [String(g.groupKey['system.subtype']), g.metrics.count]) ) expect(byKey['published']).toBe(1) // The old group must be gone or zero — never still counting the entity. @@ -98,7 +101,7 @@ describe('aggregation + query field-resolution law', () => { it('source.where on a reserved field filters instead of matching nothing', async () => { brain.defineAggregate({ name: 'notes_only', - source: { type: NounType.Document, where: { subtype: 'note' } }, + source: { type: NounType.Document, where: { 'system.subtype': 'note' } }, groupBy: ['team'], metrics: { count: { op: 'count' } } }) diff --git a/tests/integration/all-apis-comprehensive.test.ts b/tests/integration/all-apis-comprehensive.test.ts index d25f23c8..7d82afea 100644 --- a/tests/integration/all-apis-comprehensive.test.ts +++ b/tests/integration/all-apis-comprehensive.test.ts @@ -331,8 +331,10 @@ describe('Comprehensive All-APIs Test', () => { it('should handle metadata queries efficiently', async () => { const start = Date.now() + // system.type — the legacy where.type→noun alias is dead; bare 'type' + // in where now addresses the user's own metadata field. const results = await brain.find({ - where: { type: NounType.Document }, + where: { 'system.type': NounType.Document }, limit: 100 }) diff --git a/tests/integration/fact-log-dual-write.test.ts b/tests/integration/fact-log-dual-write.test.ts index 5ec66273..3c4eee4a 100644 --- a/tests/integration/fact-log-dual-write.test.ts +++ b/tests/integration/fact-log-dual-write.test.ts @@ -11,7 +11,7 @@ import { describe, it, expect, beforeEach, afterEach } from 'vitest' import * as fs from 'node:fs' import * as os from 'node:os' import * as path from 'node:path' -import { Brainy, ProtectedArtifactError, type CommitFact } from '../../src/index.js' +import { Brainy, ProtectedArtifactError, splitNounMetadataRecord, type CommitFact } from '../../src/index.js' async function allFacts(brain: any): Promise { const scan = brain.scanFacts() @@ -65,7 +65,13 @@ describe('fact log dual-write (memory adapter)', () => { const updateFact = facts[facts.length - 1] const op = updateFact.ops.find((o) => o.id === id)! expect(op.record).not.toBeNull() - expect((op.record!.metadata as any).v).toBe('new') + // The fact log is byte-faithful: op.record.metadata is the RAW stored + // record (v2 nested-bag since the field-addressing law) — read the user + // field through the shape-aware split, like every other reader. + const { custom } = splitNounMetadataRecord( + op.record!.metadata as Record + ) + expect(custom.v).toBe('new') }) it('a transact commits ONE fact carrying all its ops, with meta', async () => { diff --git a/tests/integration/lens-consistency.test.ts b/tests/integration/lens-consistency.test.ts index 64484a38..1b1cef81 100644 --- a/tests/integration/lens-consistency.test.ts +++ b/tests/integration/lens-consistency.test.ts @@ -2,8 +2,8 @@ * @module tests/integration/lens-consistency * @description The three metadata "lenses" over one corpus must agree with * canonical ground truth id-for-id, warm AND after a cold reopen: - * - combined: find({ type: T, where: { subtype: S } }) - * - subtype-only: find({ where: { subtype: S } }) + * - combined: find({ type: T, where: { 'system.subtype': S } }) + * - subtype-only: find({ where: { 'system.subtype': S } }) * - type-only: find({ type: T }) * Ported from the fresh-brain probe that closed the type+subtype lens-drop * investigation (a restored pre-8.2.2 torn capture had entities visible to the @@ -63,8 +63,11 @@ async function assertAllLenses(brain: any): Promise { const subtypes = [...new Set(CORPUS.map((c) => c.subtype))] for (const { type, subtype } of CORPUS) { - const combined = idSet(await brain.find({ type, where: { subtype }, limit: 1000 })) - const subtypeOnly = idSet(await brain.find({ where: { subtype }, limit: 1000 })) + // system.subtype — subtype is an add()/update() param (an engine scalar), + // never a user metadata field; bare 'subtype' now addresses the user's + // own metadata bag under the sealed field-addressing law. + const combined = idSet(await brain.find({ type, where: { 'system.subtype': subtype }, limit: 1000 })) + const subtypeOnly = idSet(await brain.find({ where: { 'system.subtype': subtype }, limit: 1000 })) const truthPair = await groundTruth(brain, { type, subtype }) const truthSubtype = await groundTruth(brain, { subtype }) @@ -82,7 +85,7 @@ async function assertAllLenses(brain: any): Promise { // Count cross-check against the corpus definition itself. for (const subtype of subtypes) { const expected = CORPUS.filter((c) => c.subtype === subtype).reduce((s, c) => s + c.count, 0) - const got = (await brain.find({ where: { subtype }, limit: 1000 })).length + const got = (await brain.find({ where: { 'system.subtype': subtype }, limit: 1000 })).length expect(got).toBe(expected) } } @@ -123,16 +126,16 @@ describe('lens consistency — combined vs subtype-only vs canonical ground trut it('after an update() flips type AND subtype, every lens tracks the move exactly', async () => { // The historical cross-bucket-staleness path: change (concept, action) -> (task, review). - const victims = await brain.find({ type: 'concept', where: { subtype: 'action' }, limit: 1 }) + const victims = await brain.find({ type: 'concept', where: { 'system.subtype': 'action' }, limit: 1 }) expect(victims.length).toBe(1) const id = victims[0].id await brain.update({ id, type: 'task', subtype: 'review' }) - const oldCombined = idSet(await brain.find({ type: 'concept', where: { subtype: 'action' }, limit: 1000 })) + const oldCombined = idSet(await brain.find({ type: 'concept', where: { 'system.subtype': 'action' }, limit: 1000 })) expect(oldCombined.has(id)).toBe(false) // unposted from the old buckets - const newCombined = idSet(await brain.find({ type: 'task', where: { subtype: 'review' }, limit: 1000 })) + const newCombined = idSet(await brain.find({ type: 'task', where: { 'system.subtype': 'review' }, limit: 1000 })) expect(newCombined.has(id)).toBe(true) // posted to the new buckets - const subtypeOnly = idSet(await brain.find({ where: { subtype: 'review' }, limit: 1000 })) + const subtypeOnly = idSet(await brain.find({ where: { 'system.subtype': 'review' }, limit: 1000 })) expect(subtypeOnly.has(id)).toBe(true) }) }) diff --git a/tests/integration/migration.test.ts b/tests/integration/migration.test.ts index daa5fb2d..f6c3741b 100644 --- a/tests/integration/migration.test.ts +++ b/tests/integration/migration.test.ts @@ -20,6 +20,16 @@ import { MigrationRunner, MIGRATIONS } from '../../src/migration/index.js' import type { Migration } from '../../src/migration/index.js' import { NounType, VerbType } from '../../src/types/graphTypes.js' +// THE VIEW CONTRACT (field-addressing law): transforms receive engine fields +// top-level and the USER's bag nested under `metadata` — user-field changes +// go inside the bag. These two helpers keep the one-liner migrations tidy. +const bagOf = (m: Record): Record => + m.metadata as Record +const withBag = ( + m: Record, + patch: Record +): Record => ({ ...m, metadata: { ...bagOf(m), ...patch } }) + // Helper to temporarily inject migrations into the MIGRATIONS array function withMigrations(migrations: Migration[], fn: () => Promise): Promise { const original = MIGRATIONS.splice(0, MIGRATIONS.length) @@ -78,9 +88,11 @@ describe('Migration System', () => { description: 'Add version field to entities with status', applies: 'nouns', transform: (m) => { - // Only transform entities that have our specific 'status' field - if ('status' in m && !('version' in m)) { - return { ...m, version: 1 } + // Only transform entities that have our specific 'status' USER field + // (user fields live in the nested bag — the view contract). + const bag = m.metadata as Record + if ('status' in bag && !('version' in bag)) { + return { ...m, metadata: { ...bag, version: 1 } } } return null } @@ -94,7 +106,8 @@ describe('Migration System', () => { // All 3 entities have 'status' metadata expect(p.affectedEntities).toBeGreaterThanOrEqual(3) expect(p.sampleChanges.length).toBeGreaterThan(0) - expect(p.sampleChanges[0].after.version).toBe(1) + // Samples carry the VIEW shape: user fields inside `.metadata`. + expect(p.sampleChanges[0].after.metadata.version).toBe(1) // Verify no data was modified (dry-run) const entity = await brain.get(id1) @@ -111,9 +124,10 @@ describe('Migration System', () => { description: 'Rename state to status', applies: 'nouns', transform: (m) => { - if ('state' in m) { - const { state, ...rest } = m - return { ...rest, status: state } + const bag = m.metadata as Record + if ('state' in bag) { + const { state, ...rest } = bag + return { ...m, metadata: { ...rest, status: state } } } return null } @@ -124,11 +138,12 @@ describe('Migration System', () => { const p = preview as any expect(p.sampleChanges.length).toBeGreaterThanOrEqual(1) - // Find the sample for our entity (it has the 'state' field) - const sample = p.sampleChanges.find((s: any) => s.before.state === 'draft') + // Find the sample for our entity (it has the 'state' USER field — + // samples carry the VIEW shape, user fields inside `.metadata`) + const sample = p.sampleChanges.find((s: any) => s.before.metadata.state === 'draft') expect(sample).toBeDefined() - expect(sample.after.status).toBe('draft') - expect(sample.after.state).toBeUndefined() + expect(sample.after.metadata.status).toBe('draft') + expect(sample.after.metadata.state).toBeUndefined() }) }) }) @@ -149,8 +164,8 @@ describe('Migration System', () => { description: 'Add migrated flag to entities with priority', applies: 'nouns', transform: (m) => { - if ('priority' in m && !('migrated' in m)) { - return { ...m, migrated: true } + if ('priority' in bagOf(m) && !('migrated' in bagOf(m))) { + return withBag(m, { migrated: true }) } return null } @@ -179,8 +194,8 @@ describe('Migration System', () => { description: 'Uppercase status field only when present', applies: 'nouns', transform: (m) => { - if (typeof m.status === 'string') { - return { ...m, status: (m.status as string).toUpperCase() } + if (typeof bagOf(m).status === 'string') { + return withBag(m, { status: (bagOf(m).status as string).toUpperCase() }) } return null } @@ -203,7 +218,7 @@ describe('Migration System', () => { version: '1.0.0', description: 'Double count', applies: 'nouns', - transform: (m) => typeof m.count === 'number' ? { ...m, count: (m.count as number) * 2 } : null + transform: (m) => typeof bagOf(m).count === 'number' ? withBag(m, { count: (bagOf(m).count as number) * 2 }) : null } const migration2: Migration = { @@ -211,7 +226,7 @@ describe('Migration System', () => { version: '1.1.0', description: 'Add 10 to count', applies: 'nouns', - transform: (m) => typeof m.count === 'number' ? { ...m, count: (m.count as number) + 10 } : null + transform: (m) => typeof bagOf(m).count === 'number' ? withBag(m, { count: (bagOf(m).count as number) + 10 }) : null } await withMigrations([migration1, migration2], async () => { @@ -229,7 +244,7 @@ describe('Migration System', () => { version: '1.0.0', description: 'Increment v', applies: 'nouns', - transform: (m) => typeof m.v === 'number' ? { ...m, v: (m.v as number) + 1 } : null + transform: (m) => typeof bagOf(m).v === 'number' ? withBag(m, { v: (bagOf(m).v as number) + 1 }) : null } await withMigrations([migration], async () => { @@ -266,7 +281,7 @@ describe('Migration System', () => { version: '2.0.0', description: 'Add y field to entities with x', applies: 'nouns', - transform: (m) => 'x' in m && !('y' in m) ? { ...m, y: 2 } : null + transform: (m) => 'x' in bagOf(m) && !('y' in bagOf(m)) ? withBag(m, { y: 2 }) : null } await withMigrations([migration], async () => { @@ -290,8 +305,8 @@ describe('Migration System', () => { description: 'Replace original with migrated', applies: 'nouns', transform: (m) => { - if (m.original === true) { - return { ...m, original: false, migrated: true } + if (bagOf(m).original === true) { + return withBag(m, { original: false, migrated: true }) } return null } @@ -323,7 +338,7 @@ describe('Migration System', () => { version: '4.0.0', description: 'Add field', applies: 'nouns', - transform: (m) => 'q' in m && !('r' in m) ? { ...m, r: 2 } : null + transform: (m) => 'q' in bagOf(m) && !('r' in bagOf(m)) ? withBag(m, { r: 2 }) : null } await withMigrations([migration], async () => { @@ -384,7 +399,7 @@ describe('Migration System', () => { version: '1.0.0', description: 'Auto migrate test', applies: 'nouns', - transform: (m) => 'legacy' in m ? { ...m, legacy: false, upgraded: true } : null + transform: (m) => 'legacy' in bagOf(m) ? withBag(m, { legacy: false, upgraded: true }) : null } await withMigrations([migration], async () => { @@ -410,7 +425,7 @@ describe('Migration System', () => { version: '1.0.0', description: 'Add y to entities with x', applies: 'nouns', - transform: (m) => 'x' in m ? { ...m, y: true } : null + transform: (m) => 'x' in bagOf(m) ? withBag(m, { y: true }) : null } const progressCalls: any[] = [] @@ -444,7 +459,7 @@ describe('Migration System', () => { version: '1.0.0', description: 'Increment v on entities that have it', applies: 'nouns', - transform: (m) => typeof m.v === 'number' ? { ...m, v: (m.v as number) + 1 } : null + transform: (m) => typeof bagOf(m).v === 'number' ? withBag(m, { v: (bagOf(m).v as number) + 1 }) : null } await withMigrations([migration], async () => { @@ -477,9 +492,10 @@ describe('Migration System', () => { description: 'Rename strength to intensity', applies: 'verbs', transform: (m) => { - if ('strength' in m) { - const { strength, ...rest } = m - return { ...rest, intensity: strength } + const bag = bagOf(m) + if ('strength' in bag) { + const { strength, ...rest } = bag + return { ...m, metadata: { ...rest, intensity: strength } } } return null } @@ -507,7 +523,7 @@ describe('Migration System', () => { version: '1.0.0', description: 'Update tag from old to new', applies: 'both', - transform: (m) => m.tag === 'old' ? { ...m, tag: 'new' } : null + transform: (m) => bagOf(m).tag === 'old' ? withBag(m, { tag: 'new' }) : null } await withMigrations([migration], async () => { @@ -577,11 +593,11 @@ describe('Migration System', () => { description: 'Transform that throws on non-number values', applies: 'nouns', transform: (m) => { - if ('value' in m) { - if (typeof m.value !== 'number') { + if ('value' in bagOf(m)) { + if (typeof bagOf(m).value !== 'number') { throw new Error('value must be a number') } - return { ...m, value: (m.value as number) * 10 } + return withBag(m, { value: (bagOf(m).value as number) * 10 }) } return null } @@ -615,7 +631,7 @@ describe('Migration System', () => { description: 'Always throws', applies: 'nouns', transform: (m) => { - if ('boom' in m) { + if ('boom' in bagOf(m)) { throw new Error('deliberate failure') } return null diff --git a/tests/integration/orderby-sort-bug.test.ts b/tests/integration/orderby-sort-bug.test.ts index db40fe12..aeb7ff66 100644 --- a/tests/integration/orderby-sort-bug.test.ts +++ b/tests/integration/orderby-sort-bug.test.ts @@ -56,7 +56,7 @@ describe('find({ orderBy }) sort bug regression', () => { const results = await brain.find({ type: NounType.Concept, - orderBy: 'createdAt', + orderBy: 'system.createdAt', order: 'desc', limit: 1 }) @@ -76,7 +76,7 @@ describe('find({ orderBy }) sort bug regression', () => { const results = await brain.find({ type: NounType.Concept, - orderBy: 'createdAt', + orderBy: 'system.createdAt', order: 'asc', limit: 1 }) @@ -94,7 +94,7 @@ describe('find({ orderBy }) sort bug regression', () => { const results = await brain.find({ type: NounType.Concept, - orderBy: 'createdAt', + orderBy: 'system.createdAt', order: 'desc' }) @@ -115,7 +115,7 @@ describe('find({ orderBy }) sort bug regression', () => { const results = await brain.find({ type: NounType.Concept, - orderBy: 'updatedAt', + orderBy: 'system.updatedAt', order: 'desc', limit: 1 }) @@ -136,7 +136,7 @@ describe('find({ orderBy }) sort bug regression', () => { const id3 = await brain.add({ data: 'third', type: NounType.Concept }) const results = await brain.find({ - orderBy: 'createdAt', + orderBy: 'system.createdAt', order: 'desc', limit: 2 }) diff --git a/tests/regression/metadata-index-cleanup.unit.test.ts b/tests/regression/metadata-index-cleanup.unit.test.ts index 0984d727..3746e833 100644 --- a/tests/regression/metadata-index-cleanup.unit.test.ts +++ b/tests/regression/metadata-index-cleanup.unit.test.ts @@ -244,7 +244,10 @@ describe('Metadata index cleanup after remove / removeMany', () => { const noConfidenceId = await addEntity({ type: 'thing' }) const withConfidenceId = await addEntity({ type: 'thing', confidence: 0.9 }) - const results = await brain.find({ where: { confidence: { exists: true } } }) + // system.confidence — confidence is an engine scalar (an add() param), + // never a metadata field; bare 'confidence' now addresses the user's + // own metadata bag under the sealed field-addressing law. + const results = await brain.find({ where: { 'system.confidence': { exists: true } } }) const ids = results.map(r => r.id) expect(ids).toContain(withConfidenceId) @@ -255,7 +258,8 @@ describe('Metadata index cleanup after remove / removeMany', () => { const noWeightId = await addEntity({ type: 'thing' }) const withWeightId = await addEntity({ type: 'thing', weight: 0.5 }) - const results = await brain.find({ where: { weight: { exists: true } } }) + // system.weight — same reasoning as system.confidence above. + const results = await brain.find({ where: { 'system.weight': { exists: true } } }) const ids = results.map(r => r.id) expect(ids).toContain(withWeightId) @@ -269,11 +273,12 @@ describe('Metadata index cleanup after remove / removeMany', () => { const id = await addEntity({ type: 'thing' }) await brain.remove(id) - // Entity must not appear in any confidence query - const existsTrue = await brain.find({ where: { confidence: { exists: true } } }) + // Entity must not appear in any confidence query. system.confidence — + // same addressing as the two tests above. + const existsTrue = await brain.find({ where: { 'system.confidence': { exists: true } } }) expect(existsTrue.map(r => r.id)).not.toContain(id) - const existsFalse = await brain.find({ where: { confidence: { exists: false } } }) + const existsFalse = await brain.find({ where: { 'system.confidence': { exists: false } } }) expect(existsFalse.map(r => r.id)).not.toContain(id) }) }) diff --git a/tests/unit/brainy/find-orderby-pagek.test.ts b/tests/unit/brainy/find-orderby-pagek.test.ts index 9a453f8d..49fccb02 100644 --- a/tests/unit/brainy/find-orderby-pagek.test.ts +++ b/tests/unit/brainy/find-orderby-pagek.test.ts @@ -42,7 +42,8 @@ describe('find({ where, orderBy }) bounds the sort to the page (CTX-BR-FIND-ORDE return real(f, ob, o, topK) } - const results = await brain.find({ where: { bucket: 'x' }, orderBy: 'createdAt', order: 'desc', limit: 5 }) + // system.createdAt — entity age, not a user metadata field named 'createdAt'. + const results = await brain.find({ where: { bucket: 'x' }, orderBy: 'system.createdAt', order: 'desc', limit: 5 }) expect(results).toHaveLength(5) // Page-bounded: ~ limit (5) + a small hidden-tier over-fetch — NOT all 50 matches. diff --git a/tests/unit/brainy/reserved-field-policy.test.ts b/tests/unit/brainy/reserved-field-policy.test.ts deleted file mode 100644 index c5f37af4..00000000 --- a/tests/unit/brainy/reserved-field-policy.test.ts +++ /dev/null @@ -1,251 +0,0 @@ -/** - * @module tests/unit/brainy/reserved-field-policy - * @description The 8.0 `reservedFieldPolicy` matrix — what happens when an - * untyped (JavaScript) caller smuggles a Brainy-reserved field INSIDE the - * `metadata` bag of a write call, past the compile-time guard. - * - * 8.0 is a clean break with no silent failures. The decided contract: - * - `'throw'` (DEFAULT): a reserved key in the bag throws a clear Error naming - * the offending key(s) and the correct write path. No remap, no data loss. - * - `'warn'`: legacy remap PLUS a one-shot (per method+field, per process) - * warning for EVERY reserved key found. - * - `'remap'`: the pre-8.0 silent remap, no warning. - * - * The deep correctness of the remap itself (top-level precedence, system-managed - * drops, transact()/with() mirrors, read-side splitting) lives in - * tests/unit/brainy/update-reserved-metadata-remap.test.ts (which now runs under - * `reservedFieldPolicy: 'remap'`). This file pins the POLICY SELECTION and the - * throw/warn behaviors. - * - * Compile-time callers can't write these shapes at all (see - * tests/unit/types/reserved-metadata-keys.test-d.ts); the `as object` widenings - * below simulate untyped callers. - */ - -import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest' -import { Brainy } from '../../../src/index.js' -import { NounType, VerbType } from '../../../src/types/graphTypes.js' -import { createTestConfig } from '../../helpers/test-factory.js' -import { prodLog } from '../../../src/utils/logger.js' - -describe('reservedFieldPolicy', () => { - describe("default policy is 'throw'", () => { - let brain: Brainy - - beforeEach(async () => { - // No reservedFieldPolicy override → resolves to 'throw'. - brain = new Brainy(createTestConfig()) - await brain.init() - }) - - afterEach(async () => { - await brain.close() - }) - - it('add() throws naming the offending key and the correct write path', async () => { - await expect( - brain.add({ - type: NounType.Concept, - subtype: 'general', - data: 'x', - metadata: { confidence: 0.8 } as object - }) - ).rejects.toThrow(/metadata\.confidence is a reserved field/) - - // The error names the right param and the reserved list for discoverability. - await expect( - brain.add({ - type: NounType.Concept, - subtype: 'general', - data: 'x', - metadata: { confidence: 0.8 } as object - }) - ).rejects.toThrow(/'confidence' param.*RESERVED_ENTITY_FIELDS/s) - }) - - it('add() lists EVERY offending key when several are present', async () => { - const err = await brain - .add({ - type: NounType.Person, - data: 'multi', - metadata: { confidence: 0.5, weight: 0.6, subtype: 'employee' } as object - }) - .catch((e) => e as Error) - expect(err).toBeInstanceOf(Error) - expect(err.message).toMatch(/confidence/) - expect(err.message).toMatch(/weight/) - expect(err.message).toMatch(/subtype/) - }) - - it('update() throws on a reserved key in the patch', async () => { - const id = await brain.add({ type: NounType.Concept, subtype: 'general', data: 'y' }) - await expect( - brain.update({ id, metadata: { confidence: 0.3 } as object }) - ).rejects.toThrow(/metadata\.confidence is a reserved field/) - }) - - it('relate() throws on a reserved key in the bag', async () => { - const a = await brain.add({ type: NounType.Person, subtype: 'employee', data: 'A' }) - const b = await brain.add({ type: NounType.Person, subtype: 'employee', data: 'B' }) - await expect( - brain.relate({ - from: a, - to: b, - type: VerbType.RelatedTo, - subtype: 'colleague', - metadata: { confidence: 0.4 } as object - }) - ).rejects.toThrow(/metadata\.confidence is a reserved field.*RESERVED_RELATION_FIELDS/s) - }) - - it('updateRelation() throws on a reserved key in the patch', async () => { - const a = await brain.add({ type: NounType.Person, subtype: 'employee', data: 'A' }) - const b = await brain.add({ type: NounType.Person, subtype: 'employee', data: 'B' }) - const relId = await brain.relate({ - from: a, - to: b, - type: VerbType.ReportsTo, - subtype: 'direct' - }) - await expect( - brain.updateRelation({ id: relId, metadata: { weight: 0.2 } as object }) - ).rejects.toThrow(/metadata\.weight is a reserved field/) - }) - - it('transact() add op throws on a reserved key in the bag', async () => { - await expect( - brain.transact([ - { - op: 'add', - type: NounType.Concept, - subtype: 'general', - data: 'tx', - metadata: { confidence: 0.7 } as object - } - ]) - ).rejects.toThrow(/metadata\.confidence is a reserved field/) - }) - - it('a custom (non-reserved) key in the bag does NOT throw', async () => { - const id = await brain.add({ - type: NounType.Concept, - subtype: 'general', - data: 'ok', - metadata: { status: 'draft', rating: 4 } - }) - const entity = await brain.get(id) - expect(entity?.metadata).toEqual({ status: 'draft', rating: 4 }) - }) - }) - - describe("'remap' policy remaps silently (no warning)", () => { - let brain: Brainy - let warnSpy: ReturnType - - beforeEach(async () => { - warnSpy = vi.spyOn(prodLog, 'warn').mockImplementation(() => {}) - brain = new Brainy(createTestConfig({ reservedFieldPolicy: 'remap' })) - await brain.init() - }) - - afterEach(async () => { - await brain.close() - warnSpy.mockRestore() - }) - - it('lifts user-mutable reserved fields to top-level without warning', async () => { - const id = await brain.add({ - type: NounType.Person, - data: 'remap lift', - metadata: { confidence: 0.8, weight: 0.6, subtype: 'employee', dept: 'eng' } as object - }) - const entity = await brain.get(id) - expect(entity?.confidence).toBe(0.8) - expect(entity?.weight).toBe(0.6) - expect(entity?.subtype).toBe('employee') - expect(entity?.metadata).toEqual({ dept: 'eng' }) - // 'remap' is silent about reserved fields (unrelated storage logs may fire, - // so assert specifically that no reserved-field warning was emitted). - const reservedWarned = warnSpy.mock.calls.some((c) => - String(c[0]).includes('reserved field') - ) - expect(reservedWarned).toBe(false) - }) - - it('preserves _originalId on natural-key ids through the remap path', async () => { - // A speculative view applies the same normalization and maps a natural-key - // id to a stable UUID, preserving the caller's original string. - const base = await brain.now() - const speculative = await base.with([ - { - op: 'add', - id: 'remap-spec-entity', - type: NounType.Concept, - subtype: 'general', - data: 'spec', - metadata: { confidence: 0.65, custom: 'spec' } as object - } - ]) - const entity = await speculative.get('remap-spec-entity') - expect(entity?.confidence).toBe(0.65) - expect(entity?.metadata).toEqual({ custom: 'spec', _originalId: 'remap-spec-entity' }) - await speculative.release() - await base.release() - }) - }) - - describe("'warn' policy remaps AND warns once per key", () => { - let brain: Brainy - let warnSpy: ReturnType - - beforeEach(async () => { - warnSpy = vi.spyOn(prodLog, 'warn').mockImplementation(() => {}) - brain = new Brainy(createTestConfig({ reservedFieldPolicy: 'warn' })) - await brain.init() - }) - - afterEach(async () => { - await brain.close() - warnSpy.mockRestore() - }) - - it('remaps the value (same as remap) and emits a warning naming the field', async () => { - // Use a method+field combo unique to this test so the per-process one-shot - // registry has not already consumed it. - const id = await brain.add({ - type: NounType.Person, - data: 'warn lift', - // weight is user-mutable → remapped; this is the only 'warn'-policy - // add({ weight }) in the suite, so the one-shot warning fires here. - metadata: { weight: 0.42, dept: 'eng' } as object - }) - const entity = await brain.get(id) - // Value is honored (remap still happens under 'warn'). - expect(entity?.weight).toBe(0.42) - expect(entity?.metadata).toEqual({ dept: 'eng' }) - // And a warning was emitted naming the reserved field. - expect(warnSpy).toHaveBeenCalled() - const warned = warnSpy.mock.calls.some((c) => - String(c[0]).includes("'weight'") - ) - expect(warned).toBe(true) - }) - - it('warns for system-managed keys too (closes the historical gap)', async () => { - // Pre-8.0 only system-managed fields warned; 'warn' warns for every key. - // 'createdBy' (system-managed on update) is unique to this test. - const id = await brain.add({ type: NounType.Concept, subtype: 'general', data: 'sys' }) - warnSpy.mockClear() - await brain.update({ id, metadata: { createdBy: 'nope', keep: 'me' } as object }) - const entity = await brain.get(id) - // System-managed key dropped; custom field merged. - expect((entity?.metadata as Record)?.createdBy).toBeUndefined() - expect((entity?.metadata as Record)?.keep).toBe('me') - // A warning was emitted for the dropped system-managed key. - const warned = warnSpy.mock.calls.some((c) => - String(c[0]).includes("'createdBy'") - ) - expect(warned).toBe(true) - }) - }) -}) diff --git a/tests/unit/brainy/update-reserved-metadata-remap.test.ts b/tests/unit/brainy/update-reserved-metadata-remap.test.ts deleted file mode 100644 index 31713f99..00000000 --- a/tests/unit/brainy/update-reserved-metadata-remap.test.ts +++ /dev/null @@ -1,403 +0,0 @@ -/** - * @module tests/unit/brainy/update-reserved-metadata-remap - * @description Regression tests for the reserved-field metadata-bag trap, - * ported from the 7.x fix and extended to the full 8.0 contract. - * - * History: `add({metadata: {confidence}})` lifted reserved fields to their - * canonical top-level location, but `update({metadata: {confidence}})` - * silently dropped the same shape — the patch value survived the merge and - * was then clobbered by the preserve-existing spread. A production - * consumer's confidence-evolution writes no-oped for weeks before being - * caught by reading values back. - * - * These tests pin the LEGACY REMAP behavior, which in 8.0 is opt-in via - * `reservedFieldPolicy: 'remap'` (the default is `'throw'` — see the policy - * matrix in tests/unit/brainy/reserved-field-policy.test.ts). The brain in - * every test below is constructed with `reservedFieldPolicy: 'remap'` so these - * deep correctness assertions about the remap path stay exercised. - * - * Remap contract under test (every write path, entities AND relationships): - * - user-mutable reserved fields (`confidence`, `weight`, `subtype` — plus - * `service`/`createdBy` at add()/relate() time) remap from the metadata - * bag to their dedicated top-level param, with top-level winning when both - * are present; - * - system-managed reserved fields (`createdAt`, `_rev`, `noun`/`verb`, - * `data`, …) are dropped from the bag; - * - the same normalization applies to `transact()` operations and `with()` - * speculative views; - * - reads NEVER echo a reserved field inside `metadata`. - * - * TypeScript callers can't write these shapes at all (compile-time guard on - * the metadata param types — see tests/unit/types/reserved-metadata-keys.test-d.ts); - * these tests simulate untyped (JavaScript) callers, hence the `as object` - * widenings on the metadata literals. - */ - -import { describe, it, expect, beforeEach, afterEach } from 'vitest' -import { Brainy } from '../../../src/index.js' -import { NounType, VerbType } from '../../../src/types/graphTypes.js' -import { createTestConfig } from '../../helpers/test-factory.js' - -describe('reserved-field metadata remap (8.0 legacy remap path)', () => { - let brain: Brainy - - beforeEach(async () => { - // The remap path is opt-in in 8.0 (default policy is 'throw'). - brain = new Brainy(createTestConfig({ reservedFieldPolicy: 'remap' })) - await brain.init() - }) - - afterEach(async () => { - await brain.close() - }) - - describe('update() — the ported 7.x regression', () => { - it('remaps metadata.confidence to the top-level field (the production repro)', async () => { - const id = await brain.add({ - type: NounType.Concept, - subtype: 'general', - data: 'x', - metadata: { confidence: 0.8 } as object - }) - - // Top-level write works (always did) - await brain.update({ id, confidence: 0.42 }) - let entity = await brain.get(id) - expect(entity?.confidence).toBe(0.42) - - // Metadata-patch write — silently dropped pre-fix, remapped now - await brain.update({ id, metadata: { confidence: 0.33 } as object }) - entity = await brain.get(id) - expect(entity?.confidence).toBe(0.33) - // The reserved key must not linger inside the metadata bag - expect((entity?.metadata as Record)?.confidence).toBeUndefined() - }) - - it('remaps metadata.weight and metadata.subtype the same way', async () => { - const id = await brain.add({ - type: NounType.Concept, - subtype: 'general', - data: 'y', - metadata: {} - }) - - await brain.update({ id, metadata: { weight: 0.7, subtype: 'specialized' } as object }) - const entity = await brain.get(id) - expect(entity?.weight).toBe(0.7) - expect(entity?.subtype).toBe('specialized') - expect((entity?.metadata as Record)?.weight).toBeUndefined() - expect((entity?.metadata as Record)?.subtype).toBeUndefined() - }) - - it('top-level param wins when both top-level and metadata-patch carry the field', async () => { - const id = await brain.add({ - type: NounType.Concept, - subtype: 'general', - data: 'z', - metadata: { confidence: 0.5 } as object - }) - - await brain.update({ id, confidence: 0.9, metadata: { confidence: 0.1 } as object }) - const entity = await brain.get(id) - expect(entity?.confidence).toBe(0.9) - }) - - it('drops system-managed fields from patches without corrupting the entity', async () => { - const id = await brain.add({ - type: NounType.Concept, - subtype: 'general', - data: 'w', - metadata: { keep: 'me' } - }) - const before = await brain.get(id) - - await brain.update({ - id, - metadata: { createdAt: 1, _rev: 999, noun: 'organization', other: 'applied' } as object - }) - const after = await brain.get(id) - - expect(after?.createdAt).toBe(before?.createdAt) // immutable - expect(after?.type).toBe('concept') // noun patch ignored - expect(after?._rev).toBe((before?._rev ?? 1) + 1) // _rev patch ignored; normal bump applied - expect((after?.metadata as Record)?.other).toBe('applied') // custom fields still merge - expect((after?.metadata as Record)?.keep).toBe('me') - expect((after?.metadata as Record)?._rev).toBeUndefined() - expect((after?.metadata as Record)?.createdAt).toBeUndefined() - expect((after?.metadata as Record)?.noun).toBeUndefined() - }) - - it('custom (non-reserved) metadata patches are unaffected by the remap', async () => { - const id = await brain.add({ - type: NounType.Concept, - subtype: 'general', - data: 'v', - metadata: { status: 'draft' } - }) - - await brain.update({ id, metadata: { status: 'reviewed', rating: 4.5 } }) - const entity = await brain.get(id) - expect((entity?.metadata as Record)?.status).toBe('reviewed') - expect((entity?.metadata as Record)?.rating).toBe(4.5) - }) - }) - - describe('add() — explicit lift, identical contract', () => { - it('lifts confidence/weight/subtype out of the bag to top level', async () => { - const id = await brain.add({ - type: NounType.Person, - data: 'lift check', - metadata: { confidence: 0.8, weight: 0.6, subtype: 'employee', dept: 'eng' } as object - }) - - const entity = await brain.get(id) - expect(entity?.confidence).toBe(0.8) - expect(entity?.weight).toBe(0.6) - expect(entity?.subtype).toBe('employee') - expect(entity?.metadata).toEqual({ dept: 'eng' }) - }) - - it('lifts service (settable at add time) and lets the top-level param win', async () => { - const lifted = await brain.add({ - type: NounType.Person, - subtype: 'employee', - data: 'service lift', - metadata: { service: 'orders' } as object - }) - expect((await brain.get(lifted))?.service).toBe('orders') - - const topLevelWins = await brain.add({ - type: NounType.Person, - subtype: 'employee', - data: 'service precedence', - service: 'billing', - metadata: { service: 'orders' } as object - }) - const entity = await brain.get(topLevelWins) - expect(entity?.service).toBe('billing') - expect((entity?.metadata as Record)?.service).toBeUndefined() - }) - - it('a remapped subtype satisfies subtype enforcement like a top-level one', async () => { - brain.requireSubtype(NounType.Document) - - // Top-level missing, but the bag carries it — must not throw. - const id = await brain.add({ - type: NounType.Document, - data: 'enforcement via remap', - metadata: { subtype: 'invoice' } as object - }) - expect((await brain.get(id))?.subtype).toBe('invoice') - - // Neither place carries it — must throw. - await expect( - brain.add({ type: NounType.Document, data: 'no subtype anywhere' }) - ).rejects.toThrow(/subtype/) - }) - }) - - describe('transact() — same remap on add and update ops', () => { - it('normalizes reserved fields in transact add + update ops', async () => { - const db1 = await brain.transact([ - { - op: 'add', - type: NounType.Concept, - subtype: 'general', - data: 'tx', - metadata: { confidence: 0.7, custom: 'a' } as object - } - ]) - const id = db1.receipt!.ids[0] - - let entity = await brain.get(id) - expect(entity?.confidence).toBe(0.7) - expect(entity?.metadata).toEqual({ custom: 'a' }) - - await brain.transact([ - { op: 'update', id, metadata: { confidence: 0.25, custom: 'b' } as object } - ]) - entity = await brain.get(id) - expect(entity?.confidence).toBe(0.25) - expect(entity?.metadata).toEqual({ custom: 'b' }) - expect((entity?.metadata as Record)?.confidence).toBeUndefined() - }) - - it('historical asOf() reads surface reserved fields ONLY top-level', async () => { - const db1 = await brain.transact([ - { - op: 'add', - type: NounType.Concept, - subtype: 'general', - data: 'historical', - metadata: { confidence: 0.9, custom: 'past' } as object - } - ]) - const id = db1.receipt!.ids[0] - - // Move the world forward so generation db1 is historical. - await brain.transact([{ op: 'update', id, confidence: 0.1, metadata: { custom: 'now' } }]) - - const past = await brain.asOf(db1.generation) - const historical = await past.get(id) - expect(historical?.confidence).toBe(0.9) - expect(historical?.metadata).toEqual({ custom: 'past' }) - await past.release() - }) - - it('with() speculative views apply the same normalization', async () => { - const base = await brain.now() - const speculative = await base.with([ - { - op: 'add', - id: 'spec-entity', - type: NounType.Concept, - subtype: 'general', - data: 'spec', - metadata: { confidence: 0.65, custom: 'spec' } as object - } - ]) - - const entity = await speculative.get('spec-entity') - expect(entity?.confidence).toBe(0.65) - // 8.0 id normalization: a natural-key id is mapped to a stable UUID and - // the caller's original string is preserved under _originalId — surfaced - // here exactly as the durable transact()/add() paths do. - expect(entity?.metadata).toEqual({ custom: 'spec', _originalId: 'spec-entity' }) - await speculative.release() - await base.release() - }) - }) - - describe('read paths never echo reserved fields inside metadata', () => { - it('find() (storage pagination path) returns custom-only metadata with reserved fields top-level', async () => { - const id = await brain.add({ - type: NounType.Person, - subtype: 'employee', - data: 'pagination echo check', - confidence: 0.8, - weight: 0.6, - metadata: { dept: 'eng' } - }) - - // No query/filter → served by the direct storage pagination path - // (getNounsWithPagination), which historically echoed the full flat - // record (noun/subtype/createdAt/… inside metadata). - const results = await brain.find({ limit: 50 }) - const result = results.find((r) => r.id === id) - expect(result).toBeDefined() - expect(result?.entity.metadata).toEqual({ dept: 'eng' }) - expect(result?.entity.type).toBe(NounType.Person) - expect(result?.entity.subtype).toBe('employee') - expect(result?.entity.confidence).toBe(0.8) - expect(result?.entity.weight).toBe(0.6) - expect(typeof result?.entity.createdAt).toBe('number') - expect(result?.entity._rev).toBe(1) - }) - - it('related() by target surfaces reserved fields top-level, custom-only metadata', async () => { - const a = await brain.add({ type: NounType.Person, subtype: 'employee', data: 'src' }) - const b = await brain.add({ type: NounType.Person, subtype: 'employee', data: 'tgt' }) - const relId = await brain.relate({ - from: a, - to: b, - type: VerbType.ReportsTo, - subtype: 'direct', - confidence: 0.9, - weight: 0.5, - service: 'orders', - metadata: { note: 'target path' } - }) - - const relations = await brain.related({ to: b }) - const rel = relations.find((r) => r.id === relId) - expect(rel).toBeDefined() - expect(rel?.metadata).toEqual({ note: 'target path' }) - expect(rel?.subtype).toBe('direct') - expect(rel?.confidence).toBe(0.9) - expect(rel?.weight).toBe(0.5) - expect(rel?.service).toBe('orders') - expect(typeof rel?.createdAt).toBe('number') - }) - }) - - describe('relationships — relate() / updateRelation() mirror', () => { - let a: string - let b: string - - beforeEach(async () => { - a = await brain.add({ type: NounType.Person, subtype: 'employee', data: 'A' }) - b = await brain.add({ type: NounType.Person, subtype: 'employee', data: 'B' }) - }) - - it('relate() persists the top-level confidence and service params', async () => { - const relId = await brain.relate({ - from: a, - to: b, - type: VerbType.ReportsTo, - subtype: 'direct', - confidence: 0.77, - service: 'orders' - }) - - const relations = await brain.related({ from: a }) - const rel = relations.find((r) => r.id === relId) - expect(rel?.confidence).toBe(0.77) - expect(rel?.service).toBe('orders') - }) - - it('relate() remaps reserved fields out of the metadata bag', async () => { - const relId = await brain.relate({ - from: a, - to: b, - type: VerbType.RelatedTo, - subtype: 'colleague', - metadata: { confidence: 0.4, weight: 0.3, role: 'peer' } as object - }) - - const relations = await brain.related({ from: a }) - const rel = relations.find((r) => r.id === relId) - expect(rel?.confidence).toBe(0.4) - expect(rel?.weight).toBe(0.3) - expect(rel?.metadata).toEqual({ role: 'peer' }) - }) - - it('relation.metadata never echoes the verb type key', async () => { - const relId = await brain.relate({ - from: a, - to: b, - type: VerbType.RelatedTo, - subtype: 'colleague', - metadata: { note: 'no echo' } - }) - - const relations = await brain.related({ from: a }) - const rel = relations.find((r) => r.id === relId) - expect(rel?.type).toBe(VerbType.RelatedTo) - expect((rel?.metadata as Record)?.verb).toBeUndefined() - expect(rel?.metadata).toEqual({ note: 'no echo' }) - }) - - it('updateRelation() remaps the user-mutable trio and preserves service', async () => { - const relId = await brain.relate({ - from: a, - to: b, - type: VerbType.ReportsTo, - subtype: 'direct', - service: 'orders', - metadata: { keep: 'me' } - }) - - await brain.updateRelation({ - id: relId, - metadata: { confidence: 0.55, subtype: 'dotted-line', extra: 'applied' } as object - }) - - const relations = await brain.related({ from: a }) - const rel = relations.find((r) => r.id === relId) - expect(rel?.confidence).toBe(0.55) - expect(rel?.subtype).toBe('dotted-line') - expect(rel?.service).toBe('orders') // fixed at relate() time, never erased by updates - expect(rel?.metadata).toEqual({ keep: 'me', extra: 'applied' }) - }) - }) -}) diff --git a/tests/unit/brainy/visibility.test.ts b/tests/unit/brainy/visibility.test.ts index a5a02422..dd4540d7 100644 --- a/tests/unit/brainy/visibility.test.ts +++ b/tests/unit/brainy/visibility.test.ts @@ -198,60 +198,47 @@ describe('visibility (8.0 reserved field)', () => { expect(entity?.visibility).toBeUndefined() }) - it('an untyped caller passing visibility inside metadata is normalized under reservedFieldPolicy:"remap" (lifted to top-level)', async () => { - // Simulate a JavaScript caller smuggling the reserved key past the compile-time guard. - // The legacy remap behavior is now opt-in (8.0 default is 'throw'). - const remapBrain = new Brainy(createTestConfig({ reservedFieldPolicy: 'remap' })) - await remapBrain.init() - try { - const id = await remapBrain.add({ - type: NounType.Concept, - data: 'y', - metadata: { visibility: 'internal', tag: 't' } as object - }) - const entity = await remapBrain.get(id) - // Lifted to the top-level field… - expect(entity?.visibility).toBe('internal') - // …and stripped from the metadata bag. - expect((entity?.metadata as Record)?.visibility).toBeUndefined() - expect((entity?.metadata as Record)?.tag).toBe('t') - // It is excluded from the default count, exactly like a top-level internal write. - expect(await remapBrain.getNounCount()).toBe(0) - } finally { - await remapBrain.close() - } + it('metadata.visibility is the USER’s field (field-addressing law) — stored verbatim, never lifted to the engine tier', async () => { + const id = await brain.add({ + type: NounType.Concept, + data: 'y', + metadata: { visibility: 'internal', tag: 't' } as object + }) + const entity = await brain.get(id) + // The user's field lives in the bag, verbatim… + expect((entity?.metadata as Record)?.visibility).toBe('internal') + expect((entity?.metadata as Record)?.tag).toBe('t') + // …and the ENGINE tier is untouched: absent === public, so the entity + // stays visible on default reads (the engine tier is set only via the + // dedicated visibility param and reads at system.visibility). + expect(entity?.visibility).toBeUndefined() + const visible = await brain.find({ type: NounType.Concept, limit: 20 }) + expect(visible.map((r) => r.id)).toContain(id) }) - it('a "system" value smuggled through metadata is dropped under reservedFieldPolicy:"remap", not honored', async () => { - // 'system' is Brainy-only; an untyped caller must not be able to set it. - const remapBrain = new Brainy(createTestConfig({ reservedFieldPolicy: 'remap' })) - await remapBrain.init() - try { - const id = await remapBrain.add({ - type: NounType.Concept, - data: 'z', - metadata: { visibility: 'system' } as object - }) - const entity = await remapBrain.get(id) - // The smuggled 'system' was dropped → entity stays public (counted, visible). - expect(entity?.visibility).toBeUndefined() - expect(await remapBrain.getNounCount()).toBe(1) - const found = await remapBrain.find({ type: NounType.Concept, limit: 10 }) - expect(found.map((r) => r.id)).toContain(id) - } finally { - await remapBrain.close() - } + it('a user field valued "system" cannot smuggle the Brainy-only tier — it is just user data', async () => { + const id = await brain.add({ + type: NounType.Concept, + data: 'z', + metadata: { visibility: 'system' } as object + }) + const entity = await brain.get(id) + // Engine tier unaffected → entity stays public (counted, visible); + // the string 'system' is ordinary user data in the bag. + expect(entity?.visibility).toBeUndefined() + expect((entity?.metadata as Record)?.visibility).toBe('system') + const found = await brain.find({ type: NounType.Concept, limit: 10 }) + expect(found.map((r) => r.id)).toContain(id) }) - it('an untyped caller passing visibility inside metadata throws under the default policy', async () => { - // 8.0 default: no silent remap — a reserved key in the bag is a loud error. + it('a forged system.visibility key in metadata refuses loudly at the write door', async () => { await expect( brain.add({ type: NounType.Concept, data: 'throws', - metadata: { visibility: 'internal', tag: 't' } as object + metadata: { 'system.visibility': 'internal' } as object }) - ).rejects.toThrow(/visibility.*reserved field/) + ).rejects.toThrow(/system\./) }) }) }) diff --git a/tests/unit/db/whereMatcher.test.ts b/tests/unit/db/whereMatcher.test.ts index 2223117c..6c0252d6 100644 --- a/tests/unit/db/whereMatcher.test.ts +++ b/tests/unit/db/whereMatcher.test.ts @@ -32,7 +32,7 @@ function entity(overrides: Partial = {}): Entity { } describe('db/whereMatcher — resolveEntityField', () => { - it('resolves standard top-level fields', () => { + it('system. resolves the entity scalar; bare/metadata. reads the metadata bag only (sealed 2026-08-03)', () => { const e = entity({ subtype: 'invoice', service: 'billing', @@ -41,17 +41,32 @@ describe('db/whereMatcher — resolveEntityField', () => { _rev: 3, data: 'payload' }) - expect(resolveEntityField(e, 'id')).toBe('e-1') - expect(resolveEntityField(e, 'type')).toBe(NounType.Document) - expect(resolveEntityField(e, 'noun')).toBe(NounType.Document) // alias - expect(resolveEntityField(e, 'subtype')).toBe('invoice') - expect(resolveEntityField(e, 'service')).toBe('billing') - expect(resolveEntityField(e, 'confidence')).toBe(0.9) - expect(resolveEntityField(e, 'weight')).toBe(0.5) - expect(resolveEntityField(e, '_rev')).toBe(3) - expect(resolveEntityField(e, 'createdAt')).toBe(1000) - expect(resolveEntityField(e, 'updatedAt')).toBe(2000) - expect(resolveEntityField(e, 'data')).toBe('payload') + + // system. is the ONLY spelling that reaches an entity scalar. + expect(resolveEntityField(e, 'system.id')).toBe('e-1') + expect(resolveEntityField(e, 'system.type')).toBe(NounType.Document) + expect(resolveEntityField(e, 'system.subtype')).toBe('invoice') + expect(resolveEntityField(e, 'system.service')).toBe('billing') + expect(resolveEntityField(e, 'system.confidence')).toBe(0.9) + expect(resolveEntityField(e, 'system.weight')).toBe(0.5) + expect(resolveEntityField(e, 'system.createdAt')).toBe(1000) + expect(resolveEntityField(e, 'system.updatedAt')).toBe(2000) + + // Plumbing (_rev, data) is invisible even via system. — not in the + // ten-scalar map, so this internal resolver reads it as absent (the typed + // refusal for these lives one layer up, at the query-surface parser). + expect(resolveEntityField(e, 'system._rev')).toBeUndefined() + expect(resolveEntityField(e, 'system.data')).toBeUndefined() + + // Bare names are ALWAYS the user's metadata field — even when they share + // a spelling with an engine scalar, or with the now-dead 'noun' alias. + // This entity's metadata bag is empty, so every bare name below reads + // absent rather than silently falling back to the entity scalar. + expect(resolveEntityField(e, 'id')).toBeUndefined() + expect(resolveEntityField(e, 'type')).toBeUndefined() + expect(resolveEntityField(e, 'noun')).toBeUndefined() // legacy alias is dead + expect(resolveEntityField(e, 'subtype')).toBeUndefined() + expect(resolveEntityField(e, 'createdAt')).toBeUndefined() }) it('resolves custom fields from the metadata bag', () => { diff --git a/tests/unit/test-suite-coverage-guard.test.ts b/tests/unit/test-suite-coverage-guard.test.ts index 93db4421..21f918f1 100644 --- a/tests/unit/test-suite-coverage-guard.test.ts +++ b/tests/unit/test-suite-coverage-guard.test.ts @@ -30,6 +30,9 @@ function allTestFiles(dir: string, out: string[] = []): string[] { * conscious decision — a NEW orphan not listed here fails the guard below. */ const MANUAL_ONLY = new Set([ + // Conformance suites run as an explicit gate stage (both engines run them + // by direct invocation), never swept into the unit/integration configs. + 'tests/conformance/collider-fidelity.test.ts', 'tests/api/performance-benchmarks.test.ts', 'tests/critical-neural-validation.test.ts', 'tests/critical-performance-benchmark.test.ts', @@ -38,7 +41,15 @@ const MANUAL_ONLY = new Set([ 'tests/package-size-limit.test.ts', 'tests/performance/graph-scale-performance.test.ts', 'tests/performance/triple-intelligence-scale.test.ts', - 'tests/performance/typeAware.bench.test.ts' + 'tests/performance/typeAware.bench.test.ts', + // Cross-engine field-addressing conformance suite: pinned bit-for-bit against + // the native accelerator's implementation of the SAME contract, and invoked + // directly (`npx vitest run tests/conformance/namespace-law.test.ts`), never + // swept into the unit/integration gates — a run against a branch where the + // resolver hasn't landed yet must SKIP loudly (see the file's own SELF-SKIP + // doc), not silently pass/fail as a side effect of which gate happened to + // pick it up. + 'tests/conformance/namespace-law.test.ts' ]) function inGate(rel: string): boolean { diff --git a/tests/unit/types/nestedBagRecord.test.ts b/tests/unit/types/nestedBagRecord.test.ts new file mode 100644 index 00000000..b8e9be46 --- /dev/null +++ b/tests/unit/types/nestedBagRecord.test.ts @@ -0,0 +1,127 @@ +/** + * @module tests/unit/types/nestedBagRecord + * @description Unit pins for the v2 (nested-bag) stored-record layer — the + * storage half of the field-addressing law. The write door accepts ANY user + * metadata name; what makes that lossless on disk is the record shape: + * engine fields top-level, the user bag NESTED verbatim, discriminated by + * the engine-written format stamp (never by names — names are the user's). + * These pins hold the builders, the discriminator, and the shape-aware + * split that every read path (live, batch, historical) routes through. + */ +import { describe, it, expect } from 'vitest' +import { + buildNounMetadataRecord, + buildVerbMetadataRecord, + splitNounMetadataRecord, + splitVerbMetadataRecord, + isNestedBagRecord, + METADATA_RECORD_FORMAT_KEY, + NESTED_BAG_FORMAT +} from '../../../src/types/reservedFields.js' + +const COLLIDER_BAG = { + confidence: 'user-confidence', + weight: 'user-weight', + subtype: 'user-subtype', + createdAt: 'user-createdAt', + service: 'user-service', + data: 'user-data', + noun: 'user-noun', + _rev: 'user-rev', + level: 7, + plain: 'control' +} + +describe('v2 nested-bag stored records — build / discriminate / split', () => { + it('build → split round-trips a fully colliding user bag VERBATIM', () => { + const record = buildNounMetadataRecord( + { noun: 'document', confidence: 0.25, createdAt: 111, updatedAt: 222, _rev: 1 }, + { ...COLLIDER_BAG } + ) + expect(isNestedBagRecord(record)).toBe(true) + expect(record[METADATA_RECORD_FORMAT_KEY]).toBe(NESTED_BAG_FORMAT) + + const { reserved, custom } = splitNounMetadataRecord(record) + // The engine half is exactly what the engine wrote… + expect(reserved.noun).toBe('document') + expect(reserved.confidence).toBe(0.25) + expect(reserved._rev).toBe(1) + // …and the user bag comes back byte-for-byte, colliders included. + expect(custom).toEqual(COLLIDER_BAG) + }) + + it('the verb mirror round-trips an edge collider bag verbatim', () => { + const record = buildVerbMetadataRecord( + { verb: 'relatedTo', weight: 1.0, confidence: 0.5, createdAt: 333 }, + { verb: 'user-verb', confidence: 'user-c', tag: 't' } + ) + expect(isNestedBagRecord(record)).toBe(true) + const { reserved, custom } = splitVerbMetadataRecord(record) + expect(reserved.verb).toBe('relatedTo') + expect(reserved.confidence).toBe(0.5) + expect(custom).toEqual({ verb: 'user-verb', confidence: 'user-c', tag: 't' }) + }) + + it('a LEGACY flat record (no stamp) splits BY NAME — sound because the pre-law door refused colliders', () => { + const legacy = { + noun: 'document', + confidence: 0.75, + createdAt: 111, + _rev: 2, + legacyField: 'legacy-value' + } + expect(isNestedBagRecord(legacy)).toBe(false) + const { reserved, custom } = splitNounMetadataRecord(legacy) + expect(reserved.confidence).toBe(0.75) + expect(reserved._rev).toBe(2) + expect(custom).toEqual({ legacyField: 'legacy-value' }) + }) + + it('the stamp is the discriminator, never the name: a legacy user OBJECT field named `metadata` does not fake a v2 record', () => { + // Pre-law, 'metadata' was never a reserved name — a flat record could + // legally carry a user object field spelled exactly 'metadata'. Without + // the engine-written stamp it must split as legacy, with that object + // preserved as an ordinary user field. + const legacyWithMetadataField = { + noun: 'document', + confidence: 0.5, + metadata: { nested: 'user-object' } + } + expect(isNestedBagRecord(legacyWithMetadataField)).toBe(false) + const { reserved, custom } = splitNounMetadataRecord(legacyWithMetadataField) + expect(reserved.confidence).toBe(0.5) + expect(custom).toEqual({ metadata: { nested: 'user-object' } }) + }) + + it('a malformed stamp (right key, wrong value / non-object bag) never discriminates as v2', () => { + expect( + isNestedBagRecord({ [METADATA_RECORD_FORMAT_KEY]: 999, metadata: {} }) + ).toBe(false) + expect( + isNestedBagRecord({ [METADATA_RECORD_FORMAT_KEY]: NESTED_BAG_FORMAT, metadata: 'not-a-bag' }) + ).toBe(false) + expect( + isNestedBagRecord({ [METADATA_RECORD_FORMAT_KEY]: NESTED_BAG_FORMAT, metadata: [1, 2] }) + ).toBe(false) + expect(isNestedBagRecord(null)).toBe(false) + expect(isNestedBagRecord(undefined)).toBe(false) + }) + + it('the v2 split never surfaces the stamp or the bag container as fields', () => { + const record = buildNounMetadataRecord({ noun: 'document', _rev: 1 }, { a: 1 }) + const { reserved, custom } = splitNounMetadataRecord(record) + expect(METADATA_RECORD_FORMAT_KEY in reserved).toBe(false) + expect(METADATA_RECORD_FORMAT_KEY in custom).toBe(false) + expect('metadata' in reserved).toBe(false) + expect(custom).toEqual({ a: 1 }) + }) + + it('builders copy the bag (no aliasing): later caller mutation cannot reach the record', () => { + const bag: Record = { a: 1 } + const record = buildNounMetadataRecord({ noun: 'document' }, bag) + bag.a = 999 + bag.b = 'sneaky' + expect((record.metadata as Record).a).toBe(1) + expect('b' in (record.metadata as Record)).toBe(false) + }) +}) diff --git a/tests/unit/types/reserved-metadata-keys.test-d.ts b/tests/unit/types/reserved-metadata-keys.test-d.ts deleted file mode 100644 index 37fceefa..00000000 --- a/tests/unit/types/reserved-metadata-keys.test-d.ts +++ /dev/null @@ -1,265 +0,0 @@ -/** - * @module tests/unit/types/reserved-metadata-keys.test-d - * @description Compile-time tests for the reserved-field contract (layer 1 of - * three — see src/types/reservedFields.ts): a literal reserved key inside any - * `metadata` param is a TypeScript error, while the generic `T` ergonomics - * stay intact (typed bags, untyped brains, index-signature shapes, and the - * documented exemption for consumers who explicitly declare a reserved key in - * their own metadata type). - * - * Runs under vitest typecheck mode (`test.typecheck` in - * tests/configs/vitest.unit.config.ts) — these assertions are validated by - * `tsc`, never executed. The runtime half of the contract (the write-path - * remap for untyped callers) is pinned by - * tests/unit/brainy/update-reserved-metadata-remap.test.ts. - */ - -import { describe, it, assertType } from 'vitest' -import type { - AddParams, - UpdateParams, - RelateParams, - UpdateRelationParams, - TxOperation -} from '../../../src/index.js' -import { NounType, VerbType } from '../../../src/types/graphTypes.js' - -describe('reserved entity keys in metadata are compile errors', () => { - it('AddParams (untyped brain) rejects every reserved key but stays open for custom fields', () => { - // Custom fields of any shape remain legal — exactly the pre-8.0 latitude. - assertType({ - type: NounType.Person, - subtype: 'employee', - data: 'x', - metadata: { dept: 'eng', level: 3, tags: ['a', 'b'], nested: { ok: true } } - }) - - assertType({ - type: NounType.Person, - subtype: 'employee', - data: 'x', - // @ts-expect-error — 'noun' is reserved (the entity type travels via the top-level 'type' param) - metadata: { noun: 'organization' } - }) - assertType({ - type: NounType.Person, - subtype: 'employee', - data: 'x', - // @ts-expect-error — 'subtype' is reserved (use the top-level 'subtype' param) - metadata: { subtype: 'contractor' } - }) - assertType({ - type: NounType.Person, - subtype: 'employee', - data: 'x', - // @ts-expect-error — 'createdAt' is reserved (system-managed) - metadata: { createdAt: Date.now() } - }) - assertType({ - type: NounType.Person, - subtype: 'employee', - data: 'x', - // @ts-expect-error — 'updatedAt' is reserved (system-managed) - metadata: { updatedAt: Date.now() } - }) - assertType({ - type: NounType.Person, - subtype: 'employee', - data: 'x', - // @ts-expect-error — 'confidence' is reserved (use the top-level 'confidence' param) - metadata: { confidence: 0.8 } - }) - assertType({ - type: NounType.Person, - subtype: 'employee', - data: 'x', - // @ts-expect-error — 'weight' is reserved (use the top-level 'weight' param) - metadata: { weight: 0.5 } - }) - assertType({ - type: NounType.Person, - subtype: 'employee', - data: 'x', - // @ts-expect-error — 'service' is reserved (use the top-level 'service' param) - metadata: { service: 'orders' } - }) - assertType({ - type: NounType.Person, - subtype: 'employee', - data: 'x', - // @ts-expect-error — 'data' is reserved (use the top-level 'data' param) - metadata: { data: 'content' } - }) - assertType({ - type: NounType.Person, - subtype: 'employee', - data: 'x', - // @ts-expect-error — 'createdBy' is reserved (use the top-level 'createdBy' param) - metadata: { createdBy: { augmentation: 'importer', version: '1.0' } } - }) - assertType({ - type: NounType.Person, - subtype: 'employee', - data: 'x', - // @ts-expect-error — '_rev' is reserved (system-managed revision counter) - metadata: { _rev: 7 } - }) - }) - - it('AddParams (typed brain) rejects reserved keys alongside the declared shape', () => { - interface EmployeeMeta { - dept: string - level: number - } - - assertType>({ - type: NounType.Person, - subtype: 'employee', - data: 'x', - metadata: { dept: 'eng', level: 3 } - }) - - assertType>({ - type: NounType.Person, - subtype: 'employee', - data: 'x', - // @ts-expect-error — 'confidence' is reserved even when T declares other fields - metadata: { dept: 'eng', level: 3, confidence: 0.8 } - }) - }) - - it('documented exemptions: T-declared reserved keys and index-signature shapes stay assignable', () => { - // A consumer who *explicitly* types a reserved key into their metadata - // shape keeps a working (if unwise) type — the guard exempts keyof T. - interface LegacyMeta { - confidence: number - note: string - } - assertType>({ - type: NounType.Person, - subtype: 'employee', - data: 'x', - metadata: { confidence: 0.8, note: 'declared by the consumer type' } - }) - - // Index-signature metadata types (keyof T = string) remain fully open. - assertType>>({ - type: NounType.Person, - subtype: 'employee', - data: 'x', - metadata: { anything: 'goes', confidence: 0.8 } - }) - }) - - it('UpdateParams patch rejects reserved keys but accepts partial custom patches', () => { - interface EmployeeMeta { - dept: string - level: number - } - - // Partial patch of the declared shape is legal. - assertType>({ id: 'e1', metadata: { dept: 'sales' } }) - // Untyped patch with custom fields is legal. - assertType({ id: 'e1', metadata: { status: 'reviewed', rating: 4.5 } }) - - // @ts-expect-error — 'confidence' is reserved (use the top-level 'confidence' param) - assertType({ id: 'e1', metadata: { confidence: 0.33 } }) - // @ts-expect-error — 'subtype' is reserved (use the top-level 'subtype' param) - assertType({ id: 'e1', metadata: { subtype: 'specialized' } }) - // @ts-expect-error — '_rev' is reserved (pass 'ifRev' for optimistic concurrency) - assertType({ id: 'e1', metadata: { _rev: 3 } }) - // @ts-expect-error — 'confidence' is reserved even when T declares other fields - assertType>({ id: 'e1', metadata: { confidence: 0.1 } }) - }) -}) - -describe('reserved relationship keys in metadata are compile errors', () => { - it('RelateParams rejects reserved keys but stays open for custom edge fields', () => { - assertType({ - from: 'a', - to: 'b', - type: VerbType.ReportsTo, - subtype: 'direct', - metadata: { role: 'peer', since: 2024 } - }) - - assertType({ - from: 'a', - to: 'b', - type: VerbType.ReportsTo, - subtype: 'direct', - // @ts-expect-error — 'verb' is reserved (the relationship type travels via the top-level 'type' param) - metadata: { verb: 'relatedTo' } - }) - assertType({ - from: 'a', - to: 'b', - type: VerbType.ReportsTo, - subtype: 'direct', - // @ts-expect-error — 'confidence' is reserved (use the top-level 'confidence' param) - metadata: { confidence: 0.9 } - }) - assertType({ - from: 'a', - to: 'b', - type: VerbType.ReportsTo, - subtype: 'direct', - // @ts-expect-error — 'weight' is reserved (use the top-level 'weight' param) - metadata: { weight: 0.4 } - }) - assertType({ - from: 'a', - to: 'b', - type: VerbType.ReportsTo, - subtype: 'direct', - // @ts-expect-error — 'service' is reserved (use the top-level 'service' param) - metadata: { service: 'orders' } - }) - }) - - it('UpdateRelationParams patch rejects reserved keys', () => { - assertType({ id: 'r1', metadata: { note: 'fine' } }) - - // @ts-expect-error — 'confidence' is reserved (use the top-level 'confidence' param) - assertType({ id: 'r1', metadata: { confidence: 0.5 } }) - // @ts-expect-error — 'subtype' is reserved (use the top-level 'subtype' param) - assertType({ id: 'r1', metadata: { subtype: 'dotted-line' } }) - // @ts-expect-error — 'createdAt' is reserved (system-managed) - assertType({ id: 'r1', metadata: { createdAt: 1 } }) - }) -}) - -describe('transact() operations inherit the same guard', () => { - it('TxOperation add/update/relate metadata rejects reserved keys', () => { - assertType({ - op: 'add', - type: NounType.Concept, - subtype: 'general', - data: 'tx', - metadata: { custom: 'a' } - }) - assertType({ - op: 'add', - type: NounType.Concept, - subtype: 'general', - data: 'tx', - // @ts-expect-error — 'confidence' is reserved on transact add ops too - metadata: { confidence: 0.7 } - }) - assertType({ - op: 'update', - id: 'e1', - // @ts-expect-error — 'weight' is reserved on transact update ops too - metadata: { weight: 0.2 } - }) - assertType({ - op: 'relate', - from: 'a', - to: 'b', - type: VerbType.RelatedTo, - subtype: 'colleague', - // @ts-expect-error — 'verb' is reserved on transact relate ops too - metadata: { verb: 'contains' } - }) - }) -}) diff --git a/tests/unit/utils/paramValidation.test.ts b/tests/unit/utils/paramValidation.test.ts index 4dc83554..7e5212b8 100644 --- a/tests/unit/utils/paramValidation.test.ts +++ b/tests/unit/utils/paramValidation.test.ts @@ -56,11 +56,15 @@ describe('Zero-Config Parameter Validation', () => { })).toThrow('cannot specify both query and vector') }) - it('should reject both cursor and offset', () => { + it('should refuse cursor outright — even paired with offset — as an unimplemented option', () => { + // cursor is now a typed, unconditional refusal (UnsupportedFindOptionError): + // it used to be accepted-and-ignored, only conflicting when offset was also + // given. Accepted-and-ignored died as a class — cursor refuses on its own, + // so pairing it with offset refuses too, but with the SAME message. expect(() => validateFindParams({ cursor: 'abc123', offset: 10 - })).toThrow('cannot use both cursor and offset pagination') + })).toThrow("find() option 'cursor' is not implemented") }) it('should validate vector dimensions', () => {