From 7492b6cb59362a88e3af8f739001a4e0926860d7 Mon Sep 17 00:00:00 2001 From: David Snelling Date: Mon, 3 Aug 2026 15:53:13 -0700 Subject: [PATCH] =?UTF-8?q?feat(namespace):=20aggregation=20reads=20under?= =?UTF-8?q?=20the=20law=20+=20epoch=203=20(the=20key-split=20rebuild)=20+?= =?UTF-8?q?=20THE=20ARMING=20COMMIT=20=E2=80=94=20the=20capability=20const?= =?UTF-8?q?ant,=20the=20law=20module,=20and=20the=20typed=20refusals=20exp?= =?UTF-8?q?ort=20from=20the=20package=20root;=20both=20engines'=20conforma?= =?UTF-8?q?nce=20suites=20light=20on=20this=20signal?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/aggregation/AggregationIndex.ts | 31 ++++++++++++++----- src/brainy.ts | 11 +++++-- src/db/fieldAddressing.ts | 8 +++++ src/index.ts | 19 ++++++++++++ src/storage/brainFormat.ts | 15 +++++---- tests/integration/level-field-shadow.test.ts | 4 +-- tests/unit/brainy/migration-deference.test.ts | 7 +++-- 7 files changed, 73 insertions(+), 22 deletions(-) diff --git a/src/aggregation/AggregationIndex.ts b/src/aggregation/AggregationIndex.ts index f9382218..407b1fe0 100644 --- a/src/aggregation/AggregationIndex.ts +++ b/src/aggregation/AggregationIndex.ts @@ -14,7 +14,22 @@ */ import type { StorageAdapter, HNSWNounWithMetadata } from '../coreTypes.js' -import { resolveEntityField } from '../coreTypes.js' +import { parseFieldAddress, readEntityFieldAddress } from '../db/fieldAddressing.js' +import type { HNSWNounWithMetadata as AddressedEntity } from '../coreTypes.js' + +/** + * Read a user-supplied field name under the one addressing law (sealed + * 2026-08-03): bare / `metadata.` = the user's metadata field, `system.` = + * the ruled engine scalar, malformed = typed refusal. The aggregation engine + * NEVER resolves names any other way — the pre-law resolver made bare + * `subtype`/`confidence` read engine scalars, silently shadowing user fields. + */ +function readAddressed(e: unknown, name: string): unknown { + return readEntityFieldAddress( + e as AddressedEntity, + parseFieldAddress(name, 'entity') + ) +} import type { AggregateDefinition, AggregateGroupState, @@ -97,7 +112,7 @@ function matchesSource(entity: Record, source: AggregateDefinit const e = entity as unknown as HNSWNounWithMetadata const resolved: Record = {} for (const key of Object.keys(source.where)) { - resolved[key] = resolveEntityField(e, key) + resolved[key] = readAddressed(e, key) } if (!matchesMetadataFilter(resolved, source.where)) return false } @@ -129,11 +144,11 @@ function computeGroupKeys( for (const dim of groupBy) { if (typeof dim === 'string') { - const val = resolveEntityField(e, dim) + const val = readAddressed(e, dim) const v = val !== undefined && val !== null ? String(val) : '__null__' for (const k of keys) k[dim] = v } else if ('unnest' in dim) { - const val = resolveEntityField(e, dim.field) + const val = readAddressed(e, dim.field) const raw = Array.isArray(val) ? val : val !== undefined && val !== null ? [val] : [] // Distinct elements: an entity with duplicate tags counts once per distinct tag. const elems = Array.from(new Set(raw.map(x => String(x)))) @@ -145,7 +160,7 @@ function computeGroupKeys( keys = next } else { // Time-windowed field - const val = resolveEntityField(e, dim.field) + const val = readAddressed(e, dim.field) const v = typeof val === 'number' ? bucketTimestamp(val, dim.window) : '__null__' for (const k of keys) k[dim.field] = v } @@ -174,7 +189,7 @@ function computeGroupKey( * in metadata are both handled in one place. */ function getNumericField(entity: Record, field: string): number | undefined { - const val = resolveEntityField(entity as unknown as HNSWNounWithMetadata, field) + const val = readAddressed(entity as unknown as HNSWNounWithMetadata, field) if (typeof val === 'number' && !isNaN(val)) return val if (typeof val === 'string') { const num = parseFloat(val) @@ -990,7 +1005,7 @@ export class AggregationIndex { // distinctCount tracks distinct values of ANY type (strings, numbers, booleans), // keyed by their string form — NOT numeric-coerced, since its primary use is // categorical (distinct categories / users / tags), not numeric columns. - const raw = resolveEntityField(entity as unknown as HNSWNounWithMetadata, metricDef.field!) + const raw = readAddressed(entity as unknown as HNSWNounWithMetadata, metricDef.field!) if (raw !== undefined && raw !== null) { if (!state.valueCounts) state.valueCounts = {} const key = String(raw) @@ -1034,7 +1049,7 @@ export class AggregationIndex { state.count = Math.max(0, state.count - 1) state.sum = Math.max(0, state.sum - 1) } else if (metricDef.op === 'distinctCount') { - const raw = resolveEntityField(entity as unknown as HNSWNounWithMetadata, metricDef.field!) + const raw = readAddressed(entity as unknown as HNSWNounWithMetadata, metricDef.field!) if (raw !== undefined && raw !== null && state.valueCounts) { const key = String(raw) const c = state.valueCounts[key] diff --git a/src/brainy.ts b/src/brainy.ts index a8df56bd..3dd8ef93 100644 --- a/src/brainy.ts +++ b/src/brainy.ts @@ -5619,7 +5619,7 @@ export class Brainy implements BrainyInterface { this._aggregationIndex!.defineAggregate({ name: aggregateName, source: {}, - groupBy: perType ? [name, 'noun'] : [name], + groupBy: perType ? [name, 'system.type'] : [name], metrics: { count: { op: 'count' } } }) } @@ -5630,7 +5630,12 @@ export class Brainy implements BrainyInterface { * and `counts.byField()` agree on the convention. */ private fieldCountsAggregateName(name: string): string { - return `__fieldCounts__${name}` + // v2 suffix: the per-type dimension moved from the legacy 'noun' alias to + // 'system.type' under the addressing law — a NEW name makes the ensure + // block re-define and BACKFILL from canonical instead of silently serving + // the old-dim definition (whose 'noun' key now reads user metadata and + // would drift). The v1 rows are derived state, superseded not lost. + return `__fieldCounts_v2__${name}` } /** @@ -11852,7 +11857,7 @@ export class Brainy implements BrainyInterface { // don't have the tracked field at all (e.g. the VFS root) bucket under // '__null__' and would otherwise pollute the count map. if (value === undefined || value === null || value === '__null__') continue - if (options?.type !== undefined && row.groupKey?.['noun'] !== options.type) continue + if (options?.type !== undefined && row.groupKey?.['system.type'] !== options.type) continue const key = String(value) result[key] = (result[key] || 0) + (typeof row.metrics?.count === 'number' ? row.metrics.count : row.count) } diff --git a/src/db/fieldAddressing.ts b/src/db/fieldAddressing.ts index cd63871c..056ba3f2 100644 --- a/src/db/fieldAddressing.ts +++ b/src/db/fieldAddressing.ts @@ -283,3 +283,11 @@ export class UnsupportedFindOptionError extends Error { this.option = option } } + +/** + * @description The capability signal both engines' conformance suites arm on + * (never a version guess): its presence at the package root means the one + * field-addressing law is LIVE on every query surface — bare = user metadata, + * `system.*` = the ruled scalars, plumbing invisible, refusals typed. + */ +export const FIELD_ADDRESSING_CAPABILITY = 'field-addressing/v1' diff --git a/src/index.ts b/src/index.ts index ae8eef5c..3876a903 100644 --- a/src/index.ts +++ b/src/index.ts @@ -106,6 +106,25 @@ export type { // Export Aggregation Engine export { AggregationIndex, AggregateMaterializer, bucketTimestamp, parseBucketRange } from './aggregation/index.js' +// THE ONE FIELD-ADDRESSING LAW (sealed 2026-08-03) — the arming surface both +// engines' conformance suites detect: bare names = user metadata, system.* = +// the ten ruled scalars, plumbing invisible, refusals typed with the fix in +// the message. See docs/concepts/field-addressing.md. +export { + FIELD_ADDRESSING_CAPABILITY, + SYSTEM_ENTITY_SCALARS, + SYSTEM_RELATION_SCALARS, + PLUMBING_FIELDS, + parseFieldAddress, + readEntityFieldAddress, + readRelationFieldAddress, + buildUnresolvableMessage, + InvalidFieldAddressError, + UnresolvableFieldError, + UnsupportedFindOptionError +} from './db/fieldAddressing.js' +export type { FieldAddress, FieldAddressKind } from './db/fieldAddressing.js' + // Export Neural Import (AI data understanding) export { NeuralImport } from './neural/neuralImport.js' export type { diff --git a/src/storage/brainFormat.ts b/src/storage/brainFormat.ts index a1241fe0..6ef913d4 100644 --- a/src/storage/brainFormat.ts +++ b/src/storage/brainFormat.ts @@ -69,12 +69,15 @@ export const BRAIN_FORMAT_PATH = '_system/brain-format.json' * (the 8.0 GA baseline). An on-disk `indexEpoch` that differs from this — or an * absent marker — triggers a full derived-index rebuild on open. */ -// Epoch 2 (2026-08-03, paired with the native accelerator's same-day release): -// user metadata fields named `level` become indexable on both engines — the -// derived posting set changed, so every pre-fix brain must rebuild its -// metadata index from canonical at first open (poisoned multi-valued `level` -// columns heal through this rebuild; no bespoke heal path). -export const EXPECTED_INDEX_EPOCH = 2 +// Epoch 3 (2026-08-03, the namespace-law pair): the index key format split +// the two namespaces — user fields keep bare flattened keys, the ten system +// scalars moved to literal 'system.' keys (the legacy 'noun' column +// spelling died with them). Every brain rebuilds its derived indexes from +// canonical at first open onto the frozen keys. +// Epoch 2 (2026-08-03, same day, the interim pair): user metadata fields +// named `level` became indexable on both engines; poisoned multi-valued +// `level` columns healed through the rebuild. +export const EXPECTED_INDEX_EPOCH = 3 /** * @description The data-layer format string this build writes and runs as. diff --git a/tests/integration/level-field-shadow.test.ts b/tests/integration/level-field-shadow.test.ts index ab5ffb9a..cfe34c13 100644 --- a/tests/integration/level-field-shadow.test.ts +++ b/tests/integration/level-field-shadow.test.ts @@ -141,8 +141,8 @@ describe('level field shadow — user metadata named level is a real field', () expect(Array.isArray(after?.vector) && after!.vector!.length).toBe(384) }) - it('this build runs index epoch 2 (the paired level-indexability rebuild)', () => { - expect(EXPECTED_INDEX_EPOCH).toBe(2) + it('this build runs index epoch 3 (the namespace-law key split rebuild)', () => { + expect(EXPECTED_INDEX_EPOCH).toBe(3) }) }) diff --git a/tests/unit/brainy/migration-deference.test.ts b/tests/unit/brainy/migration-deference.test.ts index f03bba9c..5968c620 100644 --- a/tests/unit/brainy/migration-deference.test.ts +++ b/tests/unit/brainy/migration-deference.test.ts @@ -245,9 +245,10 @@ describe('rc.8 no-freeze migration deference (isMigrating / stampBrainFormat / b it('the brain-format marker module exports the compiled epoch + data-format constants', () => { // cor imports these from '@soulcraft/brainy/brain-format' (Hook 3) so both // sides share ONE source of truth — no duplicated constant to drift. - // Epoch 2: user metadata named `level` became indexable (the reserved-name - // shadow fix, 2026-08-03) — pre-fix brains rebuild derived indexes at open. - expect(EXPECTED_INDEX_EPOCH).toBe(2) + // Epoch 3: the namespace-law key split (bare user keys · literal + // 'system.' scalars, 2026-08-03) — every brain rebuilds onto the + // frozen keys at first open. (Epoch 2 same day: `level` indexability.) + expect(EXPECTED_INDEX_EPOCH).toBe(3) expect(CURRENT_DATA_FORMAT).toBe('8.0') }) })