feat(namespace): the index speaks the frozen keys — record-frame scalars index under literal 'system.<field>' (legacy 'noun' spelling folds into system.type; plumbing never indexed from a record frame), user fields stay bare in every shape; filter + sorted paths route every address through parseFieldAddress; storage fallbacks read the addressed side of the record

src/utils/metadataIndex.ts

@@ -5,6 +5,7 @@
  */
 
 import { StorageAdapter, resolveEntityField, NounMetadata, VerbMetadata } from '../coreTypes.js'
+import { SYSTEM_ENTITY_SCALARS, parseFieldAddress } from '../db/fieldAddressing.js'
 import { ColumnStore } from '../indexes/columnStore/ColumnStore.js'
 import type { MetadataIndexProvider } from '../plugin.js'
 import { MetadataIndexCache, MetadataIndexCacheConfig } from './metadataIndexCache.js'
@@ -43,8 +44,8 @@ import { BrainyError } from '../errors/brainyError.js'
  * bucketed field is added (e.g. a compressed float), add it here too.
  */
 const BUCKETED_INDEX_FIELDS: ReadonlySet<string> = new Set([
-  'createdAt',
-  'updatedAt'
+  'system.createdAt',
+  'system.updatedAt'
 ])
 
 export interface MetadataIndexEntry {
@@ -1218,12 +1219,56 @@ export class MetadataIndexManager implements MetadataIndexProvider {
     // the reserved entity-identity field, resolved specially by find().)
     const NEVER_INDEX = new Set(['vector', 'embedding', 'embeddings', 'connections', 'id'])
 
-    const extract = (obj: any, prefix = ''): void => {
+    // THE FROZEN INDEX KEY FORMAT (cross-engine, sealed 2026-08-03; the native
+    // accelerator keys identically — epoch 3 rebuilds every brain onto it):
+    // user fields index under BARE keys exactly as the caller wrote them;
+    // the ten system scalars index under literal 'system.<field>' keys — the
+    // key IS the query address, so the two namespaces can never collide
+    // inside the index again. `origin` tracks which side of the record a key
+    // came from: 'record' = the entity/stored-record frame (system scalars,
+    // plumbing, and the metadata bag live here — the WRITE PATH's reserved-
+    // name remap guarantees a record-frame key matching a system name IS the
+    // system value); 'user' = inside the flattened metadata bag (everything
+    // is the user's, including natural names like `level` and `data`).
+    // Frame kinds: 'entity-record' = entityForIndexing shape (user fields
+    // nested under `metadata`; stray top-level keys are DROPPED, not guessed —
+    // epoch-3's rebuild-from-canonical normalizes historical shapes);
+    // 'flat-record' = the stored metadata-record shape (user fields FLAT
+    // beside the reserved ones — the write path's reserved-name remap
+    // guarantees a key matching a system name IS the system value, so
+    // non-system keys here are the user's and index bare); 'user' = inside
+    // the metadata bag (everything is the user's, including natural names
+    // like `level` and `data`).
+    type Frame = 'entity-record' | 'flat-record' | 'user'
+    const extract = (obj: any, prefix = '', frame: Frame = 'entity-record'): void => {
       for (const [key, value] of Object.entries(obj)) {
-        const fullKey = prefix ? `${prefix}.${key}` : key
+        let fullKey = prefix ? `${prefix}.${key}` : key
 
-        // Skip fields in never-index list (CRITICAL: prevents vector indexing bug + HNSW fields)
-        if (!prefix && NEVER_INDEX.has(key)) continue
+        if (!prefix && frame !== 'user') {
+          if (key === 'metadata' && typeof value === 'object' && value !== null && !Array.isArray(value)) {
+            extract(value, '', 'user') // the user's namespace: bare keys
+            continue
+          }
+          if (key === 'type' || key === 'noun') {
+            fullKey = 'system.type' // legacy 'noun' spelling folds into the frozen key
+          } else if (SYSTEM_ENTITY_SCALARS.has(key) && key !== 'id') {
+            fullKey = `system.${key}`
+          } else if (
+            key === 'data' || key === '_rev' || key === 'level' || NEVER_INDEX.has(key)
+          ) {
+            continue // plumbing / identity / bulk payloads — never indexed from a record frame
+          } else if (frame === 'entity-record') {
+            continue // stray entity-frame key: dropped, not guessed
+          }
+          // flat-record fallthrough: a non-system, non-plumbing key IS a user
+          // field (flat beside the reserved ones) — indexes bare via fullKey.
+        } else if (!prefix && NEVER_INDEX.has(key)) {
+          // User frame: only the bulk-payload guards apply — natural names
+          // like `level` and `data` are real user fields here. (`id` as a
+          // user metadata field remains un-indexed this train — documented
+          // limitation; system.id resolves via the id mapper, never a column.)
+          continue
+        }
 
         // Skip purely numeric field names (array indices converted to object keys)
         // Legitimate field names should never be purely numeric
@@ -1233,21 +1278,12 @@ export class MetadataIndexManager implements MetadataIndexProvider {
         // Skip fields based on user configuration
         if (!this.shouldIndexField(fullKey)) continue
 
-        // Special handling for metadata field at top level
-        // Flatten metadata fields to top-level (no prefix) for cleaner queries
-        // Standard fields are already at top-level, custom fields go in metadata
-        // By flattening here, queries can use { category: 'B' } instead of { 'metadata.category': 'B' }
-        if (key === 'metadata' && !prefix && typeof value === 'object' && !Array.isArray(value)) {
-          extract(value, '')  // Flatten to top-level, no prefix
-          continue
-        }
-
         // Skip large arrays (> 10 elements) - likely vectors or bulk data
         if (Array.isArray(value) && value.length > 10) continue
 
         if (value && typeof value === 'object' && !Array.isArray(value)) {
-          // Recurse into nested objects (but not arrays)
-          extract(value, fullKey)
+          // Recurse into nested objects (but not arrays), keeping the frame
+          extract(value, fullKey, frame)
         } else if (Array.isArray(value) && value.length <= 10) {
           // Small arrays: index as multi-value field (all with same field name)
           // Example: tags: ["javascript", "node"] → field="tags", value="javascript" + field="tags", value="node"
@@ -1258,16 +1294,21 @@ export class MetadataIndexManager implements MetadataIndexProvider {
             }
           }
         } else {
-          // Primitive value: index it
-          // Map 'type' → 'noun' for backward compatibility
-          const indexField = (!prefix && key === 'type') ? 'noun' : fullKey
-          fields.push({ field: indexField, value })
+          // Primitive value: index it under the frozen key computed above.
+          // (The legacy 'type'→'noun' remap is gone — 'noun' columns die at
+          // the epoch-3 rebuild; system.type is the one spelling.)
+          fields.push({ field: fullKey, value })
         }
       }
     }
 
     if (data && typeof data === 'object') {
-      extract(data)
+      // Shape detection for the top frame: an object carrying a nested
+      // `metadata` bag is the entityForIndexing shape; anything else is the
+      // flat stored-record shape (user fields flat beside reserved ones).
+      const entityShaped =
+        'metadata' in data && typeof data.metadata === 'object' && data.metadata !== null
+      extract(data, '', entityShaped ? 'entity-record' : 'flat-record')
     }
 
     // Extract words for hybrid text search
@@ -1911,22 +1952,15 @@ export class MetadataIndexManager implements MetadataIndexProvider {
       // Skip logical operators
       if (rawField === 'allOf' || rawField === 'anyOf' || rawField === 'not') continue
 
-      // Metadata is FLATTENED at index time (metadata.entry.title indexes as
-      // entry.title), so a `metadata.`-prefixed where key is almost always
-      // the caller spelling the STORAGE shape rather than the index shape.
-      // Accept both spellings: when the key as spelled is unindexed but its
-      // stripped spelling is, query the stripped one. A literal nested
-      // custom key named `metadata` still wins when indexed as spelled
-      // (checked first), so that rare shape keeps working.
-      let field = rawField
-      if (
-        rawField.startsWith('metadata.') &&
-        this.columnStore &&
-        !this.columnStore.hasField(rawField) &&
-        this.columnStore.hasField(rawField.slice('metadata.'.length))
-      ) {
-        field = rawField.slice('metadata.'.length)
-      }
+      // THE ONE ADDRESSING LAW (sealed 2026-08-03): every filter key routes
+      // through parseFieldAddress — bare and 'metadata.'-prefixed spellings
+      // address the user's fields (indexed under BARE keys), 'system.<field>'
+      // addresses the ten engine scalars (indexed under their literal
+      // 'system.<field>' keys). A malformed address (system.<not-in-map>,
+      // plumbing in the system spelling) throws typed BEFORE any index read —
+      // an accepted name either works or refuses.
+      const address = parseFieldAddress(rawField, 'entity')
+      const field = address.scope === 'system' ? `system.${address.field}` : address.field
 
       let fieldResults: string[] = []
 
@@ -2207,9 +2241,18 @@ export class MetadataIndexManager implements MetadataIndexProvider {
     order: 'asc' | 'desc' = 'asc',
     topK?: number
   ): Promise<string[]> {
+    // THE ONE ADDRESSING LAW — the orderBy address routes through the same
+    // parse the filter path uses (the historical asymmetry where the filter
+    // path understood 'metadata.' but the sorted path never did is dead).
+    // Bare / 'metadata.' → the user's bare index key; 'system.<field>' → the
+    // literal frozen key; malformed addresses throw typed before any read.
+    const orderAddress = parseFieldAddress(orderBy, 'entity')
+    const orderKey =
+      orderAddress.scope === 'system' ? `system.${orderAddress.field}` : orderAddress.field
+
     // Column store path: O(K log S) sort via k-way merge across segments.
     // No per-entity storage reads, no precision loss from bucketing.
-    if (this.columnStore && this.columnStore.hasField(orderBy)) {
+    if (this.columnStore && this.columnStore.hasField(orderKey)) {
       // Get filtered IDs from existing roaring bitmap path
       const hasFilter = filter && Object.keys(filter).length > 0
       const filteredIds = hasFilter ? await this.getIdsForFilter(filter) : []
@@ -2229,12 +2272,12 @@ export class MetadataIndexManager implements MetadataIndexProvider {
         // log K) heap, not a full sort materialization.
         const k = topK !== undefined ? Math.min(topK, filteredIds.length) : filteredIds.length
         sortedIntIds = await this.columnStore.filteredSortTopK(
-          filterBitmap, orderBy, order, k
+          filterBitmap, orderKey, order, k
         )
       } else {
         // Unfiltered sort — column store handles the full entity set efficiently
         sortedIntIds = await this.columnStore.sortTopK(
-          orderBy, order, topK !== undefined ? Math.min(topK, this.idMapper.size) : this.idMapper.size
+          orderKey, order, topK !== undefined ? Math.min(topK, this.idMapper.size) : this.idMapper.size
         )
       }
 
@@ -2255,7 +2298,7 @@ export class MetadataIndexManager implements MetadataIndexProvider {
 
     const idValuePairs: Array<{ id: string, value: any }> = []
     for (const id of filteredIds) {
-      const value = await this.getFieldValueForEntity(id, orderBy)
+      const value = await this.getFieldValueForEntity(id, orderKey)
       idValuePairs.push({ id, value })
     }
 
@@ -2320,10 +2363,17 @@ export class MetadataIndexManager implements MetadataIndexProvider {
    * @public (called from brainy.ts for sorted queries)
    */
   async getFieldValueForEntity(entityId: string, field: string): Promise<any> {
-    // Path 1: Bucketed fields need the actual value from storage.
+    // `field` arrives as a FROZEN INDEX KEY (bare = user metadata;
+    // 'system.<field>' = engine scalar). Storage fallbacks read the matching
+    // side of the record — a system key reads the record scalar, a bare key
+    // reads the user's metadata bag; the two can never shadow each other.
+    const systemInner = field.startsWith('system.') ? field.slice('system.'.length) : null
+
+    // Path 1: Bucketed fields need the actual (un-bucketed) value from storage.
     if (BUCKETED_INDEX_FIELDS.has(field)) {
       const noun = await this.storage.getNoun(entityId)
-      return noun ? resolveEntityField(noun, field) : undefined
+      if (!noun) return undefined
+      return (noun as unknown as Record<string, unknown>)[systemInner as string]
     }
 
     // Path 3 precondition: entity must be in the id mapper for bitmap lookup.
@@ -2340,7 +2390,11 @@ export class MetadataIndexManager implements MetadataIndexProvider {
     // yet indexed. resolveEntityField handles the shape contract.
     if (!sparseIndex) {
       const noun = await this.storage.getNoun(entityId)
-      return noun ? resolveEntityField(noun, field) : undefined
+      if (!noun) return undefined
+      if (systemInner !== null) {
+        return (noun as unknown as Record<string, unknown>)[systemInner]
+      }
+      return (noun as { metadata?: Record<string, unknown> }).metadata?.[field]
     }
 
     // Path 3: Search sparse index chunks for this entity's value.