fix: user metadata named 'level' is a real field everywhere — the engine-internal node layer no longer shadows it in sort/filter/aggregation, and the indexing views stop stamping a phantom 0 into its column; index epoch 2 rebuilds existing brains at first open
Some checks failed
CI / Node 22 (push) Successful in 12m17s
CI / Node 24 (push) Has been cancelled
CI / Bun (latest) (push) Has been cancelled

Also completes the v8.10.2 write-granularity law for the transact() plan
path: a metadata-only batch update never rewrites the vector-bearing noun
record (planUpdate staged the unconditional save the update() fix removed).
Seven pins in tests/integration/level-field-shadow.test.ts including the
reporting consumer's exact repro rows; orderBy JSDoc documents the ordering
contract and the announced field-addressing law.
This commit is contained in:
David Snelling 2026-08-03 11:57:32 -07:00
parent cb717be275
commit 1a09be0628
8 changed files with 259 additions and 17 deletions

View file

@ -2123,11 +2123,13 @@ export class Brainy<T = any> implements BrainyInterface<T> {
// If undefined values are included as explicit keys, extractIndexableFields indexes
// them as '__NULL__' entries that removeFromIndex can never clean up (storageMetadata
// omits those keys entirely via conditional spreading, so the fields don't match).
// No `level` here: engine plumbing never enters the indexing view — a
// hardcoded level:0 landed in the SAME flattened index column as user
// metadata named `level`, poisoning it multi-valued ([0, real]).
const entityForIndexing = {
id,
vector,
connections: new Map(),
level: 0,
type: params.type,
...(params.subtype !== undefined && { subtype: params.subtype }),
...(params.visibility !== undefined &&
@ -3102,12 +3104,13 @@ export class Brainy<T = any> implements BrainyInterface<T> {
})
}
// Build entity structure for metadata index (with top-level fields)
// Build entity structure for metadata index (with top-level fields).
// No `level`: engine plumbing never enters the indexing view (it
// poisoned the flattened user `level` column — VENUE-BRAINY-ORDERBY-NOOP).
const entityForIndexing = {
id: params.id,
vector,
connections: new Map(),
level: 0,
type: params.type || existing.type,
subtype: params.subtype !== undefined ? params.subtype : existing.subtype,
...(((params.visibility ?? existing.visibility) ?? 'public') !== 'public' && {
@ -9377,7 +9380,7 @@ export class Brainy<T = any> implements BrainyInterface<T> {
id,
vector,
connections: new Map(),
level: 0,
// no `level` — plumbing never enters the indexing view
type: params.type,
...(params.subtype !== undefined && { subtype: params.subtype }),
...(params.visibility !== undefined &&
@ -9528,7 +9531,7 @@ export class Brainy<T = any> implements BrainyInterface<T> {
id: params.id,
vector,
connections: new Map(),
level: 0,
// no `level` — plumbing never enters the indexing view
type: params.type || existing.type,
subtype: params.subtype !== undefined ? params.subtype : existing.subtype,
...(((params.visibility ?? existing.visibility) ?? 'public') !== 'public' && {
@ -9556,16 +9559,22 @@ export class Brainy<T = any> implements BrainyInterface<T> {
}
plan.operations.push(
new UpdateNounMetadataOperation(this.storage, params.id, updatedMetadata),
new SaveNounOperation(this.storage, {
id: params.id,
vector,
connections: new Map(),
level: 0
})
new UpdateNounMetadataOperation(this.storage, params.id, updatedMetadata)
)
// Noun-record write + HNSW reindex ONLY when the vector side actually
// changed — the same write-granularity law as update(): a metadata-only
// patch must never rewrite the whole vector record. This plan path is the
// one transact() updates ride, so an unconditional save here would
// re-open the read-sweep disk-saturation amplifier for exactly the
// consumers batching their stat touches through transact().
if (needsReindexing) {
plan.operations.push(
new SaveNounOperation(this.storage, {
id: params.id,
vector,
connections: new Map(),
level: 0
}),
new RemoveFromVectorIndexOperation(this.index, params.id, existing.vector),
new AddToVectorIndexOperation(this.index, params.id, vector)
)

View file

@ -284,7 +284,12 @@ export const STANDARD_ENTITY_FIELDS: ReadonlySet<string> = new Set([
'id',
'vector',
'connections',
'level',
// 'level' is deliberately ABSENT: it is HNSW plumbing, not an entity field.
// Listing it here made every by-name read of a user metadata field called
// `level` resolve to the engine's internal node layer instead — a silent
// shadow that broke sort/filter/aggregation on a perfectly natural field
// name (VENUE-BRAINY-ORDERBY-NOOP). Engine plumbing is invisible to the
// query surface; a bare `level` reads `entity.metadata.level`.
'type',
'subtype',
'visibility',

View file

@ -69,7 +69,12 @@ export const BRAIN_FORMAT_PATH = '_system/brain-format.json'
* (the 8.0 GA baseline). An on-disk `indexEpoch` that differs from this or an
* absent marker triggers a full derived-index rebuild on open.
*/
export const EXPECTED_INDEX_EPOCH = 1
// Epoch 2 (2026-08-03, paired with the native accelerator's same-day release):
// user metadata fields named `level` become indexable on both engines — the
// derived posting set changed, so every pre-fix brain must rebuild its
// metadata index from canonical at first open (poisoned multi-valued `level`
// columns heal through this rebuild; no bespoke heal path).
export const EXPECTED_INDEX_EPOCH = 2
/**
* @description The data-layer format string this build writes and runs as.

View file

@ -551,7 +551,23 @@ export interface FindParams<T = any> {
cursor?: string // Cursor-based pagination
// Sorting
orderBy?: string // Field to sort by (e.g., 'createdAt', 'title', 'metadata.priority')
/**
* Field to sort by. User metadata fields sort by their stored values
* including natural names like `level`, `rank`, or `score` (an engine-internal
* field can never shadow your metadata; fixed 2026-08 after a production
* report). System timestamps (`createdAt`, `updatedAt`) sort by entity age.
*
* Ordering contract (identical on the pure-JS engine and the native
* accelerator): entities missing the field sort LAST in both directions
* they are never dropped from the result; ties break deterministically.
*
* NOTE the field-addressing law is changing (announced 2026-08): bare
* names will mean user metadata ALWAYS, and system fields will be reached
* explicitly as `system.<field>` (e.g. `system.createdAt`), with typed
* refusals for unresolvable names. Until that release, bare `createdAt`
* and friends keep resolving to the system fields as documented above.
*/
orderBy?: string
order?: 'asc' | 'desc' // Sort direction: 'asc' (default) or 'desc'
// Advanced options