fix: user metadata named 'level' is a real field everywhere — the engine-internal node layer no longer shadows it in sort/filter/aggregation, and the indexing views stop stamping a phantom 0 into its column; index epoch 2 rebuilds existing brains at first open
Also completes the v8.10.2 write-granularity law for the transact() plan path: a metadata-only batch update never rewrites the vector-bearing noun record (planUpdate staged the unconditional save the update() fix removed). Seven pins in tests/integration/level-field-shadow.test.ts including the reporting consumer's exact repro rows; orderBy JSDoc documents the ordering contract and the announced field-addressing law.
This commit is contained in:
parent
cb717be275
commit
1a09be0628
8 changed files with 259 additions and 17 deletions
|
|
@ -2123,11 +2123,13 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
|||
// If undefined values are included as explicit keys, extractIndexableFields indexes
|
||||
// them as '__NULL__' entries that removeFromIndex can never clean up (storageMetadata
|
||||
// omits those keys entirely via conditional spreading, so the fields don't match).
|
||||
// No `level` here: engine plumbing never enters the indexing view — a
|
||||
// hardcoded level:0 landed in the SAME flattened index column as user
|
||||
// metadata named `level`, poisoning it multi-valued ([0, real]).
|
||||
const entityForIndexing = {
|
||||
id,
|
||||
vector,
|
||||
connections: new Map(),
|
||||
level: 0,
|
||||
type: params.type,
|
||||
...(params.subtype !== undefined && { subtype: params.subtype }),
|
||||
...(params.visibility !== undefined &&
|
||||
|
|
@ -3102,12 +3104,13 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
|||
})
|
||||
}
|
||||
|
||||
// Build entity structure for metadata index (with top-level fields)
|
||||
// Build entity structure for metadata index (with top-level fields).
|
||||
// No `level`: engine plumbing never enters the indexing view (it
|
||||
// poisoned the flattened user `level` column — VENUE-BRAINY-ORDERBY-NOOP).
|
||||
const entityForIndexing = {
|
||||
id: params.id,
|
||||
vector,
|
||||
connections: new Map(),
|
||||
level: 0,
|
||||
type: params.type || existing.type,
|
||||
subtype: params.subtype !== undefined ? params.subtype : existing.subtype,
|
||||
...(((params.visibility ?? existing.visibility) ?? 'public') !== 'public' && {
|
||||
|
|
@ -9377,7 +9380,7 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
|||
id,
|
||||
vector,
|
||||
connections: new Map(),
|
||||
level: 0,
|
||||
// no `level` — plumbing never enters the indexing view
|
||||
type: params.type,
|
||||
...(params.subtype !== undefined && { subtype: params.subtype }),
|
||||
...(params.visibility !== undefined &&
|
||||
|
|
@ -9528,7 +9531,7 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
|||
id: params.id,
|
||||
vector,
|
||||
connections: new Map(),
|
||||
level: 0,
|
||||
// no `level` — plumbing never enters the indexing view
|
||||
type: params.type || existing.type,
|
||||
subtype: params.subtype !== undefined ? params.subtype : existing.subtype,
|
||||
...(((params.visibility ?? existing.visibility) ?? 'public') !== 'public' && {
|
||||
|
|
@ -9556,16 +9559,22 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
|||
}
|
||||
|
||||
plan.operations.push(
|
||||
new UpdateNounMetadataOperation(this.storage, params.id, updatedMetadata),
|
||||
new SaveNounOperation(this.storage, {
|
||||
id: params.id,
|
||||
vector,
|
||||
connections: new Map(),
|
||||
level: 0
|
||||
})
|
||||
new UpdateNounMetadataOperation(this.storage, params.id, updatedMetadata)
|
||||
)
|
||||
// Noun-record write + HNSW reindex ONLY when the vector side actually
|
||||
// changed — the same write-granularity law as update(): a metadata-only
|
||||
// patch must never rewrite the whole vector record. This plan path is the
|
||||
// one transact() updates ride, so an unconditional save here would
|
||||
// re-open the read-sweep disk-saturation amplifier for exactly the
|
||||
// consumers batching their stat touches through transact().
|
||||
if (needsReindexing) {
|
||||
plan.operations.push(
|
||||
new SaveNounOperation(this.storage, {
|
||||
id: params.id,
|
||||
vector,
|
||||
connections: new Map(),
|
||||
level: 0
|
||||
}),
|
||||
new RemoveFromVectorIndexOperation(this.index, params.id, existing.vector),
|
||||
new AddToVectorIndexOperation(this.index, params.id, vector)
|
||||
)
|
||||
|
|
|
|||
|
|
@ -284,7 +284,12 @@ export const STANDARD_ENTITY_FIELDS: ReadonlySet<string> = new Set([
|
|||
'id',
|
||||
'vector',
|
||||
'connections',
|
||||
'level',
|
||||
// 'level' is deliberately ABSENT: it is HNSW plumbing, not an entity field.
|
||||
// Listing it here made every by-name read of a user metadata field called
|
||||
// `level` resolve to the engine's internal node layer instead — a silent
|
||||
// shadow that broke sort/filter/aggregation on a perfectly natural field
|
||||
// name (VENUE-BRAINY-ORDERBY-NOOP). Engine plumbing is invisible to the
|
||||
// query surface; a bare `level` reads `entity.metadata.level`.
|
||||
'type',
|
||||
'subtype',
|
||||
'visibility',
|
||||
|
|
|
|||
|
|
@ -69,7 +69,12 @@ export const BRAIN_FORMAT_PATH = '_system/brain-format.json'
|
|||
* (the 8.0 GA baseline). An on-disk `indexEpoch` that differs from this — or an
|
||||
* absent marker — triggers a full derived-index rebuild on open.
|
||||
*/
|
||||
export const EXPECTED_INDEX_EPOCH = 1
|
||||
// Epoch 2 (2026-08-03, paired with the native accelerator's same-day release):
|
||||
// user metadata fields named `level` become indexable on both engines — the
|
||||
// derived posting set changed, so every pre-fix brain must rebuild its
|
||||
// metadata index from canonical at first open (poisoned multi-valued `level`
|
||||
// columns heal through this rebuild; no bespoke heal path).
|
||||
export const EXPECTED_INDEX_EPOCH = 2
|
||||
|
||||
/**
|
||||
* @description The data-layer format string this build writes and runs as.
|
||||
|
|
|
|||
|
|
@ -551,7 +551,23 @@ export interface FindParams<T = any> {
|
|||
cursor?: string // Cursor-based pagination
|
||||
|
||||
// Sorting
|
||||
orderBy?: string // Field to sort by (e.g., 'createdAt', 'title', 'metadata.priority')
|
||||
/**
|
||||
* Field to sort by. User metadata fields sort by their stored values —
|
||||
* including natural names like `level`, `rank`, or `score` (an engine-internal
|
||||
* field can never shadow your metadata; fixed 2026-08 after a production
|
||||
* report). System timestamps (`createdAt`, `updatedAt`) sort by entity age.
|
||||
*
|
||||
* Ordering contract (identical on the pure-JS engine and the native
|
||||
* accelerator): entities missing the field sort LAST in both directions —
|
||||
* they are never dropped from the result; ties break deterministically.
|
||||
*
|
||||
* NOTE — the field-addressing law is changing (announced 2026-08): bare
|
||||
* names will mean user metadata ALWAYS, and system fields will be reached
|
||||
* explicitly as `system.<field>` (e.g. `system.createdAt`), with typed
|
||||
* refusals for unresolvable names. Until that release, bare `createdAt`
|
||||
* and friends keep resolving to the system fields as documented above.
|
||||
*/
|
||||
orderBy?: string
|
||||
order?: 'asc' | 'desc' // Sort direction: 'asc' (default) or 'desc'
|
||||
|
||||
// Advanced options
|
||||
|
|
|
|||
Reference in a new issue