diff --git a/CHANGELOG.md b/CHANGELOG.md index 7b367663..e0e47015 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,12 +2,6 @@ All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines. -### [8.10.3](https://source.soulcraft.com/soulcraft/brainy/compare/v8.10.2...v8.10.3) (2026-08-03) - -- docs: dedupe the 8.10.2 release-notes entry the cherry doubled onto the branch (8c956608) -- fix: user metadata named 'level' is a real field everywhere — the engine-internal node layer no longer shadows it in sort/filter/aggregation, and the indexing views stop stamping a phantom 0 into its column; index epoch 2 rebuilds existing brains at first open (958a0859) - - ### [8.10.2](https://source.soulcraft.com/soulcraft/brainy/compare/v8.10.1...v8.10.2) (2026-07-29) - docs: 8.10.2 consumer release notes — update() write granularity, PathResolver idle-log fix, graph-lsm key recognition (a0123b5b) diff --git a/RELEASES.md b/RELEASES.md index ab0e0d58..e68bcca1 100644 --- a/RELEASES.md +++ b/RELEASES.md @@ -31,37 +31,6 @@ is sometimes cited as a 7.x removal — those methods never existed on 7.x; the --- -## v8.10.3 — 2026-08-03 (natural field names stop colliding with engine internals) - -From a production report: sorting by a user metadata field named `level` silently -returned insertion order — the engine's internal HNSW node layer (also called -`level`) shadowed the user's field in every by-name read, and the indexing path -stamped a hardcoded `0` into the same index column (multi-valued poison). `level` -is a perfectly natural field name (game characters, priorities, floors); the -engine was wrong, not the caller. - -- **`level` is user data now, everywhere.** Engine plumbing no longer resolves by - name, never shadows metadata, and never enters the indexed views. `orderBy: - 'level'`, `where: { level: 9 }`, `groupBy: ['level']` all read YOUR field. - Regression pins: `tests/integration/level-field-shadow.test.ts` (the reporting - consumer's exact repro rows). -- **Index epoch 2.** The derived posting set changed, so every existing brain - rebuilds its metadata index from canonical at first open — poisoned columns - heal automatically; no manual step. First open after upgrade pays one rebuild - (observable via `getIndexStatus()`); pair this release with the same-day - native-accelerator release, which makes `level` indexable on the native path. -- **`transact()` metadata-only updates stop rewriting the vector record** — the - v8.10.2 write-granularity law now covers the batch/plan path too (it was - fixed for `update()` but the transact plan builder still staged the - unconditional save). If you batch stat touches through `transact()`, this is - your write-amplification fix. -- Coming next (announced so parsers and call sites can prepare): one - field-addressing law — bare names = user metadata, `system.` for - engine fields, typed refusals for unresolvable names. Ships as its own - release with a migration advisory; nothing changes in this release. - ---- - ## v8.10.2 — 2026-07-29 (metadata-only updates stop rewriting the vector record) From a production incident on a large deployment: a read-heavy sweep that bumped diff --git a/package-lock.json b/package-lock.json index 823b2625..a0fb6f6d 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "@soulcraft/brainy", - "version": "8.10.3", + "version": "8.10.2", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@soulcraft/brainy", - "version": "8.10.3", + "version": "8.10.2", "license": "MIT", "dependencies": { "@msgpack/msgpack": "^3.1.2", diff --git a/package.json b/package.json index 0d8b266b..5d140743 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@soulcraft/brainy", - "version": "8.10.3", + "version": "8.10.2", "description": "Universal Knowledge Protocol™ - World's first Triple Intelligence database unifying vector, graph, and document search in one API. Stage 3 CANONICAL: 42 nouns × 127 verbs covering 96-97% of all human knowledge.", "main": "dist/index.js", "module": "dist/index.js", diff --git a/src/brainy.ts b/src/brainy.ts index 1a501f0e..1b5321e9 100644 --- a/src/brainy.ts +++ b/src/brainy.ts @@ -2123,13 +2123,11 @@ export class Brainy implements BrainyInterface { // If undefined values are included as explicit keys, extractIndexableFields indexes // them as '__NULL__' entries that removeFromIndex can never clean up (storageMetadata // omits those keys entirely via conditional spreading, so the fields don't match). - // No `level` here: engine plumbing never enters the indexing view — a - // hardcoded level:0 landed in the SAME flattened index column as user - // metadata named `level`, poisoning it multi-valued ([0, real]). const entityForIndexing = { id, vector, connections: new Map(), + level: 0, type: params.type, ...(params.subtype !== undefined && { subtype: params.subtype }), ...(params.visibility !== undefined && @@ -3104,13 +3102,12 @@ export class Brainy implements BrainyInterface { }) } - // Build entity structure for metadata index (with top-level fields). - // No `level`: engine plumbing never enters the indexing view (it - // poisoned the flattened user `level` column — VENUE-BRAINY-ORDERBY-NOOP). + // Build entity structure for metadata index (with top-level fields) const entityForIndexing = { id: params.id, vector, connections: new Map(), + level: 0, type: params.type || existing.type, subtype: params.subtype !== undefined ? params.subtype : existing.subtype, ...(((params.visibility ?? existing.visibility) ?? 'public') !== 'public' && { @@ -9341,7 +9338,7 @@ export class Brainy implements BrainyInterface { id, vector, connections: new Map(), - // no `level` — plumbing never enters the indexing view + level: 0, type: params.type, ...(params.subtype !== undefined && { subtype: params.subtype }), ...(params.visibility !== undefined && @@ -9492,7 +9489,7 @@ export class Brainy implements BrainyInterface { id: params.id, vector, connections: new Map(), - // no `level` — plumbing never enters the indexing view + level: 0, type: params.type || existing.type, subtype: params.subtype !== undefined ? params.subtype : existing.subtype, ...(((params.visibility ?? existing.visibility) ?? 'public') !== 'public' && { @@ -9520,22 +9517,16 @@ export class Brainy implements BrainyInterface { } plan.operations.push( - new UpdateNounMetadataOperation(this.storage, params.id, updatedMetadata) + new UpdateNounMetadataOperation(this.storage, params.id, updatedMetadata), + new SaveNounOperation(this.storage, { + id: params.id, + vector, + connections: new Map(), + level: 0 + }) ) - // Noun-record write + HNSW reindex ONLY when the vector side actually - // changed — the same write-granularity law as update(): a metadata-only - // patch must never rewrite the whole vector record. This plan path is the - // one transact() updates ride, so an unconditional save here would - // re-open the read-sweep disk-saturation amplifier for exactly the - // consumers batching their stat touches through transact(). if (needsReindexing) { plan.operations.push( - new SaveNounOperation(this.storage, { - id: params.id, - vector, - connections: new Map(), - level: 0 - }), new RemoveFromVectorIndexOperation(this.index, params.id, existing.vector), new AddToVectorIndexOperation(this.index, params.id, vector) ) diff --git a/src/coreTypes.ts b/src/coreTypes.ts index 90fc4462..e0248d17 100644 --- a/src/coreTypes.ts +++ b/src/coreTypes.ts @@ -284,12 +284,7 @@ export const STANDARD_ENTITY_FIELDS: ReadonlySet = new Set([ 'id', 'vector', 'connections', - // 'level' is deliberately ABSENT: it is HNSW plumbing, not an entity field. - // Listing it here made every by-name read of a user metadata field called - // `level` resolve to the engine's internal node layer instead — a silent - // shadow that broke sort/filter/aggregation on a perfectly natural field - // name (VENUE-BRAINY-ORDERBY-NOOP). Engine plumbing is invisible to the - // query surface; a bare `level` reads `entity.metadata.level`. + 'level', 'type', 'subtype', 'visibility', diff --git a/src/storage/brainFormat.ts b/src/storage/brainFormat.ts index a1241fe0..2e6488e9 100644 --- a/src/storage/brainFormat.ts +++ b/src/storage/brainFormat.ts @@ -69,12 +69,7 @@ export const BRAIN_FORMAT_PATH = '_system/brain-format.json' * (the 8.0 GA baseline). An on-disk `indexEpoch` that differs from this — or an * absent marker — triggers a full derived-index rebuild on open. */ -// Epoch 2 (2026-08-03, paired with the native accelerator's same-day release): -// user metadata fields named `level` become indexable on both engines — the -// derived posting set changed, so every pre-fix brain must rebuild its -// metadata index from canonical at first open (poisoned multi-valued `level` -// columns heal through this rebuild; no bespoke heal path). -export const EXPECTED_INDEX_EPOCH = 2 +export const EXPECTED_INDEX_EPOCH = 1 /** * @description The data-layer format string this build writes and runs as. diff --git a/src/types/brainy.types.ts b/src/types/brainy.types.ts index 89be78b9..b5f286ed 100644 --- a/src/types/brainy.types.ts +++ b/src/types/brainy.types.ts @@ -551,23 +551,7 @@ export interface FindParams { cursor?: string // Cursor-based pagination // Sorting - /** - * Field to sort by. User metadata fields sort by their stored values — - * including natural names like `level`, `rank`, or `score` (an engine-internal - * field can never shadow your metadata; fixed 2026-08 after a production - * report). System timestamps (`createdAt`, `updatedAt`) sort by entity age. - * - * Ordering contract (identical on the pure-JS engine and the native - * accelerator): entities missing the field sort LAST in both directions — - * they are never dropped from the result; ties break deterministically. - * - * NOTE — the field-addressing law is changing (announced 2026-08): bare - * names will mean user metadata ALWAYS, and system fields will be reached - * explicitly as `system.` (e.g. `system.createdAt`), with typed - * refusals for unresolvable names. Until that release, bare `createdAt` - * and friends keep resolving to the system fields as documented above. - */ - orderBy?: string + orderBy?: string // Field to sort by (e.g., 'createdAt', 'title', 'metadata.priority') order?: 'asc' | 'desc' // Sort direction: 'asc' (default) or 'desc' // Advanced options diff --git a/tests/integration/level-field-shadow.test.ts b/tests/integration/level-field-shadow.test.ts deleted file mode 100644 index d50593ff..00000000 --- a/tests/integration/level-field-shadow.test.ts +++ /dev/null @@ -1,147 +0,0 @@ -/** - * @module tests/integration/level-field-shadow - * @description The reserved-name shadow fix (VENUE-BRAINY-ORDERBY-NOOP, - * 2026-08-03): `level` is HNSW plumbing, not an entity field — it must never - * shadow user metadata of the same name. Pre-fix, STANDARD_ENTITY_FIELDS - * listed `level`, so every by-name read returned the engine's internal 0 - * (all-equal → stable sort → insertion order, silently), and the indexing - * views stamped level:0 into the same flattened column as user values - * (multi-valued [0, real] poison). Laws: - * (1) venue's exact repro sorts: three adds with metadata.level 3/9/6 → - * find({orderBy:'level'}) returns 9,6,3 desc and 3,6,9 asc; - * (2) where {level: N} matches through filter AND egress guard; - * (3) the index column carries the user value only (no 0 poison); - * (4) update() keeps `level` readable (the update indexing view is clean too); - * (5) the transact() update path never rewrites the noun record on a - * metadata-only patch (the planUpdate granularity completion). - */ -import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest' -import { Brainy } from '../../src/brainy.js' -import { NounType } from '../../src/types/graphTypes.js' -import { EXPECTED_INDEX_EPOCH } from '../../src/storage/brainFormat.js' - -const stubEmbedding = async (text: string): Promise => { - const hash = text.split('').reduce((acc, char) => acc + char.charCodeAt(0), 0) - return new Array(384).fill(0).map((_, i) => Math.sin(hash + i)) -} - -describe('level field shadow — user metadata named level is a real field', () => { - let brain: Brainy - - beforeEach(async () => { - brain = new Brainy({ - requireSubtype: false, - storage: { type: 'memory' as const }, - embeddingFunction: stubEmbedding - }) - await brain.init() - }) - - afterEach(async () => { - await brain.close() - }) - - async function addProbeRows(): Promise { - const ids: string[] = [] - for (const level of [3, 9, 6]) { - ids.push( - await brain.add({ - data: `probe character level ${level}`, - type: NounType.Person, - subtype: 'probe-char', - metadata: { name: `char-${level}`, level } - }) - ) - } - return ids - } - - it("venue's exact repro: orderBy 'level' sorts desc and asc", async () => { - await addProbeRows() - - const desc = await brain.find({ - type: NounType.Person, - subtype: 'probe-char', - orderBy: 'level', - order: 'desc', - limit: 100 - }) - expect(desc.map((r: any) => r.metadata?.level)).toEqual([9, 6, 3]) - - const asc = await brain.find({ - type: NounType.Person, - subtype: 'probe-char', - orderBy: 'level', - order: 'asc', - limit: 100 - }) - expect(asc.map((r: any) => r.metadata?.level)).toEqual([3, 6, 9]) - }) - - it('ordered reads are COMPLETE — no row dropped (the 2-of-3 face)', async () => { - const ids = await addProbeRows() - const desc = await brain.find({ - type: NounType.Person, - subtype: 'probe-char', - orderBy: 'level', - order: 'desc', - limit: 100 - }) - expect(desc).toHaveLength(3) - expect(new Set(desc.map((r: any) => r.id))).toEqual(new Set(ids)) - }) - - it('where {level: N} matches through the filter and the egress guard', async () => { - const ids = await addProbeRows() - const hit = await brain.find({ where: { level: 9 } }) - expect(hit).toHaveLength(1) - expect(hit[0].id).toBe(ids[1]) - expect(hit[0].metadata?.level).toBe(9) - }) - - it('the index column carries ONLY the user value (no 0 poison)', async () => { - const ids = await addProbeRows() - const metadataIndex = (brain as any).metadataIndex - const value = await metadataIndex.getFieldValueForEntity(ids[1], 'level') - expect(value).toBe(9) - - // Zero must not match anything — pre-fix every entity carried a phantom 0. - const phantom = await brain.find({ where: { level: 0 } }) - expect(phantom).toHaveLength(0) - }) - - it('update() keeps level readable (the update indexing view is clean)', async () => { - const ids = await addProbeRows() - await brain.update({ id: ids[0], metadata: { level: 12 } }) - const desc = await brain.find({ - type: NounType.Person, - subtype: 'probe-char', - orderBy: 'level', - order: 'desc', - limit: 100 - }) - expect(desc.map((r: any) => r.metadata?.level)).toEqual([12, 9, 6]) - }) - - it('transact() metadata-only update never rewrites the noun record', async () => { - const ids = await addProbeRows() - const storage = (brain as any).storage - const saveNounSpy = vi.spyOn(storage, 'saveNoun') - - await brain.transact([ - { op: 'update', id: ids[0], metadata: { level: 4 } }, - { op: 'update', id: ids[2], metadata: { level: 7 } } - ]) - - expect(saveNounSpy).not.toHaveBeenCalled() - saveNounSpy.mockRestore() - - const after = await brain.get(ids[0], { includeVectors: true }) - expect(after?.metadata?.level).toBe(4) - expect(Array.isArray(after?.vector) && after!.vector!.length).toBe(384) - }) - - it('this build runs index epoch 2 (the paired level-indexability rebuild)', () => { - expect(EXPECTED_INDEX_EPOCH).toBe(2) - }) -}) diff --git a/tests/integration/orderby-sort-bug.test.ts b/tests/integration/orderby-sort-bug.test.ts index db40fe12..9c28b1c9 100644 --- a/tests/integration/orderby-sort-bug.test.ts +++ b/tests/integration/orderby-sort-bug.test.ts @@ -215,6 +215,7 @@ describe('resolveEntityField helper', () => { 'id', 'vector', 'connections', + 'level', 'type', 'confidence', 'weight', @@ -227,9 +228,5 @@ describe('resolveEntityField helper', () => { for (const field of expected) { expect(STANDARD_ENTITY_FIELDS.has(field)).toBe(true) } - // `level` is deliberately NOT resolvable: it is HNSW plumbing, and listing - // it here shadowed user metadata named `level` in every by-name read - // (the reserved-name shadow bug). Plumbing stays out of the resolver. - expect(STANDARD_ENTITY_FIELDS.has('level')).toBe(false) }) }) diff --git a/tests/unit/brainy/migration-deference.test.ts b/tests/unit/brainy/migration-deference.test.ts index f03bba9c..6471b4ef 100644 --- a/tests/unit/brainy/migration-deference.test.ts +++ b/tests/unit/brainy/migration-deference.test.ts @@ -245,9 +245,7 @@ describe('rc.8 no-freeze migration deference (isMigrating / stampBrainFormat / b it('the brain-format marker module exports the compiled epoch + data-format constants', () => { // cor imports these from '@soulcraft/brainy/brain-format' (Hook 3) so both // sides share ONE source of truth — no duplicated constant to drift. - // Epoch 2: user metadata named `level` became indexable (the reserved-name - // shadow fix, 2026-08-03) — pre-fix brains rebuild derived indexes at open. - expect(EXPECTED_INDEX_EPOCH).toBe(2) + expect(EXPECTED_INDEX_EPOCH).toBe(1) expect(CURRENT_DATA_FORMAT).toBe('8.0') }) })